@vellumai/cli 0.7.1 → 0.7.3

package/src/lib/statefulset.ts

@@ -0,0 +1,375 @@
+/**
+ * Declarative StatefulSet spec for the assistant service group.
+ *
+ * Defines the static topology of all three containers (assistant, gateway,
+ * credential-executor), their volumes, ports, and env vars. Used by both
+ * the hatch and upgrade flows to build `docker run` argument arrays.
+ */
+
+import { existsSync } from "fs";
+
+import {
+  ASSISTANT_INTERNAL_PORT,
+  GATEWAY_INTERNAL_PORT,
+} from "./environments/paths.js";
+import { PROVIDER_ENV_VAR_NAMES } from "../shared/provider-env-vars.js";
+
+const AVATAR_DEVICE_ENV_VAR = "VELLUM_AVATAR_DEVICE";
+
+/** Logical service name used throughout the CLI. */
+export type ServiceName = "assistant" | "gateway" | "credential-executor";
+
+/**
+ /**
+  * The four fields from `dockerResourceNames()` that the builder actually uses.
+  * Container/network names come from here; volume names are generated by the
+  * `volumeClaimTemplates` in the spec. `ReturnType<typeof dockerResourceNames>`
+  * structurally satisfies this interface (it has all these fields plus more).
+  */
+ export interface DockerResourceNames {
+   assistantContainer: string;
+   cesContainer: string;
+   gatewayContainer: string;
+   network: string;
+ }
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** A static env var whose value is known at spec-definition time. */
+interface StaticEnv {
+  kind: "static";
+  name: string;
+  value: string;
+}
+
+/**
+ * An env var whose value comes from the opts object passed to
+ * `buildServiceRunArgs` at container-start time.
+ */
+interface SecretEnv {
+  kind: "secret";
+  name: string;
+  /** Key in `DockerRunSecrets`. */
+  secret: keyof DockerRunSecrets;
+}
+
+/**
+ * An env var conditionally forwarded from the host process environment.
+ * Only emitted when `process.env[hostVar]` is set.
+ * Defaults to `name` for `hostVar` when omitted.
+ */
+interface HostEnv {
+  kind: "host";
+  name: string;
+  hostVar?: string;
+}
+
+type EnvEntry = StaticEnv | SecretEnv | HostEnv;
+
+interface VolumeMount {
+  /** Logical volume name — maps to a Docker volume via `DockerVolumeClaimTemplate`. */
+  volumeName: string;
+  mountPath: string;
+  readOnly?: boolean;
+}
+
+interface PortSpec {
+  containerPort: number;
+  /**
+   * Host-side port. Literal string = use as-is. `"{{ gatewayPort }}"` is
+   * a sentinel replaced with the instance-specific gateway port at build time.
+   */
+  hostPort?: string;
+  description?: string;
+}
+
+export interface DockerContainerSpec {
+  /** K8s-style container name (matches `stateful_template.yaml`). */
+  name: string;
+  /** Internal CLI `ServiceName` key used by `SERVICE_START_ORDER`. */
+  internalName: ServiceName;
+  /**
+   * Network mode:
+   * - "bridge": owns the network (assistant only — gateway port published here).
+   * - "container": share the assistant container's network namespace.
+   */
+  network: "bridge" | "container";
+  ports?: PortSpec[];
+  env: EnvEntry[];
+  volumeMounts: VolumeMount[];
+}
+
+export interface DockerVolumeClaimTemplate {
+  name: string;
+  dockerVolume: (instanceName: string) => string;
+}
+
+export interface DockerStatefulSetSpec {
+  containers: DockerContainerSpec[];
+  startOrder: ServiceName[];
+  volumeClaimTemplates: DockerVolumeClaimTemplate[];
+  readiness: {
+    endpoint: string;
+    timeoutMs: number;
+    intervalMs: number;
+  };
+}
+
+/** Secrets injected at container-start time (from hatch / upgrade opts). */
+export interface DockerRunSecrets {
+  signingKey?: string;
+  bootstrapSecret?: string;
+  cesServiceToken?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Spec
+// ---------------------------------------------------------------------------
+
+export const DOCKER_STATEFUL_SET_SPEC: DockerStatefulSetSpec = {
+  startOrder: ["assistant", "gateway", "credential-executor"],
+
+  readiness: {
+    endpoint: "/readyz",
+    timeoutMs: 180_000,
+    intervalMs: 1_000,
+  },
+
+  volumeClaimTemplates: [
+    { name: "assistant-workspace",  dockerVolume: (n) => `${n}-workspace` },
+    { name: "ces-bootstrap-socket", dockerVolume: (n) => `${n}-socket` },
+    { name: "assistant-ipc-socket", dockerVolume: (n) => `${n}-assistant-ipc` },
+    { name: "gateway-ipc-socket",   dockerVolume: (n) => `${n}-gateway-ipc` },
+    { name: "gateway-security",     dockerVolume: (n) => `${n}-gateway-sec` },
+    { name: "ces-security",         dockerVolume: (n) => `${n}-ces-sec` },
+  ],
+
+  containers: [
+    {
+      name: "assistant-container",
+      internalName: "assistant",
+      network: "bridge",
+      ports: [
+        {
+          containerPort: GATEWAY_INTERNAL_PORT,
+          hostPort: "{{ gatewayPort }}",
+          description: "Gateway reverse-proxy (published to host)",
+        },
+        {
+          containerPort: ASSISTANT_INTERNAL_PORT,
+          hostPort: `${ASSISTANT_INTERNAL_PORT}`,
+          description: "Assistant HTTP API",
+        },
+      ],
+      env: [
+        { kind: "static", name: "IS_CONTAINERIZED",         value: "true" },
+        { kind: "static", name: "DEBUG_STDOUT_LOGS",         value: "1" },
+        { kind: "static", name: "VELLUM_CLOUD",              value: "docker" },
+        { kind: "static", name: "RUNTIME_HTTP_HOST",         value: "0.0.0.0" },
+        { kind: "static", name: "VELLUM_WORKSPACE_DIR",      value: "/workspace" },
+        { kind: "static", name: "VELLUM_BACKUP_DIR",         value: "/workspace/.backups" },
+        { kind: "static", name: "VELLUM_BACKUP_KEY_PATH",    value: "/workspace/.backup.key" },
+        { kind: "static", name: "CES_CREDENTIAL_URL",        value: "http://localhost:8090" },
+        { kind: "static", name: "GATEWAY_IPC_SOCKET_DIR",    value: "/run/gateway-ipc" },
+        { kind: "static", name: "ASSISTANT_IPC_SOCKET_DIR",  value: "/run/assistant-ipc" },
+        { kind: "secret", name: "CES_SERVICE_TOKEN",         secret: "cesServiceToken" },
+        { kind: "secret", name: "ACTOR_TOKEN_SIGNING_KEY",   secret: "signingKey" },
+        { kind: "secret", name: "GUARDIAN_BOOTSTRAP_SECRET", secret: "bootstrapSecret" },
+        // Provider API keys — forwarded from host if set
+        ...Object.values(PROVIDER_ENV_VAR_NAMES).map(
+          (v): HostEnv => ({ kind: "host", name: v }),
+        ),
+        { kind: "host", name: "VELLUM_ENVIRONMENT" },
+        { kind: "host", name: "VELLUM_PLATFORM_URL" },
+      ],
+      volumeMounts: [
+        { volumeName: "assistant-workspace",  mountPath: "/workspace" },
+        { volumeName: "ces-bootstrap-socket", mountPath: "/run/ces-bootstrap" },
+        { volumeName: "assistant-ipc-socket", mountPath: "/run/assistant-ipc" },
+        { volumeName: "gateway-ipc-socket",   mountPath: "/run/gateway-ipc" },
+      ],
+    },
+
+    {
+      name: "gateway-sidecar",
+      internalName: "gateway",
+      network: "container",
+      env: [
+        { kind: "static", name: "VELLUM_WORKSPACE_DIR",      value: "/workspace" },
+        { kind: "static", name: "GATEWAY_SECURITY_DIR",      value: "/gateway-security" },
+        { kind: "static", name: "ASSISTANT_HOST",            value: "localhost" },
+        { kind: "static", name: "CES_CREDENTIAL_URL",        value: "http://localhost:8090" },
+        { kind: "static", name: "GATEWAY_IPC_SOCKET_DIR",    value: "/run/gateway-ipc" },
+        { kind: "static", name: "ASSISTANT_IPC_SOCKET_DIR",  value: "/run/assistant-ipc" },
+        { kind: "static", name: "GATEWAY_PORT",              value: `${GATEWAY_INTERNAL_PORT}` },
+        { kind: "static", name: "RUNTIME_HTTP_PORT",         value: `${ASSISTANT_INTERNAL_PORT}` },
+        { kind: "secret", name: "CES_SERVICE_TOKEN",         secret: "cesServiceToken" },
+        { kind: "secret", name: "ACTOR_TOKEN_SIGNING_KEY",   secret: "signingKey" },
+        { kind: "secret", name: "GUARDIAN_BOOTSTRAP_SECRET", secret: "bootstrapSecret" },
+        { kind: "host",   name: "VELLUM_ENVIRONMENT" },
+        { kind: "host",   name: "VELLUM_PLATFORM_URL" },
+        { kind: "host",   name: "VELAY_BASE_URL" },
+      ],
+      volumeMounts: [
+        { volumeName: "assistant-workspace",  mountPath: "/workspace" },
+        { volumeName: "gateway-security",     mountPath: "/gateway-security" },
+        { volumeName: "assistant-ipc-socket", mountPath: "/run/assistant-ipc" },
+        { volumeName: "gateway-ipc-socket",   mountPath: "/run/gateway-ipc" },
+      ],
+    },
+
+    {
+      name: "credential-executor-sidecar",
+      internalName: "credential-executor",
+      network: "container",
+      env: [
+        { kind: "static", name: "CES_MODE",                  value: "managed" },
+        { kind: "static", name: "VELLUM_WORKSPACE_DIR",      value: "/workspace" },
+        { kind: "static", name: "CES_BOOTSTRAP_SOCKET_DIR",  value: "/run/ces-bootstrap" },
+        { kind: "static", name: "CREDENTIAL_SECURITY_DIR",   value: "/ces-security" },
+        { kind: "secret", name: "CES_SERVICE_TOKEN",         secret: "cesServiceToken" },
+      ],
+      volumeMounts: [
+        { volumeName: "assistant-workspace",  mountPath: "/workspace", readOnly: true },
+        { volumeName: "ces-bootstrap-socket", mountPath: "/run/ces-bootstrap" },
+        { volumeName: "ces-security",         mountPath: "/ces-security" },
+      ],
+    },
+  ],
+};
+
+// ---------------------------------------------------------------------------
+// Builder
+// ---------------------------------------------------------------------------
+
+export interface BuildServiceRunArgsOpts extends DockerRunSecrets {
+  gatewayPort: number;
+  imageTags: Record<ServiceName, string>;
+  instanceName: string;
+  res: DockerResourceNames;
+  extraAssistantEnv?: Record<string, string>;
+  defaultWorkspaceConfigPath?: string;
+  /** Avatar device path, if available. Injected by `docker.ts` after resolving. */
+  avatarDevicePath?: string;
+}
+
+function resolveVolume(
+  spec: DockerStatefulSetSpec,
+  instanceName: string,
+  volumeName: string,
+): string {
+  const claim = spec.volumeClaimTemplates.find((v) => v.name === volumeName);
+  if (!claim) throw new Error(`docker-statefulset: unknown volume "${volumeName}"`);
+  return claim.dockerVolume(instanceName);
+}
+
+/**
+ * Build `docker run` argument arrays for each container in the StatefulSet.
+ * Returns `Record<ServiceName, () => string[]>` — callers evaluate lazily.
+ */
+export function buildServiceRunArgs(
+  opts: BuildServiceRunArgsOpts,
+  spec = DOCKER_STATEFUL_SET_SPEC,
+): Record<ServiceName, () => string[]> {
+  const {
+    gatewayPort,
+    imageTags,
+    instanceName,
+    res,
+    extraAssistantEnv,
+    defaultWorkspaceConfigPath,
+    avatarDevicePath,
+  } = opts;
+
+  const result = {} as Record<ServiceName, () => string[]>;
+
+  for (const container of spec.containers) {
+    const svc = container.internalName;
+
+    result[svc] = () => {
+      const args: string[] = ["run", "--init", "-d"];
+
+      // Container name
+      const containerName =
+        svc === "assistant"
+          ? res.assistantContainer
+          : svc === "gateway"
+            ? res.gatewayContainer
+            : res.cesContainer;
+      args.push("--name", containerName);
+
+      // Network
+      if (container.network === "bridge") {
+        args.push(`--network=${res.network}`);
+        for (const port of container.ports ?? []) {
+          const hostSide =
+            port.hostPort === "{{ gatewayPort }}"
+              ? `${gatewayPort}`
+              : port.hostPort;
+          if (hostSide !== undefined) {
+            args.push("-p", `${hostSide}:${port.containerPort}`);
+          }
+        }
+      } else {
+        args.push(`--network=container:${res.assistantContainer}`);
+      }
+
+      // Volume mounts
+      for (const mount of container.volumeMounts) {
+        const vol = resolveVolume(spec, instanceName, mount.volumeName);
+        args.push("-v", mount.readOnly ? `${vol}:${mount.mountPath}:ro` : `${vol}:${mount.mountPath}`);
+      }
+
+      // Env vars from spec
+      for (const entry of container.env) {
+        if (entry.kind === "static") {
+          args.push("-e", `${entry.name}=${entry.value}`);
+        } else if (entry.kind === "secret") {
+          const val = opts[entry.secret];
+          if (val) args.push("-e", `${entry.name}=${val}`);
+        } else {
+          const hostVar = entry.hostVar ?? entry.name;
+          const val = process.env[hostVar];
+          if (val) args.push("-e", `${entry.name}=${val}`);
+        }
+      }
+
+      // Assistant-only computed / optional additions
+      if (svc === "assistant") {
+        args.push(
+          "-e", `VELLUM_ASSISTANT_NAME=${instanceName}`,
+          "-e", `GATEWAY_INTERNAL_URL=http://localhost:${GATEWAY_INTERNAL_PORT}`,
+        );
+
+        if (defaultWorkspaceConfigPath) {
+          const cPath = `/tmp/vellum-default-workspace-config-${Date.now()}.json`;
+          args.push(
+            "-v", `${defaultWorkspaceConfigPath}:${cPath}:ro`,
+            "-e", `VELLUM_DEFAULT_WORKSPACE_CONFIG_PATH=${cPath}`,
+          );
+        }
+
+        if (extraAssistantEnv) {
+          for (const [k, v] of Object.entries(extraAssistantEnv)) {
+            args.push("-e", `${k}=${v}`);
+          }
+        }
+
+        if (avatarDevicePath && existsSync(avatarDevicePath)) {
+          args.push(
+            "--device", `${avatarDevicePath}:${avatarDevicePath}`,
+            "-e", `${AVATAR_DEVICE_ENV_VAR}=${avatarDevicePath}`,
+          );
+        }
+      }
+
+      // Image is always last
+      args.push(imageTags[svc]);
+      return args;
+    };
+  }
+
+  return result;
+}

package/src/lib/upgrade-lifecycle.ts

@@ -1,4 +1,7 @@
 import { randomBytes } from "crypto";
+import { spawnSync } from "child_process";
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import { join } from "path";
 
 import type { AssistantEntry } from "./assistant-config.js";
 import { saveAssistantEntry } from "./assistant-config.js";
@@ -12,12 +15,68 @@ import {
   startContainers,
   stopContainers,
 } from "./docker.js";
+import { getStateDir } from "./environments/paths.js";
+import { getCurrentEnvironment } from "./environments/resolve.js";
 import { loadGuardianToken } from "./guardian-token.js";
 import { getPlatformUrl } from "./platform-client.js";
 import { resolveImageRefs } from "./platform-releases.js";
 import { exec, execOutput } from "./step-runner.js";
 import { compareVersions } from "./version-compat.js";
 
+// ---------------------------------------------------------------------------
+// Failure log capture
+// ---------------------------------------------------------------------------
+
+/** XDG-compliant directory for upgrade failure logs, scoped to the current environment. */
+function getUpgradeLogsDir(): string {
+  return join(getStateDir(getCurrentEnvironment()), "upgrade-logs");
+}
+
+/**
+ * Capture stdout/stderr from all three containers after a readiness failure
+ * and write them to an XDG state directory. Returns the directory path so
+ * the caller can print it for the user.
+ *
+ * Runs best-effort — never throws.
+ */
+export async function captureUpgradeFailureLogs(
+  res: ReturnType<typeof dockerResourceNames>,
+  label: string,
+): Promise<string | null> {
+  const isoTimestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const logDir = join(getUpgradeLogsDir(), `${label}-${isoTimestamp}`);
+  try {
+    mkdirSync(logDir, { recursive: true });
+
+    const containers: [string, string][] = [
+      [res.assistantContainer, "assistant.log"],
+      [res.gatewayContainer, "gateway.log"],
+      [res.cesContainer, "credential-executor.log"],
+    ];
+
+    for (const [container, filename] of containers) {
+      try {
+        // Capture stdout + stderr together so container logs written to either
+        // stream (docker logs writes container stdout→stdout, stderr→stderr)
+        // are preserved in a single file. spawnSync avoids the execOutput
+        // limitation of returning only stdout on success.
+        const result = spawnSync("docker", ["logs", "--tail", "500", container], {
+          encoding: "utf8",
+          maxBuffer: 10 * 1024 * 1024, // 10 MB
+        });
+        const output = [result.stdout, result.stderr].filter(Boolean).join("");
+        if (output) writeFileSync(join(logDir, filename), output);
+      } catch {
+        // Container may not exist or may have already been removed
+      }
+    }
+
+    return existsSync(logDir) ? logDir : null;
+  } catch {
+    return null;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Shared constants & builders for upgrade / rollback lifecycle events
 // ---------------------------------------------------------------------------
@@ -147,6 +206,38 @@ export async function fetchCurrentVersion(
   return undefined;
 }
 
+/**
+ * Best-effort fetch of the assistant's configured public ingress URL from the
+ * gateway `integrations/ingress/config` endpoint.  Returns `undefined` when
+ * the gateway is unreachable, the bearer token is missing, or no public URL
+ * is configured.
+ */
+export async function fetchAssistantIngressUrl(
+  runtimeUrl: string,
+  bearerToken?: string,
+): Promise<string | undefined> {
+  if (!bearerToken) return undefined;
+  try {
+    const resp = await fetch(`${runtimeUrl}/integrations/ingress/config`, {
+      headers: { Authorization: `Bearer ${bearerToken}` },
+      signal: AbortSignal.timeout(5000),
+    });
+    if (resp.ok) {
+      const body = (await resp.json()) as {
+        publicBaseUrl?: string;
+        managedCallbacks?: boolean;
+      };
+      // Ignore managed-callback URLs — those belong to the platform, not the
+      // self-hosted assistant's own ingress.
+      if (body.managedCallbacks) return undefined;
+      return body.publicBaseUrl || undefined;
+    }
+  } catch {
+    // Best-effort
+  }
+  return undefined;
+}
+
 /**
  * Determine the version that was running before the current one.
  *
@@ -545,7 +636,7 @@ export async function performDockerRollback(
   const signingKey =
     capturedEnv["ACTOR_TOKEN_SIGNING_KEY"] || randomBytes(32).toString("hex");
 
-  // Build extra env vars, excluding keys managed by serviceDockerRunArgs
+  // Build extra env vars, excluding keys managed by buildServiceRunArgs
   const envKeysSetByRunArgs = new Set(CONTAINER_ENV_EXCLUDE_KEYS);
   for (const envVar of ["ANTHROPIC_API_KEY", "VELLUM_PLATFORM_URL"]) {
     if (process.env[envVar]) {
@@ -734,6 +825,11 @@ export async function performDockerRollback(
     // Failure path — attempt auto-rollback to original version
     console.error(`\n❌ Containers failed to become ready within the timeout.`);
 
+    const logDir = await captureUpgradeFailureLogs(res, `${instanceName}-rollback-failure`);
+    if (logDir) {
+      console.log(`📋 Container logs saved to: ${logDir}`);
+    }
+
     if (currentImageRefs) {
       await broadcastUpgradeEvent(
         entry.runtimeUrl,