@vellumai/assistant 0.6.2 → 0.6.4

package/src/runtime/routes/identity-routes.ts

@@ -3,12 +3,11 @@
  */
 
 import { existsSync, readFileSync, statfsSync, statSync } from "node:fs";
-import { cpus, totalmem } from "node:os";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
+import { availableParallelism, cpus, totalmem } from "node:os";
 
 import { z } from "zod";
 
+import { getCpuLimit, getIsPlatform } from "../../config/env-registry.js";
 import { parseIdentityFields } from "../../daemon/handlers/identity.js";
 import { getProfilerRuntimeStatus } from "../../daemon/profiler-run-store.js";
 import { getMaxMigrationVersion } from "../../memory/migrations/registry.js";
@@ -16,6 +15,7 @@ import {
   getWorkspaceDir,
   getWorkspacePromptPath,
 } from "../../util/platform.js";
+import { APP_VERSION } from "../../version.js";
 import { WORKSPACE_MIGRATIONS } from "../../workspace/migrations/registry.js";
 import { getLastWorkspaceMigrationId } from "../../workspace/migrations/runner.js";
 import { httpError } from "../http-errors.js";
@@ -54,10 +54,60 @@ interface MemoryInfo {
   maxMb: number;
 }
 
-// Read the container memory limit from cgroups if available, falling back to host total.
-// cgroups v2: /sys/fs/cgroup/memory.max (returns "max" when unlimited)
-// cgroups v1: /sys/fs/cgroup/memory/memory.limit_in_bytes (large sentinel when unlimited)
+/**
+ * Parse a Kubernetes-style memory string (e.g. "3Gi", "512Mi", "1G") into bytes.
+ * Returns null if the value is not a recognized format.
+ */
+function parseK8sMemoryBytes(value: string): number | null {
+  const match = value
+    .trim()
+    .match(/^(\d+(?:\.\d+)?)\s*(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E|m)?$/);
+  if (!match) return null;
+  const num = parseFloat(match[1]);
+  const unit = match[2] ?? "";
+  const multipliers: Record<string, number> = {
+    "": 1,
+    m: 1e-3,
+    k: 1e3,
+    M: 1e6,
+    G: 1e9,
+    T: 1e12,
+    P: 1e15,
+    E: 1e18,
+    Ki: 1024,
+    Mi: 1024 ** 2,
+    Gi: 1024 ** 3,
+    Ti: 1024 ** 4,
+    Pi: 1024 ** 5,
+    Ei: 1024 ** 6,
+  };
+  const mult = multipliers[unit];
+  if (mult === undefined) return null;
+  const bytes = Math.round(num * mult);
+  return bytes > 0 ? bytes : null;
+}
+
+/**
+ * Read the memory limit from the VELLUM_MEMORY_LIMIT env var (K8s resource format),
+ * then fall back to cgroups, then to os.totalmem().
+ *
+ * In platform mode the container runs under gVisor where cgroup files may report
+ * the node's memory rather than the container limit. VELLUM_MEMORY_LIMIT is set
+ * by the StatefulSet template to the exact K8s memory limit (e.g. "3Gi").
+ */
 function getContainerMemoryLimitBytes(): number | null {
+  // 1. Prefer the explicit env var set by the platform StatefulSet template.
+  try {
+    const envLimit = process.env.VELLUM_MEMORY_LIMIT;
+    if (envLimit) {
+      const parsed = parseK8sMemoryBytes(envLimit);
+      if (parsed !== null) return parsed;
+    }
+  } catch {
+    /* env var parsing failed – fall through to cgroups */
+  }
+
+  // 2. Try cgroups v2.
   try {
     const v2 = readFileSync("/sys/fs/cgroup/memory.max", "utf-8").trim();
     if (v2 !== "max") {
@@ -67,6 +117,8 @@ function getContainerMemoryLimitBytes(): number | null {
   } catch {
     /* not available */
   }
+
+  // 3. Try cgroups v1.
   try {
     const v1 = readFileSync(
       "/sys/fs/cgroup/memory/memory.limit_in_bytes",
@@ -81,10 +133,56 @@ function getContainerMemoryLimitBytes(): number | null {
   return null;
 }
 
+/**
+ * Read the container's current memory usage from cgroup files.
+ *
+ * Tries cgroups v2 (`memory.current`) first, then cgroups v1
+ * (`memory/memory.usage_in_bytes`), mirroring the v2-then-v1 fallback used by
+ * `getContainerMemoryLimitBytes`. Returns null if neither file is available
+ * or readable.
+ *
+ * Unlike the limit lookup, no env-var override is needed: the gVisor issue
+ * that motivates VELLUM_MEMORY_LIMIT is specifically about the *limit* files
+ * exposing the host node's memory instead of the sandbox limit. The *usage*
+ * files (memory.current / memory.usage_in_bytes) reflect the sandbox's own
+ * accounting and are accurate under gVisor.
+ */
+function getContainerMemoryUsageBytes(): number | null {
+  // 1. Try cgroups v2.
+  try {
+    const v2 = readFileSync("/sys/fs/cgroup/memory.current", "utf-8").trim();
+    const bytes = parseInt(v2, 10);
+    if (!isNaN(bytes) && bytes > 0) return bytes;
+  } catch {
+    /* not available */
+  }
+
+  // 2. Try cgroups v1.
+  try {
+    const v1 = readFileSync(
+      "/sys/fs/cgroup/memory/memory.usage_in_bytes",
+      "utf-8",
+    ).trim();
+    const bytes = parseInt(v1, 10);
+    if (!isNaN(bytes) && bytes > 0) return bytes;
+  } catch {
+    /* not available */
+  }
+  return null;
+}
+
 function getMemoryInfo(): MemoryInfo {
   const bytesToMb = (b: number) => Math.round((b / (1024 * 1024)) * 100) / 100;
+  // In platform-managed mode the daemon shares its Node process with whatever
+  // the container is doing as a whole; `process.memoryUsage().rss` only sees
+  // this process's resident set, which understates the container footprint
+  // operators care about. Read the cgroup usage file directly so /v1/health
+  // matches what the StatefulSet's memory limit is enforced against.
+  const currentBytes =
+    (getIsPlatform() ? getContainerMemoryUsageBytes() : null) ??
+    process.memoryUsage().rss;
   return {
-    currentMb: bytesToMb(process.memoryUsage().rss),
+    currentMb: bytesToMb(currentBytes),
     maxMb: bytesToMb(getContainerMemoryLimitBytes() ?? totalmem()),
   };
 }
@@ -94,52 +192,183 @@ interface CpuInfo {
   maxCores: number;
 }
 
+/**
+ * Parse a Kubernetes-style CPU string (e.g. "2000m", "1", "500m") into
+ * fractional cores. Returns null if the value is not a recognized format.
+ */
+function parseK8sCpuCores(value: string): number | null {
+  const trimmed = value.trim();
+  const milliMatch = trimmed.match(/^(\d+)m$/);
+  if (milliMatch) {
+    const millis = parseInt(milliMatch[1], 10);
+    return millis > 0 ? millis / 1000 : null;
+  }
+  if (/^\d+(\.\d+)?$/.test(trimmed)) {
+    const num = parseFloat(trimmed);
+    return !isNaN(num) && num > 0 ? num : null;
+  }
+  return null;
+}
+
+/**
+ * Read the container's CPU core limit.
+ *
+ * Resolution order:
+ * 1. VELLUM_CPU_LIMIT env var (K8s resource format, e.g. "2000m" or "2").
+ *    In platform mode the container runs under gVisor where cgroup files may
+ *    report the node's CPU count rather than the sandbox limit.
+ * 2. cgroups v2 cpu.max (quota / period → fractional cores).
+ * 3. cgroups v1 cpu.cfs_quota_us / cpu.cfs_period_us.
+ * 4. os.cpus().length as last resort.
+ */
+function getContainerCpuCores(): number {
+  // 1. Prefer the explicit env var set by the platform StatefulSet template.
+  try {
+    const envLimit = getCpuLimit();
+    if (envLimit) {
+      const parsed = parseK8sCpuCores(envLimit);
+      if (parsed !== null) return parsed;
+    }
+  } catch {
+    /* env var parsing failed – fall through */
+  }
+
+  // 2. Try cgroups v2: /sys/fs/cgroup/cpu.max contains "$MAX $PERIOD".
+  try {
+    const raw = readFileSync("/sys/fs/cgroup/cpu.max", "utf-8").trim();
+    if (!raw.startsWith("max")) {
+      const parts = raw.split(/\s+/);
+      const quota = parseInt(parts[0], 10);
+      const period = parseInt(parts[1], 10);
+      if (!isNaN(quota) && !isNaN(period) && period > 0 && quota > 0) {
+        const cores = quota / period;
+        // Sanity check: if the value looks like the node's full CPU count
+        // and we're on a platform pod, it's likely gVisor leaking the host value.
+        if (cores < cpus().length * 0.9 || !getIsPlatform()) {
+          return cores;
+        }
+      }
+    }
+  } catch {
+    /* not available */
+  }
+
+  // 3. Try cgroups v1.
+  try {
+    const quota = parseInt(
+      readFileSync("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "utf-8").trim(),
+      10,
+    );
+    const period = parseInt(
+      readFileSync("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "utf-8").trim(),
+      10,
+    );
+    if (!isNaN(quota) && !isNaN(period) && period > 0 && quota > 0) {
+      const cores = quota / period;
+      if (cores < cpus().length * 0.9 || !getIsPlatform()) {
+        return cores;
+      }
+    }
+  } catch {
+    /* not available */
+  }
+
+  return cpus().length || availableParallelism();
+}
+
+/**
+ * Read the container's CPU usage from cgroup accounting files.
+ *
+ * Returns total CPU microseconds consumed by the container since boot.
+ * We use the delta between two samples to compute percentage.
+ */
+function getContainerCpuUsageUs(): number | null {
+  // cgroups v2: cpu.stat has a "usage_usec" line.
+  try {
+    const stat = readFileSync("/sys/fs/cgroup/cpu.stat", "utf-8");
+    for (const line of stat.split("\n")) {
+      if (line.startsWith("usage_usec")) {
+        const val = parseInt(line.split(/\s+/)[1], 10);
+        if (!isNaN(val) && val > 0) return val;
+      }
+    }
+  } catch {
+    /* not available */
+  }
+
+  // cgroups v1: cpuacct.usage is in nanoseconds.
+  try {
+    const ns = parseInt(
+      readFileSync("/sys/fs/cgroup/cpuacct/cpuacct.usage", "utf-8").trim(),
+      10,
+    );
+    if (!isNaN(ns) && ns > 0) return ns / 1000; // convert ns → µs
+  } catch {
+    /* not available */
+  }
+
+  return null;
+}
+
 // Track CPU usage over a rolling window so /v1/health reports near-real-time
 // utilization instead of a lifetime average (total CPU time / total uptime).
 const CPU_SAMPLE_INTERVAL_MS = 5_000;
-let _lastCpuUsage: NodeJS.CpuUsage = process.cpuUsage();
+let _lastProcessCpuUsage: NodeJS.CpuUsage = process.cpuUsage();
+let _lastCgroupCpuUs: number | null = getContainerCpuUsageUs();
 let _lastCpuTime: number = Date.now();
 let _cachedCpuPercent = 0;
 
 // Kick off the background sampler. unref() so it never prevents process exit.
 setInterval(() => {
   const now = Date.now();
-  const newUsage = process.cpuUsage();
   const elapsedMs = now - _lastCpuTime;
-  if (elapsedMs > 0) {
-    const deltaCpuUs =
-      newUsage.user -
-      _lastCpuUsage.user +
-      (newUsage.system - _lastCpuUsage.system);
-    const deltaCpuMs = deltaCpuUs / 1000;
-    const numCores = cpus().length;
+  if (elapsedMs <= 0) return;
+
+  const numCores = getContainerCpuCores();
+
+  // Always sample process-level CPU so the baseline stays fresh. This
+  // prevents a spike if the platform cgroup path later falls back to
+  // process.cpuUsage() after cgroup stats were previously available.
+  const newProcessUsage = process.cpuUsage();
+  const processDeltaUs =
+    newProcessUsage.user -
+    _lastProcessCpuUsage.user +
+    (newProcessUsage.system - _lastProcessCpuUsage.system);
+  _lastProcessCpuUsage = newProcessUsage;
+
+  if (getIsPlatform()) {
+    // In platform mode, prefer cgroup-level CPU usage so we see the full
+    // container footprint, not just this process.
+    const cgroupUs = getContainerCpuUsageUs();
+    if (cgroupUs !== null && _lastCgroupCpuUs !== null) {
+      const deltaCpuUs = cgroupUs - _lastCgroupCpuUs;
+      const deltaCpuMs = deltaCpuUs / 1000;
+      _cachedCpuPercent =
+        Math.round((deltaCpuMs / (elapsedMs * numCores)) * 10000) / 100;
+    } else {
+      // cgroup CPU stats unavailable (e.g. gVisor) – fall back to process-level.
+      const deltaCpuMs = processDeltaUs / 1000;
+      _cachedCpuPercent =
+        Math.round((deltaCpuMs / (elapsedMs * numCores)) * 10000) / 100;
+    }
+    _lastCgroupCpuUs = cgroupUs;
+  } else {
+    // Non-platform: use process.cpuUsage() (accurate for single-process mode).
+    const deltaCpuMs = processDeltaUs / 1000;
     _cachedCpuPercent =
       Math.round((deltaCpuMs / (elapsedMs * numCores)) * 10000) / 100;
   }
-  _lastCpuUsage = newUsage;
+
   _lastCpuTime = now;
 }, CPU_SAMPLE_INTERVAL_MS).unref();
 
 function getCpuInfo(): CpuInfo {
   return {
     currentPercent: _cachedCpuPercent,
-    maxCores: cpus().length,
+    maxCores: Math.ceil(getContainerCpuCores()),
   };
 }
 
-function getPackageVersion(): string | undefined {
-  try {
-    const pkgPath = join(
-      dirname(fileURLToPath(import.meta.url)),
-      "../../../package.json",
-    );
-    const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
-    return pkg.version;
-  } catch {
-    return undefined;
-  }
-}
-
 export function handleHealth(): Response {
   return Response.json({ status: "ok" });
 }
@@ -155,7 +384,7 @@ export function handleDetailedHealth(): Response {
   return Response.json({
     status: "healthy",
     timestamp: new Date().toISOString(),
-    version: getPackageVersion(),
+    version: APP_VERSION,
     disk: getDiskSpaceInfo(),
     memory: getMemoryInfo(),
     cpu: getCpuInfo(),
@@ -181,7 +410,7 @@ export function handleGetIdentity(): Response {
   const content = readFileSync(identityPath, "utf-8");
   const fields = parseIdentityFields(content);
 
-  const version = getPackageVersion();
+  const version = APP_VERSION;
 
   // Read createdAt from IDENTITY.md file birthtime
   let createdAt: string | undefined;

package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 
-import type { SpeechToTextProvider } from "../../../providers/speech-to-text/types.js";
+import type { BatchTranscriber } from "../../../stt/types.js";
+import { SttError } from "../../../stt/types.js";
 
 // ---------------------------------------------------------------------------
 // Mocks — must be set up before importing the module under test
@@ -17,7 +18,7 @@ let mockAttachments: Array<{
   thumbnailBase64: string | null;
   createdAt: number;
 }> = [];
-let mockProvider: SpeechToTextProvider | null = null;
+let mockTranscriber: BatchTranscriber | null = null;
 
 mock.module("../../../config/assistant-feature-flags.js", () => ({
   isAssistantFeatureFlagEnabled: () => mockFeatureFlagEnabled,
@@ -35,7 +36,18 @@ mock.module("../../../memory/attachments-store.js", () => ({
 }));
 
 mock.module("../../../providers/speech-to-text/resolve.js", () => ({
-  resolveSpeechToTextProvider: async () => mockProvider,
+  resolveBatchTranscriber: async () => mockTranscriber,
+}));
+
+mock.module("../../../stt/daemon-batch-transcriber.js", () => ({
+  normalizeSttError: (err: unknown): SttError => {
+    if (err instanceof SttError) return err;
+    const message = err instanceof Error ? err.message : String(err);
+    if (err instanceof Error && err.name === "AbortError") {
+      return new SttError("timeout", message);
+    }
+    return new SttError("provider-error", message);
+  },
 }));
 
 mock.module("../../../util/logger.js", () => ({
@@ -105,7 +117,7 @@ describe("tryTranscribeAudioAttachments", () => {
   beforeEach(() => {
     mockFeatureFlagEnabled = true;
     mockAttachments = [];
-    mockProvider = null;
+    mockTranscriber = null;
   });
 
   afterEach(() => {
@@ -115,7 +127,9 @@ describe("tryTranscribeAudioAttachments", () => {
   test("audio attachment is transcribed and returns transcribed result", async () => {
     const audio = makeAudioAttachment("a1");
     mockAttachments = [audio];
-    mockProvider = {
+    mockTranscriber = {
+      providerId: "openai-whisper",
+      boundaryId: "daemon-batch",
       transcribe: async () => ({ text: "Hello, how are you?" }),
     };
 
@@ -131,7 +145,9 @@ describe("tryTranscribeAudioAttachments", () => {
     const doc = makeDocumentAttachment("d1");
     const img = makeImageAttachment("i1");
     mockAttachments = [doc, img];
-    mockProvider = {
+    mockTranscriber = {
+      providerId: "openai-whisper",
+      boundaryId: "daemon-batch",
       transcribe: async () => ({ text: "should not be called" }),
     };
 
@@ -140,23 +156,26 @@ describe("tryTranscribeAudioAttachments", () => {
     expect(result).toEqual({ status: "no_audio" });
   });
 
-  test("no API key returns no_provider with helpful reason string", async () => {
+  test("no provider returns no_provider with service-agnostic reason", async () => {
     const audio = makeAudioAttachment("a1");
     mockAttachments = [audio];
-    mockProvider = null; // No provider resolved
+    mockTranscriber = null; // No transcriber resolved
 
     const result = await tryTranscribeAudioAttachments(["a1"]);
 
     expect(result.status).toBe("no_provider");
-    expect((result as { reason: string }).reason).toContain(
-      "No OpenAI API key configured",
-    );
+    const reason = (result as { reason: string }).reason;
+    expect(reason).toContain("speech-to-text");
+    expect(reason).toContain("voice message transcription");
+    expect(reason).not.toContain("OpenAI");
   });
 
   test("API failure returns error with reason", async () => {
     const audio = makeAudioAttachment("a1");
     mockAttachments = [audio];
-    mockProvider = {
+    mockTranscriber = {
+      providerId: "openai-whisper",
+      boundaryId: "daemon-batch",
       transcribe: async () => {
         throw new Error("API rate limit exceeded");
       },
@@ -183,29 +202,9 @@ describe("tryTranscribeAudioAttachments", () => {
   test("30-second timeout fires and returns error without blocking", async () => {
     const audio = makeAudioAttachment("a1");
     mockAttachments = [audio];
-    mockProvider = {
-      transcribe: async (_audio, _mime, signal) => {
-        // Simulate a provider that respects the abort signal
-        return new Promise((_resolve, reject) => {
-          if (signal?.aborted) {
-            reject(new DOMException("The operation was aborted", "AbortError"));
-            return;
-          }
-          const onAbort = () => {
-            reject(new DOMException("The operation was aborted", "AbortError"));
-          };
-          signal?.addEventListener("abort", onAbort, { once: true });
-        });
-      },
-    };
-
-    // The timeout is 30s in the real code, but the test's mock provider
-    // aborts immediately when signaled. We verify the error path works
-    // by checking the result type. For a true timeout test we'd need
-    // to override the timeout constant, but this confirms the abort
-    // path produces the correct result.
-    // Instead, let's test with a provider that checks signal state:
-    mockProvider = {
+    mockTranscriber = {
+      providerId: "openai-whisper",
+      boundaryId: "daemon-batch",
       transcribe: async () => {
         throw new DOMException("The operation was aborted", "AbortError");
       },
@@ -225,7 +224,9 @@ describe("tryTranscribeAudioAttachments", () => {
     mockAttachments = [a1, a2];
 
     let callCount = 0;
-    mockProvider = {
+    mockTranscriber = {
+      providerId: "openai-whisper",
+      boundaryId: "daemon-batch",
       transcribe: async () => {
         callCount++;
         return { text: callCount === 1 ? "First message" : "Second message" };
@@ -247,7 +248,9 @@ describe("tryTranscribeAudioAttachments", () => {
     mockAttachments = [audio, doc];
 
     let transcribeCallCount = 0;
-    mockProvider = {
+    mockTranscriber = {
+      providerId: "openai-whisper",
+      boundaryId: "daemon-batch",
       transcribe: async () => {
         transcribeCallCount++;
         return { text: "Voice transcription" };
@@ -264,7 +267,9 @@ describe("tryTranscribeAudioAttachments", () => {
   });
 
   test("empty attachment IDs returns no_audio", async () => {
-    mockProvider = {
+    mockTranscriber = {
+      providerId: "openai-whisper",
+      boundaryId: "daemon-batch",
       transcribe: async () => ({ text: "should not be called" }),
     };
 
@@ -276,7 +281,9 @@ describe("tryTranscribeAudioAttachments", () => {
   test("attachment with empty transcription returns no_audio", async () => {
     const audio = makeAudioAttachment("a1");
     mockAttachments = [audio];
-    mockProvider = {
+    mockTranscriber = {
+      providerId: "openai-whisper",
+      boundaryId: "daemon-batch",
       transcribe: async () => ({ text: "   " }), // whitespace-only
     };
 

package/src/runtime/routes/inbound-stages/transcribe-audio.ts

@@ -10,7 +10,8 @@
 import { isAssistantFeatureFlagEnabled } from "../../../config/assistant-feature-flags.js";
 import { getConfig } from "../../../config/loader.js";
 import * as attachmentsStore from "../../../memory/attachments-store.js";
-import { resolveSpeechToTextProvider } from "../../../providers/speech-to-text/resolve.js";
+import { resolveBatchTranscriber } from "../../../providers/speech-to-text/resolve.js";
+import { normalizeSttError } from "../../../stt/daemon-batch-transcriber.js";
 import { getLogger } from "../../../util/logger.js";
 
 const log = getLogger("transcribe-audio");
@@ -55,13 +56,13 @@ export async function tryTranscribeAudioAttachments(
       return { status: "no_audio" };
     }
 
-    // Resolve STT provider
-    const provider = await resolveSpeechToTextProvider();
-    if (!provider) {
+    // Resolve STT provider via daemon batch transcriber facade
+    const transcriber = await resolveBatchTranscriber();
+    if (!transcriber) {
       return {
         status: "no_provider",
         reason:
-          "No OpenAI API key configured. Set one up to enable voice message transcription.",
+          "No speech-to-text provider configured. Add an API key for your STT service to enable voice message transcription.",
       };
     }
 
@@ -89,11 +90,11 @@ export async function tryTranscribeAudioAttachments(
         }
 
         const buffer = Buffer.from(hydrated.dataBase64, "base64");
-        const result = await provider.transcribe(
-          buffer,
-          attachment.mimeType,
-          abortController.signal,
-        );
+        const result = await transcriber.transcribe({
+          audio: buffer,
+          mimeType: attachment.mimeType,
+          signal: abortController.signal,
+        });
 
         if (result.text.trim()) {
           transcriptions.push(result.text.trim());
@@ -109,12 +110,11 @@ export async function tryTranscribeAudioAttachments(
       clearTimeout(timeoutId);
     }
   } catch (err: unknown) {
+    const sttErr = normalizeSttError(err);
     const reason =
-      err instanceof Error
-        ? err.name === "AbortError"
-          ? "Transcription timed out"
-          : err.message
-        : String(err);
+      sttErr.category === "timeout"
+        ? "Transcription timed out"
+        : sttErr.message;
     log.warn({ err }, "Audio transcription failed");
     return { status: "error", reason };
   }