apteva 0.4.53 → 0.4.56

package/src/routes/api/agent-utils.ts

@@ -145,6 +145,72 @@ function buildOperatorConfig(features: Agent["features"], projectId: string | nu
   return operatorResult;
 }
 
+function buildRealtimeConfig(features: Agent["features"], projectId: string | null) {
+  // Backwards compatible: handle both boolean and RealtimeConfig
+  const realtimeVal = features.realtime;
+  const isEnabled = typeof realtimeVal === "boolean" ? realtimeVal : (realtimeVal as any)?.enabled ?? false;
+
+  if (!isEnabled) {
+    return { enabled: false, provider: "standard", stt: {}, tts: {} };
+  }
+
+  const rtConfig = typeof realtimeVal === "object" ? realtimeVal as Record<string, unknown> : {};
+  const rtProvider = (rtConfig.provider as string) || "standard";
+
+  // Native OpenAI Realtime mode
+  if (rtProvider === "openai") {
+    return {
+      enabled: true,
+      provider: "openai",
+      model: (rtConfig.model as string) || "gpt-realtime",
+      voice: (rtConfig.voice as string) || "alloy",
+      vad_type: (rtConfig.vadType as string) || "semantic_vad",
+    };
+  }
+
+  // Native Gemini Live mode
+  if (rtProvider === "gemini") {
+    return {
+      enabled: true,
+      provider: "gemini",
+      gemini_model: (rtConfig.geminiModel as string) || "",
+      gemini_voice: (rtConfig.geminiVoice as string) || "Kore",
+      google_search: (rtConfig.googleSearch as boolean) || false,
+    };
+  }
+
+  // Standard mode: STT → Core LLM → TTS pipeline
+  const sttProvider = (rtConfig.sttProvider as string) || "elevenlabs";
+  const sttModel = (rtConfig.sttModel as string) || (sttProvider === "elevenlabs" ? "scribe_v2_realtime" : undefined);
+  const ttsProvider = (rtConfig.ttsProvider as string) || "elevenlabs";
+  const ttsModel = (rtConfig.ttsModel as string) || undefined;
+
+  const sttProviderDef = PROVIDERS[sttProvider as ProviderId];
+  const ttsProviderDef = PROVIDERS[ttsProvider as ProviderId];
+
+  const sttConfig: Record<string, unknown> = { provider: sttProvider };
+  if (sttModel) sttConfig.model = sttModel;
+  // For local providers, include the base URL in the config
+  if (sttProviderDef && "isLocal" in sttProviderDef && sttProviderDef.isLocal) {
+    sttConfig.base_url = ProviderKeys.getDecryptedForProject(sttProvider, projectId) ||
+      ("defaultBaseUrl" in sttProviderDef ? (sttProviderDef as any).defaultBaseUrl : "");
+  }
+
+  const ttsConfig: Record<string, unknown> = { provider: ttsProvider };
+  if (ttsModel) ttsConfig.model = ttsModel;
+  if (ttsProviderDef && "isLocal" in ttsProviderDef && ttsProviderDef.isLocal) {
+    ttsConfig.base_url = ProviderKeys.getDecryptedForProject(ttsProvider, projectId) ||
+      ("defaultBaseUrl" in ttsProviderDef ? (ttsProviderDef as any).defaultBaseUrl : "");
+  }
+
+  return {
+    enabled: true,
+    provider: "standard",
+    stt: sttConfig,
+    tts: ttsConfig,
+  };
+}
+
 // Build agent config from apteva agent data
 // Note: POST /config expects flat structure WITHOUT "agent" wrapper
 export function buildAgentConfig(agent: Agent, providerKey: string) {
@@ -302,12 +368,7 @@ export function buildAgentConfig(agent: Agent, providerKey: string) {
       cache_ttl: "15m",
       servers: mcpServers,
     },
-    realtime: {
-      enabled: features.realtime,
-      provider: "openai",
-      model: "gpt-4o-realtime-preview",
-      voice: "alloy",
-    },
+    realtime: buildRealtimeConfig(features, agent.project_id),
     context: {
       max_messages: 30,
       max_tokens: 0,
@@ -565,6 +626,41 @@ export async function startAgentProcess(
       }
     }
 
+    // If realtime voice is enabled, pass voice provider keys/URLs for STT/TTS
+    const rtEnabled = typeof agent.features.realtime === "boolean"
+      ? agent.features.realtime
+      : (agent.features.realtime as any)?.enabled ?? false;
+    if (rtEnabled) {
+      // Cloud voice provider keys (backwards compat — always pass if available)
+      const elevenlabsKey = ProviderKeys.getDecryptedForProject("elevenlabs", agent.project_id);
+      if (elevenlabsKey) env.ELEVENLABS_API_KEY = elevenlabsKey;
+      const deepgramKey = ProviderKeys.getDecryptedForProject("deepgram", agent.project_id);
+      if (deepgramKey) env.DEEPGRAM_API_KEY = deepgramKey;
+
+      // Local voice provider URLs
+      const localVoiceProviders = ["speaches", "whisper_cpp", "kokoro", "piper", "fish_speech"] as const;
+      for (const pvId of localVoiceProviders) {
+        const pv = PROVIDERS[pvId as ProviderId];
+        if (pv) {
+          const url = ProviderKeys.getDecryptedForProject(pvId, agent.project_id);
+          if (url) env[pv.envVar] = url;
+        }
+      }
+
+      // Pass API keys for native realtime providers (OpenAI/Gemini)
+      // Agent may use a different main LLM provider (e.g., Claude) but needs these for voice
+      const rtConfig = typeof agent.features.realtime === "object" ? agent.features.realtime as Record<string, unknown> : {};
+      const rtProvider = rtConfig.provider as string;
+      if (rtProvider === "openai" && agent.provider !== "openai") {
+        const openaiKey = ProviderKeys.getDecryptedForProject("openai", agent.project_id);
+        if (openaiKey) env.OPENAI_API_KEY = openaiKey;
+      }
+      if (rtProvider === "gemini" && agent.provider !== "gemini") {
+        const geminiKey = ProviderKeys.getDecryptedForProject("gemini", agent.project_id);
+        if (geminiKey) env.GEMINI_API_KEY = geminiKey;
+      }
+    }
+
     // Get binary path dynamically (allows hot-reload of new binary versions)
     const binaryPath = getBinaryPathForAgent();
 
@@ -618,8 +714,9 @@ export async function startAgentProcess(
       console.log(`  Pushing configuration...`);
     }
     const config = buildAgentConfig(agent, providerKey);
-    console.log(`[DEBUG] operator config being pushed:`, JSON.stringify(config.operator, null, 2));
-    console.log(`[DEBUG] builtin_tools:`, JSON.stringify(config.operator?.builtin_tools));
+    if (agent.features.realtime) {
+      console.log(`[DEBUG] realtime config being pushed:`, JSON.stringify(config.realtime, null, 2));
+    }
     const configResult = await pushConfigToAgent(agent.id, port, config);
     if (!configResult.success) {
       if (!silent) {

package/src/routes/api/providers.ts

@@ -172,6 +172,70 @@ export async function handleProviderRoutes(
     }
   }
 
+  // ==================== LOCAL VOICE PROVIDERS ====================
+
+  // GET /api/providers/:id/status - Check if a local voice provider is running
+  const localVoiceStatusMatch = path.match(/^\/api\/providers\/(speaches|whisper_cpp|kokoro|piper|fish_speech)\/status$/);
+  if (localVoiceStatusMatch && method === "GET") {
+    const providerId = localVoiceStatusMatch[1];
+    const providerDef = PROVIDERS[providerId as ProviderId];
+    const baseUrl = ProviderKeys.getDecrypted(providerId) ||
+      ("defaultBaseUrl" in providerDef ? (providerDef as any).defaultBaseUrl : "");
+    const isDocker = await isRunningInDocker();
+
+    const healthEndpoints: Record<string, string> = {
+      speaches: "/v1/models",
+      whisper_cpp: "/",
+      kokoro: "/v1/models",
+      piper: "/api/voices",
+      fish_speech: "/v1/models",
+    };
+
+    try {
+      const response = await fetch(`${baseUrl}${healthEndpoints[providerId]}`, {
+        method: "GET",
+        signal: AbortSignal.timeout(3000),
+      });
+      if (response.ok) {
+        return json({ connected: true, url: baseUrl, isDocker });
+      }
+      return json({ connected: false, url: baseUrl, error: "Not responding", isDocker });
+    } catch {
+      return json({ connected: false, url: baseUrl, error: "Not reachable", isDocker });
+    }
+  }
+
+  // GET /api/providers/:id/models - Fetch available models from a local voice provider
+  const localVoiceModelsMatch = path.match(/^\/api\/providers\/(speaches|kokoro|fish_speech)\/models$/);
+  if (localVoiceModelsMatch && method === "GET") {
+    const providerId = localVoiceModelsMatch[1];
+    const providerDef = PROVIDERS[providerId as ProviderId];
+    const baseUrl = ProviderKeys.getDecrypted(providerId) ||
+      ("defaultBaseUrl" in providerDef ? (providerDef as any).defaultBaseUrl : "");
+
+    try {
+      const response = await fetch(`${baseUrl}/v1/models`, {
+        method: "GET",
+        headers: { "Accept": "application/json" },
+        signal: AbortSignal.timeout(5000),
+      });
+
+      if (!response.ok) {
+        return json({ error: "Failed to connect", models: [], connected: false }, 200);
+      }
+
+      const data = await response.json() as { data?: Array<{ id: string }> };
+      const models = (data.data || []).map((m: { id: string }) => ({
+        value: m.id,
+        label: m.id,
+      }));
+
+      return json({ models, connected: true });
+    } catch {
+      return json({ error: "Not reachable", models: [], connected: false }, 200);
+    }
+  }
+
   // ==================== ONBOARDING ====================
 
   // GET /api/onboarding/status - Check onboarding status

package/src/routes/api/telemetry.ts

@@ -34,13 +34,6 @@ export async function handleTelemetryRoutes(
         return json({ error: "agent_id and events are required" }, 400);
       }
 
-      // Debug: log raw incoming events
-      for (const event of body.events) {
-        if (event.category === "LLM") {
-          console.log(`[telemetry] RAW LLM event from ${body.agent_id}: ${JSON.stringify(event)}`);
-        }
-      }
-
       // Filter out debug events - too noisy
       const filteredEvents = body.events.filter(e => e.level !== "debug");
 

package/src/routes/share.ts

@@ -1,5 +1,5 @@
 import { createHash } from "crypto";
-import { AgentDB, type Agent } from "../db";
+import { AgentDB, isRealtimeEnabled, type Agent } from "../db";
 import { agentFetch } from "./api/agent-utils";
 
 function deriveShareToken(apiKey: string, agentId: string): string {
@@ -9,7 +9,7 @@ function deriveShareToken(apiKey: string, agentId: string): string {
     .substring(0, 32);
 }
 
-function findAgentByShareToken(token: string): Agent | null {
+export function findAgentByShareToken(token: string): Agent | null {
   const agents = AgentDB.findAll();
   for (const agent of agents) {
     const apiKey = AgentDB.getApiKey(agent.id);
@@ -46,6 +46,7 @@ export async function handleShareRequest(req: Request, path: string): Promise<Re
     return Response.json({
       name: agent.name,
       status: agent.status,
+      voiceEnabled: isRealtimeEnabled(agent.features),
     });
   }
 

package/src/server.ts

@@ -1,13 +1,14 @@
 import { type Server, type Subprocess } from "bun";
 import { handleApiRequest } from "./routes/api";
 import { handleAuthRequest } from "./routes/auth";
-import { handleShareRequest } from "./routes/share";
+import { handleShareRequest, findAgentByShareToken } from "./routes/share";
 import { serveStatic } from "./routes/static";
 import { join } from "path";
 import { homedir } from "os";
 import { mkdirSync, existsSync } from "fs";
-import { initDatabase, AgentDB, ProviderKeysDB, McpServerDB, ChannelDB, TelemetryDB, type McpServer, type Agent } from "./db";
+import { initDatabase, AgentDB, ApiKeyDB, ProviderKeysDB, McpServerDB, ChannelDB, TelemetryDB, UserDB, type McpServer, type Agent } from "./db";
 import { authMiddleware, type AuthContext } from "./auth/middleware";
+import { verifyAccessToken } from "./auth";
 import { startMcpProcess } from "./mcp-client";
 import {
   ensureBinary,
@@ -371,12 +372,54 @@ const server = Bun.serve({
     // WebSocket upgrade: /api/agents/:id/voice → proxy to agent binary
     const voiceMatch = path.match(/^\/api\/agents\/([^/]+)\/voice$/);
     if (voiceMatch && req.headers.get("upgrade")?.toLowerCase() === "websocket") {
+      // Validate auth via query param token (WebSocket can't send custom headers)
+      if (process.env.AUTH_ENABLED !== "false") {
+        const token = url.searchParams.get("token");
+        if (!token) {
+          return new Response("Authentication required", { status: 401 });
+        }
+        // Try API key first, then JWT
+        const isApiKey = token.startsWith("apt_");
+        if (isApiKey) {
+          const result = ApiKeyDB.validate(token);
+          if (!result) {
+            return new Response("Invalid API key", { status: 401 });
+          }
+        } else {
+          const payload = verifyAccessToken(token);
+          if (!payload) {
+            return new Response("Invalid or expired token", { status: 401 });
+          }
+          const user = UserDB.findById(payload.userId);
+          if (!user) {
+            return new Response("User not found", { status: 401 });
+          }
+        }
+      }
+
       const agentId = voiceMatch[1];
       const agent = AgentDB.findById(agentId);
       if (!agent || agent.status !== "running" || !agent.port) {
         return new Response("Agent not available", { status: 400 });
       }
-      const upgraded = bunServer.upgrade(req, { data: { agentId: agent.id, agentPort: agent.port } });
+      const agentApiKey = AgentDB.getApiKey(agentId) || "";
+      console.log(`[WS] Voice upgrade: agent=${agentId} port=${agent.port} hasKey=${!!agentApiKey}`);
+      const upgraded = bunServer.upgrade(req, { data: { agentId: agent.id, agentPort: agent.port, agentApiKey } });
+      if (upgraded) return undefined;
+      return new Response("WebSocket upgrade failed", { status: 500 });
+    }
+
+    // WebSocket upgrade: /share/<token>/voice → proxy to agent binary (public, share-token auth)
+    const shareVoiceMatch = path.match(/^\/share\/([a-f0-9]{32})\/voice$/);
+    if (shareVoiceMatch && req.headers.get("upgrade")?.toLowerCase() === "websocket") {
+      const shareToken = shareVoiceMatch[1];
+      const agent = findAgentByShareToken(shareToken);
+      if (!agent || agent.status !== "running" || !agent.port) {
+        return new Response("Agent not available", { status: 400 });
+      }
+      const agentApiKey = AgentDB.getApiKey(agent.id) || "";
+      console.log(`[WS] Share voice upgrade: agent=${agent.id} port=${agent.port}`);
+      const upgraded = bunServer.upgrade(req, { data: { agentId: agent.id, agentPort: agent.port, agentApiKey } });
       if (upgraded) return undefined;
       return new Response("WebSocket upgrade failed", { status: 500 });
     }
@@ -454,8 +497,10 @@ const server = Bun.serve({
   // WebSocket proxy for agent voice/realtime
   websocket: {
     open(ws: any) {
-      const { agentId, agentPort } = ws.data;
-      const agentWs = new WebSocket(`ws://localhost:${agentPort}/voice`);
+      const { agentId, agentPort, agentApiKey } = ws.data;
+      const wsTarget = `ws://localhost:${agentPort}/voice${agentApiKey ? `?api_key=${agentApiKey}` : ''}`;
+      console.log(`[WS] Voice proxy connecting: agent=${agentId} port=${agentPort}`);
+      const agentWs = new WebSocket(wsTarget);
 
       agentWs.onopen = () => {
         console.log(`[WS] Voice proxy connected: agent=${agentId} port=${agentPort}`);
@@ -464,10 +509,11 @@ const server = Bun.serve({
         try { ws.send(event.data); } catch {}
       };
       agentWs.onclose = (event: CloseEvent) => {
-        console.log(`[WS] Agent disconnected: agent=${agentId} code=${event.code}`);
+        console.log(`[WS] Agent disconnected: agent=${agentId} code=${event.code} reason=${event.reason}`);
         ws.close(event.code, event.reason);
       };
-      agentWs.onerror = () => {
+      agentWs.onerror = (err: any) => {
+        console.log(`[WS] Agent WS error: agent=${agentId}`, err?.message || err);
         ws.close(1011, "Agent WebSocket error");
       };
 

package/src/test-runner.ts

@@ -241,7 +241,7 @@ export async function runTest(testCase: TestCase): Promise<TestRun> {
 
     const chatRes = await agentFetch(runningAgent.id, runningAgent.port!, "/chat", {
       method: "POST",
-      headers: { "Content-Type": "application/json" },
+      headers: { "Content-Type": "application/json", "X-Test-Mode": "true" },
       body: JSON.stringify(chatBody),
     });
 

package/src/web/App.tsx

@@ -1,6 +1,6 @@
 import React, { useState, useEffect, useMemo, lazy, Suspense } from "react";
 import { createRoot } from "react-dom/client";
-import { Chat } from "@apteva/apteva-kit";
+import { Chat, Call } from "@apteva/apteva-kit";
 import "@apteva/apteva-kit/styles.css";
 
 // Types
@@ -8,7 +8,7 @@ import type { Agent, Provider, Route, NewAgentForm } from "./types";
 import { DEFAULT_FEATURES } from "./types";
 
 // Context
-import { TelemetryProvider, AuthProvider, ProjectProvider, ThemeProvider, useTheme, useAuth, useProjects, useAgentStatusChange, useTaskChange } from "./context";
+import { TelemetryProvider, AuthProvider, ProjectProvider, ThemeProvider, UIModeProvider, useTheme, useAuth, useProjects, useAgentStatusChange, useTaskChange } from "./context";
 
 // Hooks
 import { useAgents, useProviders, useOnboarding } from "./hooks";
@@ -348,6 +348,7 @@ function SharePage({ token }: { token: string }) {
   const { theme } = useTheme();
   const [status, setStatus] = useState<"checking" | "online" | "offline">("checking");
   const [agentName, setAgentName] = useState("Agent");
+  const [voiceEnabled, setVoiceEnabled] = useState(false);
 
   useEffect(() => {
     const check = async () => {
@@ -356,6 +357,7 @@ function SharePage({ token }: { token: string }) {
         if (res.ok) {
           const data = await res.json();
           setAgentName(data.name || "Agent");
+          setVoiceEnabled(!!data.voiceEnabled);
           setStatus(data.status === "running" ? "online" : "offline");
         } else {
           setStatus("offline");
@@ -392,17 +394,28 @@ function SharePage({ token }: { token: string }) {
   return (
     <div className="min-h-[100dvh] flex items-center justify-center p-0 md:p-4" style={{ backgroundColor: "var(--color-bg)" }}>
       <div className="w-full max-w-[640px] h-[100dvh] md:h-[calc(100dvh-32px)] md:max-h-[800px] md:rounded-xl overflow-hidden md:border flex flex-col" style={{ backgroundColor: "var(--color-bg)", borderColor: "var(--color-border)" }}>
-        <Chat
-          agentId="default"
-          apiUrl={`/share/${token}`}
-          placeholder="Type a message..."
-          variant="terminal"
-          theme={theme.id as "light" | "dark"}
-          headerTitle={agentName}
-          enableMarkdown
-          enableWidgets
-          availableWidgets={["form", "kpi"]}
-        />
+        {voiceEnabled ? (
+          <Call
+            agentId="default"
+            agentName={agentName}
+            apiUrl={`/share/${token}`}
+            variant="default"
+            theme={theme.id as "light" | "dark"}
+            showTranscript={false}
+          />
+        ) : (
+          <Chat
+            agentId="default"
+            apiUrl={`/share/${token}`}
+            placeholder="Type a message..."
+            variant="terminal"
+            theme={theme.id as "light" | "dark"}
+            headerTitle={agentName}
+            enableMarkdown
+            enableWidgets
+            availableWidgets={["form", "kpi"]}
+          />
+        )}
       </div>
     </div>
   );
@@ -422,15 +435,17 @@ function App() {
 
   return (
     <ThemeProvider>
-      <AuthProvider>
-        <ProjectProvider>
-          <MetaAgentProvider>
-            <TelemetryProvider>
-              <AppContent />
-            </TelemetryProvider>
-          </MetaAgentProvider>
-        </ProjectProvider>
-      </AuthProvider>
+      <UIModeProvider>
+        <AuthProvider>
+          <ProjectProvider>
+            <MetaAgentProvider>
+              <TelemetryProvider>
+                <AppContent />
+              </TelemetryProvider>
+            </MetaAgentProvider>
+          </ProjectProvider>
+        </AuthProvider>
+      </UIModeProvider>
     </ThemeProvider>
   );
 }

package/src/web/components/agents/AgentCard.tsx

@@ -1,6 +1,6 @@
 import React from "react";
 import { MemoryIcon, TasksIcon, VisionIcon, OperatorIcon, McpIcon, RealtimeIcon, FilesIcon, MultiAgentIcon, SkillsIcon, ActivityIcon } from "../common/Icons";
-import { useAgentActivity, useProjects } from "../../context";
+import { useAgentActivity, useProjects, useUIMode } from "../../context";
 import type { Agent, AgentFeatures } from "../../types";
 
 interface AgentCardProps {
@@ -28,6 +28,7 @@ export const AgentCard = React.memo(function AgentCard({ agent, selected, onSele
   const skills = agent.skillDetails || [];
   const { isActive, label: activityLabel } = useAgentActivity(agent.id);
   const { projects } = useProjects();
+  const { isDev } = useUIMode();
   const project = agent.projectId ? projects.find(p => p.id === agent.projectId) : null;
   const subscriptions = agent.subscriptions || [];
 
@@ -43,10 +44,12 @@ export const AgentCard = React.memo(function AgentCard({ agent, selected, onSele
       <div className="flex items-start justify-between mb-3">
         <div>
           <h3 className="font-semibold text-lg">{agent.name}</h3>
-          <p className="text-sm text-[var(--color-text-muted)]">
-            {agent.provider} / {agent.model}
-            {agent.port && <span className="text-[var(--color-text-faint)]"> · :{agent.port}</span>}
-          </p>
+          {isDev && (
+            <p className="text-sm text-[var(--color-text-muted)]">
+              {agent.provider} / {agent.model}
+              {agent.port && <span className="text-[var(--color-text-faint)]"> · :{agent.port}</span>}
+            </p>
+          )}
           {showProject && project && (
             <p className="text-sm text-[var(--color-text-muted)] flex items-center gap-1.5 mt-1">
               <span className="w-2 h-2 rounded-full" style={{ backgroundColor: project.color }} />
@@ -57,7 +60,7 @@ export const AgentCard = React.memo(function AgentCard({ agent, selected, onSele
         <StatusBadge status={agent.status} isActive={isActive && agent.status === "running"} activityLabel={activityLabel} />
       </div>
 
-      {enabledFeatures.length > 0 && (
+      {isDev && enabledFeatures.length > 0 && (
         <div className="flex flex-wrap gap-1.5 mb-3">
           {enabledFeatures.map(({ key, icon: Icon, label }) => (
             <span
@@ -74,7 +77,7 @@ export const AgentCard = React.memo(function AgentCard({ agent, selected, onSele
       )}
 
       {/* MCP Servers */}
-      {mcpServers.length > 0 && (
+      {isDev && mcpServers.length > 0 && (
         <div className="flex flex-wrap gap-1.5 mb-3">
           {mcpServers.map((server) => {
             // HTTP/remote servers are always available
@@ -98,7 +101,7 @@ export const AgentCard = React.memo(function AgentCard({ agent, selected, onSele
       )}
 
       {/* Skills */}
-      {skills.length > 0 && (
+      {isDev && skills.length > 0 && (
         <div className="flex flex-wrap gap-1.5 mb-3">
           {skills.map((skill) => (
             <span
@@ -118,7 +121,7 @@ export const AgentCard = React.memo(function AgentCard({ agent, selected, onSele
       )}
 
       {/* Subscriptions (triggers listening to) */}
-      {subscriptions.length > 0 && (
+      {isDev && subscriptions.length > 0 && (
         <div className="flex flex-wrap gap-1.5 mb-3">
           {subscriptions.map((sub) => (
             <span