@kithjs/server 0.3.0

package/Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.11-slim AS python-base
+RUN pip install uv
+COPY packages/runtime-pipecat/python /app/python
+WORKDIR /app/python
+RUN uv venv && . .venv/bin/activate && uv pip install -e .
+
+FROM oven/bun:1.3
+COPY --from=python-base /app/python /app/python
+COPY --from=python-base /usr/local/bin/python3.11 /usr/local/bin/python3.11
+COPY --from=python-base /usr/local/lib/python3.11 /usr/local/lib/python3.11
+WORKDIR /app/server
+COPY packages/server/package.json .
+COPY packages/server/src ./src
+COPY packages/server/characters ./characters
+RUN bun install
+ENV PIPECAT_PYTHON_PATH=/app/python/.venv/bin/python
+ENV PIPECAT_PYTHON_CWD=/app/python
+ENV PORT=3040
+EXPOSE 3040
+CMD ["bun", "src/index.ts"]

package/characters/example.json

@@ -0,0 +1,16 @@
+{
+  "voice": {
+    "stability": 0.5,
+    "similarityBoost": 0.85,
+    "style": 0.4,
+    "useSpeakerBoost": true,
+    "speed": 1.0
+  },
+  "slang": {
+    "gg": "good game",
+    "fr": "for real",
+    "ngl": "not gonna lie"
+  },
+  "pronunciation": {},
+  "personaMode": "neutral"
+}

package/docker-compose.yml

@@ -0,0 +1,16 @@
+version: "3.8"
+services:
+  kith-voice:
+    build:
+      context: ../..
+      dockerfile: packages/server/Dockerfile
+    ports:
+      - "3040:3040"
+    environment:
+      - ELEVENLABS_API_KEY=${ELEVENLABS_API_KEY}
+      - ELEVENLABS_VOICE_ID=${ELEVENLABS_VOICE_ID:-kPzsL2i3teMYv0FxEYQ6}
+      - ELEVENLABS_MODEL_ID=${ELEVENLABS_MODEL_ID:-eleven_v3}
+      - KITH_CHARACTER_DIR=/app/server/characters
+      - PORT=3040
+    volumes:
+      - ./characters:/app/server/characters:ro

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@kithjs/server",
+  "version": "0.3.0",
+  "description": "Standalone voice microservice for Kith — POST text, get streaming audio. Framework agnostic.",
+  "type": "module",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "files": ["src", "characters", "Dockerfile", "docker-compose.yml", "README.md"],
+  "scripts": {
+    "dev": "bun --hot src/index.ts",
+    "start": "bun src/index.ts",
+    "typecheck": "tsc --noEmit",
+    "test": "bun test"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/wbaxterh/kith",
+    "directory": "packages/server"
+  },
+  "keywords": ["kith", "voice", "tts", "microservice", "ai-companion", "streaming"],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@kithjs/core": "^0.2.0",
+    "@kithjs/runtime-pipecat": "^0.2.0",
+    "@kithjs/voice-router": "^0.2.0",
+    "@kithjs/observability": "^0.2.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.6.0"
+  }
+}

package/src/character-registry.ts

@@ -0,0 +1,57 @@
+/**
+ * Character registry — loads VoiceCharacter profiles from a directory.
+ *
+ * Characters are JSON files in a configurable directory. The filename
+ * (without .json) becomes the character ID.
+ *
+ *   characters/
+ *     kaori.json     → characterId: "kaori"
+ *     apollo.json    → characterId: "apollo"
+ */
+
+import path from "node:path";
+import type { VoiceCharacter } from "@kithjs/voice-router";
+
+export class CharacterRegistry {
+  private characters = new Map<string, VoiceCharacter>();
+  private dir: string;
+
+  constructor(dir: string) {
+    this.dir = dir;
+  }
+
+  async load(): Promise<void> {
+    const fs = await import("node:fs/promises");
+    try {
+      const files = await fs.readdir(this.dir);
+      for (const file of files) {
+        if (!file.endsWith(".json")) continue;
+        const id = file.replace(/\.json$/, "");
+        try {
+          const raw = await fs.readFile(path.join(this.dir, file), "utf-8");
+          this.characters.set(id, JSON.parse(raw) as VoiceCharacter);
+        } catch (err) {
+          console.warn(`[kith] failed to load character ${file}:`, err);
+        }
+      }
+      console.log(`[kith] loaded ${this.characters.size} character(s): ${[...this.characters.keys()].join(", ")}`);
+    } catch {
+      console.log(`[kith] no characters directory at ${this.dir}`);
+    }
+  }
+
+  get(id: string): VoiceCharacter | undefined {
+    return this.characters.get(id);
+  }
+
+  list(): { id: string; personaMode?: string }[] {
+    return [...this.characters.entries()].map(([id, c]) => ({
+      id,
+      personaMode: c.personaMode,
+    }));
+  }
+
+  has(id: string): boolean {
+    return this.characters.has(id);
+  }
+}

package/src/index.ts

@@ -0,0 +1,24 @@
+/**
+ * @kithjs/server — Standalone voice microservice for Kith.
+ *
+ * POST text, get streaming audio. Framework agnostic.
+ *
+ * Quick start:
+ *   ELEVENLABS_API_KEY=sk_... bun @kithjs/server
+ *
+ * Or programmatic:
+ *   import { KithServer } from "@kithjs/server";
+ *   const server = new KithServer({ port: 3040 });
+ *   await server.start();
+ */
+
+export { KithServer, type KithServerOptions } from "./server.ts";
+export { SessionManager, type Session, type SessionConfig } from "./session.ts";
+export { CharacterRegistry } from "./character-registry.ts";
+
+// Auto-start when run directly
+if (import.meta.main) {
+  const { KithServer } = await import("./server.ts");
+  const server = new KithServer();
+  await server.start();
+}

package/src/server.ts

@@ -0,0 +1,318 @@
+/**
+ * KithServer — standalone voice microservice.
+ *
+ * Framework-agnostic HTTP/WebSocket server. POST text, get streaming audio.
+ *
+ * Endpoints:
+ *   POST   /sessions              — create session { characterId? } → { sessionId, wsUrl }
+ *   DELETE /sessions/:id          — destroy session
+ *   POST   /sessions/:id/speak    — send text { text } → audio via WS/SSE
+ *   POST   /sessions/:id/barge-in — stop current TTS
+ *   GET    /sessions/:id/events   — SSE stream (fallback)
+ *   WS     /ws?sessionId=xxx      — full duplex WebSocket
+ *   GET    /characters            — list character profiles
+ *   GET    /health                — { ok, sessions, uptime }
+ */
+
+import path from "node:path";
+import type { ServerWebSocket } from "bun";
+
+import { CharacterRegistry } from "./character-registry.ts";
+import { SessionManager, type SessionConfig } from "./session.ts";
+
+export interface KithServerOptions {
+  port?: number;
+  characterDir?: string;
+  defaultCharacterId?: string;
+  pythonPath?: string;
+  pythonCwd?: string;
+  corsOrigins?: string;
+  maxSessions?: number;
+  pipeline?: string;
+  apiKey?: string;
+  voiceId?: string;
+  modelId?: string;
+  pipelineConfig?: Record<string, unknown>;
+}
+
+interface WsData {
+  sessionId: string;
+}
+
+export class KithServer {
+  private options: Required<KithServerOptions>;
+  private characters: CharacterRegistry;
+  private sessions: SessionManager;
+  private server: ReturnType<typeof Bun.serve> | null = null;
+  private startedAt = 0;
+
+  constructor(opts: KithServerOptions = {}) {
+    const root = path.dirname(Bun.fileURLToPath(import.meta.url));
+
+    this.options = {
+      port: opts.port ?? Number(process.env.PORT ?? 3040),
+      characterDir: opts.characterDir ?? process.env.KITH_CHARACTER_DIR ?? "./characters",
+      defaultCharacterId: opts.defaultCharacterId ?? process.env.KITH_DEFAULT_CHARACTER ?? "",
+      pythonPath:
+        opts.pythonPath ??
+        process.env.PIPECAT_PYTHON_PATH ??
+        path.resolve(root, "../node_modules/@kithjs/runtime-pipecat/python/.venv/bin/python"),
+      pythonCwd:
+        opts.pythonCwd ??
+        process.env.PIPECAT_PYTHON_CWD ??
+        path.resolve(root, "../node_modules/@kithjs/runtime-pipecat/python"),
+      corsOrigins: opts.corsOrigins ?? process.env.KITH_CORS ?? "*",
+      maxSessions: opts.maxSessions ?? 100,
+      pipeline: opts.pipeline ?? process.env.KITH_PIPELINE ?? "elevenlabs",
+      apiKey: opts.apiKey ?? process.env.ELEVENLABS_API_KEY ?? "",
+      voiceId: opts.voiceId ?? process.env.ELEVENLABS_VOICE_ID ?? "",
+      modelId: opts.modelId ?? process.env.ELEVENLABS_MODEL_ID ?? "eleven_v3",
+      pipelineConfig: opts.pipelineConfig ?? {},
+    };
+
+    if (!this.options.apiKey) {
+      console.error("[kith] ELEVENLABS_API_KEY is required. Set it via env or options.");
+      process.exit(2);
+    }
+
+    this.characters = new CharacterRegistry(this.options.characterDir);
+
+    const sessionConfig: SessionConfig = {
+      pythonPath: this.options.pythonPath,
+      pythonCwd: this.options.pythonCwd,
+      apiKey: this.options.apiKey,
+      voiceId: this.options.voiceId,
+      modelId: this.options.modelId,
+      pipelineConfig: {
+        pipeline: this.options.pipeline,
+        ...this.options.pipelineConfig,
+      },
+    };
+
+    this.sessions = new SessionManager(sessionConfig, this.options.maxSessions);
+  }
+
+  async start(): Promise<void> {
+    await this.characters.load();
+    this.startedAt = Date.now();
+
+    const self = this;
+
+    this.server = Bun.serve<WsData>({
+      port: this.options.port,
+
+      async fetch(req, server) {
+        return self.handleRequest(req, server);
+      },
+
+      websocket: {
+        async open(ws: ServerWebSocket<WsData>) {
+          const { sessionId } = ws.data;
+          console.log(`[kith] ws open session=${sessionId}`);
+
+          // Attach to existing session or create new one
+          if (self.sessions.has(sessionId)) {
+            self.sessions.attachWs(sessionId, ws);
+            ws.send(JSON.stringify({ type: "_ready", sessionId }));
+          } else {
+            try {
+              const charId = self.options.defaultCharacterId;
+              const character = charId ? self.characters.get(charId) : undefined;
+              await self.sessions.create(sessionId, character, ws);
+              ws.send(JSON.stringify({ type: "_ready", sessionId }));
+            } catch (err) {
+              console.error(`[kith] session create failed:`, err);
+              ws.close(1011, "session create failed");
+            }
+          }
+        },
+
+        async message(ws: ServerWebSocket<WsData>, raw) {
+          const { sessionId } = ws.data;
+          const session = self.sessions.get(sessionId);
+          if (!session) return;
+
+          let msg: { type: string; text?: string };
+          try {
+            msg = JSON.parse(String(raw));
+          } catch {
+            return;
+          }
+
+          if (msg.type === "speak" && typeof msg.text === "string") {
+            try {
+              await session.voice.speak(msg.text);
+            } catch (err) {
+              console.error(`[kith] speak failed session=${sessionId}:`, err);
+            }
+          } else if (msg.type === "barge-in") {
+            await session.runtime.bargeIn();
+          }
+        },
+
+        async close(ws: ServerWebSocket<WsData>) {
+          const { sessionId } = ws.data;
+          console.log(`[kith] ws close session=${sessionId}`);
+          await self.sessions.destroy(sessionId);
+        },
+      },
+    });
+
+    console.log(`[kith] server listening on http://localhost:${this.server.port}`);
+  }
+
+  async stop(): Promise<void> {
+    this.server?.stop();
+  }
+
+  private async handleRequest(
+    req: Request,
+    server: { upgrade: (req: Request, opts: any) => boolean },
+  ): Promise<Response | undefined> {
+    const url = new URL(req.url);
+    const cors = this.corsHeaders();
+
+    // CORS preflight
+    if (req.method === "OPTIONS") {
+      return new Response(null, { status: 204, headers: cors });
+    }
+
+    // WebSocket upgrade
+    if (url.pathname === "/ws") {
+      const sessionId = url.searchParams.get("sessionId") ?? crypto.randomUUID();
+      const ok = server.upgrade(req, { data: { sessionId } });
+      if (ok) return undefined;
+      return this.json({ error: "WebSocket upgrade failed" }, 500, cors);
+    }
+
+    // POST /sessions — create session
+    if (url.pathname === "/sessions" && req.method === "POST") {
+      try {
+        const body = (await req.json().catch(() => ({}))) as { characterId?: string };
+        const sessionId = crypto.randomUUID();
+        const character = body.characterId ? this.characters.get(body.characterId) : undefined;
+        await this.sessions.create(sessionId, character);
+        return this.json(
+          {
+            sessionId,
+            wsUrl: `ws://localhost:${this.options.port}/ws?sessionId=${sessionId}`,
+          },
+          201,
+          cors,
+        );
+      } catch (err: any) {
+        return this.json({ error: err.message }, 500, cors);
+      }
+    }
+
+    // DELETE /sessions/:id
+    const deleteMatch = url.pathname.match(/^\/sessions\/([^/]+)$/);
+    if (deleteMatch && req.method === "DELETE") {
+      await this.sessions.destroy(deleteMatch[1]);
+      return this.json({ ok: true }, 200, cors);
+    }
+
+    // POST /sessions/:id/speak
+    const speakMatch = url.pathname.match(/^\/sessions\/([^/]+)\/speak$/);
+    if (speakMatch && req.method === "POST") {
+      const session = this.sessions.get(speakMatch[1]);
+      if (!session) {
+        return this.json({ error: "session not found" }, 404, cors);
+      }
+      try {
+        const body = (await req.json()) as { text: string };
+        if (!body.text) return this.json({ error: "text is required" }, 400, cors);
+        session.voice.speak(body.text).catch((err) => {
+          console.error(`[kith] speak failed:`, err);
+        });
+        return this.json({ ok: true, sessionId: speakMatch[1] }, 200, cors);
+      } catch (err: any) {
+        return this.json({ error: err.message }, 500, cors);
+      }
+    }
+
+    // POST /sessions/:id/barge-in
+    const bargeMatch = url.pathname.match(/^\/sessions\/([^/]+)\/barge-in$/);
+    if (bargeMatch && req.method === "POST") {
+      const session = this.sessions.get(bargeMatch[1]);
+      if (!session) return this.json({ error: "session not found" }, 404, cors);
+      await session.runtime.bargeIn();
+      return this.json({ ok: true }, 200, cors);
+    }
+
+    // GET /sessions/:id/events — SSE stream
+    const sseMatch = url.pathname.match(/^\/sessions\/([^/]+)\/events$/);
+    if (sseMatch && req.method === "GET") {
+      const session = this.sessions.get(sseMatch[1]);
+      if (!session) return this.json({ error: "session not found" }, 404, cors);
+
+      const stream = new ReadableStream({
+        start(controller) {
+          const encoder = new TextEncoder();
+          const unsub = session.voice.on((event) => {
+            try {
+              controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`));
+            } catch {
+              unsub();
+            }
+          });
+
+          // Flush buffered events
+          for (const event of session.eventBuffer) {
+            controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`));
+          }
+          session.eventBuffer.length = 0;
+        },
+      });
+
+      return new Response(stream, {
+        headers: {
+          ...cors,
+          "Content-Type": "text/event-stream",
+          "Cache-Control": "no-cache",
+          Connection: "keep-alive",
+        },
+      });
+    }
+
+    // GET /characters
+    if (url.pathname === "/characters" && req.method === "GET") {
+      return this.json(this.characters.list(), 200, cors);
+    }
+
+    // GET /health
+    if (url.pathname === "/health") {
+      const stats = this.sessions.stats();
+      return this.json(
+        {
+          ok: true,
+          sessions: stats.count,
+          uptime: (Date.now() - this.startedAt) / 1000,
+        },
+        200,
+        cors,
+      );
+    }
+
+    return this.json(
+      { name: "@kithjs/server", docs: "https://kith.weshuber.com" },
+      200,
+      cors,
+    );
+  }
+
+  private json(data: unknown, status: number, headers: Record<string, string> = {}): Response {
+    return Response.json(data, {
+      status,
+      headers: { ...headers, "Content-Type": "application/json" },
+    });
+  }
+
+  private corsHeaders(): Record<string, string> {
+    return {
+      "Access-Control-Allow-Origin": this.options.corsOrigins,
+      "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS",
+      "Access-Control-Allow-Headers": "Content-Type, Authorization, x-kith-session",
+    };
+  }
+}

package/src/session.ts

@@ -0,0 +1,190 @@
+/**
+ * Session manager — tracks per-client voice sessions.
+ *
+ * Each session owns a PipecatRuntime + VoiceRouter pair. Sessions can be
+ * created via HTTP (before a WebSocket connects) or on WS open.
+ */
+
+import type { KithEvent } from "@kithjs/core";
+import { PipecatRuntime } from "@kithjs/runtime-pipecat";
+import { InMemoryObservability, consoleExporter } from "@kithjs/observability";
+import {
+  DEFAULT_BOARD_SPORTS_SLANG,
+  DEFAULT_ENGLISH_SLANG,
+  DEFAULT_GENZ_SLANG,
+  DEFAULT_LAUGH_TAGS,
+  VoiceRouter,
+  voiceCharacterToRuntimeConfig,
+  type VoiceCharacter,
+} from "@kithjs/voice-router";
+
+import type { ServerWebSocket } from "bun";
+
+export interface SessionConfig {
+  pythonPath: string;
+  pythonCwd: string;
+  apiKey: string;
+  voiceId: string;
+  modelId: string;
+  pipelineConfig?: Record<string, unknown>;
+}
+
+export interface Session {
+  id: string;
+  runtime: PipecatRuntime;
+  voice: VoiceRouter;
+  obs: InMemoryObservability;
+  unsubscribe: () => void;
+  ws: ServerWebSocket<{ sessionId: string }> | null;
+  character: VoiceCharacter | undefined;
+  eventBuffer: KithEvent[];
+  createdAt: number;
+}
+
+/** Clean AI-generated text for natural TTS output. */
+function cleanForTTS(text: string): string {
+  let t = text;
+  t = t.replace(/\*{1,3}([^*]+)\*{1,3}/g, "$1");
+  t = t.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
+  t = t.replace(/([!?.]){2,}/g, "$1");
+  t = t.replace(/([a-z])\1{3,}/gi, "$1$1");
+  t = t.replace(/:[a-z_]+:/g, "");
+  return t;
+}
+
+export class SessionManager {
+  private sessions = new Map<string, Session>();
+  private config: SessionConfig;
+  private maxSessions: number;
+  private bufferTtlMs = 30_000;
+
+  constructor(config: SessionConfig, maxSessions = 100) {
+    this.config = config;
+    this.maxSessions = maxSessions;
+  }
+
+  async create(
+    sessionId: string,
+    character?: VoiceCharacter,
+    ws?: ServerWebSocket<{ sessionId: string }> | null,
+  ): Promise<Session> {
+    if (this.sessions.size >= this.maxSessions) {
+      throw new Error(`max sessions reached (${this.maxSessions})`);
+    }
+
+    const obs = new InMemoryObservability();
+    obs.onRecord(consoleExporter);
+
+    const runtimeConfig: Record<string, unknown> = {
+      pipeline: this.config.pipelineConfig?.pipeline ?? "elevenlabs",
+      apiKey: this.config.apiKey,
+      voiceId: this.config.voiceId,
+      modelId: this.config.modelId,
+      ...(this.config.pipelineConfig ?? {}),
+      ...(character ? voiceCharacterToRuntimeConfig(character) : {}),
+      outputFormat: "mp3_44100_128",
+    };
+
+    const runtime = new PipecatRuntime({
+      pythonPath: this.config.pythonPath,
+      cwd: this.config.pythonCwd,
+      observability: obs,
+      config: runtimeConfig,
+    });
+
+    await runtime.connect({ sessionId });
+
+    const slang = {
+      ...DEFAULT_ENGLISH_SLANG,
+      ...DEFAULT_GENZ_SLANG,
+      ...DEFAULT_BOARD_SPORTS_SLANG,
+      ...DEFAULT_LAUGH_TAGS,
+      ...(character?.slang ?? {}),
+    };
+
+    const voice = new VoiceRouter({
+      runtime,
+      character,
+      slang,
+      transforms: [cleanForTTS],
+    });
+
+    const eventBuffer: KithEvent[] = [];
+
+    const unsubscribe = voice.on((event: KithEvent) => {
+      if (ws) {
+        try {
+          ws.send(JSON.stringify(event));
+        } catch {
+          // ws closed
+        }
+      } else {
+        // Buffer events until a WS/SSE client connects
+        eventBuffer.push(event);
+        if (eventBuffer.length > 200) eventBuffer.shift();
+      }
+    });
+
+    const session: Session = {
+      id: sessionId,
+      runtime,
+      voice,
+      obs,
+      unsubscribe,
+      ws: ws ?? null,
+      character,
+      eventBuffer,
+      createdAt: Date.now(),
+    };
+
+    this.sessions.set(sessionId, session);
+    return session;
+  }
+
+  get(id: string): Session | undefined {
+    return this.sessions.get(id);
+  }
+
+  has(id: string): boolean {
+    return this.sessions.has(id);
+  }
+
+  attachWs(sessionId: string, ws: ServerWebSocket<{ sessionId: string }>): boolean {
+    const session = this.sessions.get(sessionId);
+    if (!session) return false;
+    session.ws = ws;
+
+    // Flush buffered events
+    for (const event of session.eventBuffer) {
+      try {
+        ws.send(JSON.stringify(event));
+      } catch {
+        break;
+      }
+    }
+    session.eventBuffer.length = 0;
+
+    return true;
+  }
+
+  async destroy(sessionId: string): Promise<void> {
+    const session = this.sessions.get(sessionId);
+    if (!session) return;
+    this.sessions.delete(sessionId);
+    session.unsubscribe();
+    session.voice.destroy();
+    try {
+      await session.runtime.disconnect();
+    } catch (err) {
+      console.error(`[kith] disconnect failed session=${sessionId}:`, err);
+    }
+    console.log(`[kith] session torn down: ${sessionId}`);
+  }
+
+  stats(): { count: number; ids: string[] } {
+    return {
+      count: this.sessions.size,
+      ids: [...this.sessions.keys()],
+    };
+  }
+}