@krystalnaroo/ai-chat-core 0.1.2

package/README.md

@@ -0,0 +1,46 @@
+# @krytabo/ai-chat-core
+
+Framework-agnostic core for AI Chat WebSocket communication.
+
+## Install
+
+```sh
+yarn add @krystalnaroo/ai-chat-core
+```
+
+## Usage
+
+```js
+import { VoiceAgentClient } from "@krystalnaroo/ai-chat-core";
+
+const client = new VoiceAgentClient({
+  basePath: "/ws/audio",
+});
+
+client.on("state_update", (payload) => {
+  console.log(payload.state);
+});
+
+client.on("text", (payload) => {
+  console.log(payload.source, payload.text);
+});
+
+client.on("audio", (buffer) => {
+  console.log("tts audio", buffer.byteLength);
+});
+
+client.connect();
+
+client.sendText("hello");
+```
+
+## Supported Events
+
+- state_update
+- text
+- trace
+- llm_delta
+- agent_response
+- audio
+- tts_stop
+- error

package/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@krystalnaroo/ai-chat-core",
+  "version": "0.1.2",
+  "description": "Framework-agnostic core for AI Chat WebSocket communication.",
+  "type": "module",
+  "main": "src/index.js",
+  "exports": {
+    ".": "./src/index.js"
+  },
+  "files": [
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/VoiceAgentClient.js

@@ -0,0 +1,242 @@
+import { EventEmitter } from "./events/EventEmitter.js";
+import { createWebSocket } from "./ws/createWebSocket.js";
+import { isNonEmptyBuffer, toArrayBuffer } from "./audio/audioProcessor.js";
+
+export class VoiceAgentClient {
+  constructor(options = {}) {
+    this.options = options;
+    this.ws = undefined;
+    this.emitter = new EventEmitter();
+    this.state = "idle";
+    this.sessionId = null;
+  }
+
+  connect() {
+    if (this.state === "connecting" || this.state === "connected") {
+      return;
+    }
+
+    this.setState("connecting");
+    this.ws = createWebSocket({
+      url: this.options.url,
+      basePath: this.options.basePath,
+      host: this.options.host,
+      port: this.options.port,
+    });
+    this.ws.binaryType = "arraybuffer";
+
+    this.ws.addEventListener("open", () => {
+      this.setState("connected");
+      this.emitter.emit("connected", undefined);
+    });
+
+    this.ws.addEventListener("close", (event) => {
+      this.setState("disconnected");
+      this.emitter.emit("disconnected", event);
+    });
+
+    this.ws.addEventListener("message", (event) => {
+      if (event.data instanceof ArrayBuffer) {
+        this.emitter.emit("audio", event.data);
+        return;
+      }
+
+      if (typeof event.data === "string") {
+        this.handleJsonMessage(event.data);
+        return;
+      }
+
+      if (event.data instanceof Blob) {
+        event.data.arrayBuffer().then((buffer) => {
+          this.emitter.emit("audio", buffer);
+        });
+      }
+    });
+
+    this.ws.addEventListener("error", (event) => {
+      this.emitter.emit("error", event);
+    });
+  }
+
+  disconnect() {
+    if (!this.ws) {
+      return;
+    }
+    this.ws.close();
+    this.ws = undefined;
+    this.setState("disconnected");
+  }
+
+  sendJSON(payload, attachSessionId = true) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      return false;
+    }
+    if (attachSessionId && this.sessionId && typeof payload === 'object') {
+      payload.session_id = this.sessionId;
+    }
+    try {
+      this.ws.send(JSON.stringify(payload));
+      return true;
+    } catch (error) {
+      console.error("Failed to send JSON:", error);
+      return false;
+    }
+  }
+
+  sendText(text) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      return;
+    }
+    if (!text || typeof text !== "string") {
+      return;
+    }
+
+    const payload = {
+      type: "text_input",
+      text,
+    };
+
+    if (this.sessionId) {
+      payload.session_id = this.sessionId;
+    }
+
+    this.ws.send(JSON.stringify(payload));
+  }
+
+  sendAudio(chunk) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      return false;
+    }
+
+    const buffer = toArrayBuffer(chunk);
+    if (!isNonEmptyBuffer(buffer)) {
+      return false;
+    }
+    try {
+      this.ws.send(buffer);
+      return true;
+    } catch (error) {
+      console.error("Failed to send audio:", error);
+      return false;
+    }
+  }
+
+  sendAudioFrameBase64(base64Payload, sampleRate = 16000) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      return false;
+    }
+    if (!base64Payload) {
+      return false;
+    }
+    const payload = {
+      type: "audio_frame",
+      payload: base64Payload,
+      sample_rate: sampleRate,
+    };
+    try {
+      this.ws.send(JSON.stringify(payload));
+      return true;
+    } catch (error) {
+      console.error("Failed to send audio frame:", error);
+      return false;
+    }
+  }
+
+  sendImage(dataUrl) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      return;
+    }
+    if (typeof dataUrl !== "string" || dataUrl.length === 0) {
+      return;
+    }
+    const payload = JSON.stringify({ type: "image", dataUrl });
+    this.ws.send(payload);
+  }
+
+  on(event, handler) {
+    this.emitter.on(event, handler);
+  }
+
+  off(event, handler) {
+    this.emitter.off(event, handler);
+  }
+
+  getState() {
+    return this.state;
+  }
+
+  setState(state) {
+    this.state = state;
+    this.emitter.emit("state", state);
+  }
+
+  handleJsonMessage(raw) {
+    const payload = this.parseJsonMessage(raw);
+    if (!payload || !payload.type) {
+      this.emitter.emit("text", raw);
+      return;
+    }
+
+    if (payload.session_id) {
+      this.sessionId = payload.session_id;
+    }
+    if (payload.type === "session_config" && payload.session_id) {
+      this.sessionId = payload.session_id;
+      this.emitter.emit("session_config", payload);
+      return;
+    }
+
+    switch (payload.type) {
+      case "state_update":
+        this.emitter.emit("state_update", payload.data ?? payload);
+        break;
+      case "state":
+        this.emitter.emit("state_update", payload);
+        break;
+      case "final_text":
+        this.emitter.emit("text", payload.data ?? payload);
+        break;
+      case "text":
+        this.emitter.emit("text", payload);
+        break;
+      case "trace":
+        this.emitter.emit("trace", payload);
+        break;
+      case "llm_delta":
+        this.emitter.emit("llm_delta", payload);
+        break;
+      case "text_response":
+        this.emitter.emit("agent_response", {
+          content: payload.data?.text ?? payload.text ?? "",
+          artifacts: payload.data?.artifacts ?? payload.artifacts ?? [],
+        });
+        break;
+      case "agent_response":
+        this.emitter.emit("agent_response", payload);
+        break;
+      case "tts_stop":
+        this.emitter.emit("tts_stop", payload);
+        break;
+      case "vad_turn_start":
+        this.emitter.emit("vad_turn_start", payload);
+        break;
+      case "agent_resumed":
+        this.emitter.emit("agent_resumed", payload);
+        break;
+      case "error":
+        this.emitter.emit("error", payload);
+        break;
+      default:
+        this.emitter.emit("text", payload);
+        break;
+    }
+  }
+
+  parseJsonMessage(raw) {
+    try {
+      return JSON.parse(raw);
+    } catch (error) {
+      return null;
+    }
+  }
+}

package/src/audio/audioProcessor.js

@@ -0,0 +1,151 @@
+const DEFAULT_SAMPLE_RATE = 16000;
+
+export const toArrayBuffer = (input) => {
+  if (!input) {
+    return null;
+  }
+  if (input instanceof ArrayBuffer) {
+    return input;
+  }
+  if (ArrayBuffer.isView(input)) {
+    return input.buffer.slice(input.byteOffset, input.byteOffset + input.byteLength);
+  }
+  return null;
+};
+
+export const isNonEmptyBuffer = (buffer) =>
+  buffer instanceof ArrayBuffer && buffer.byteLength > 0;
+
+function isWavBuffer(buffer) {
+  if (buffer.byteLength < 12) {
+    return false;
+  }
+  const view = new DataView(buffer);
+  const riff =
+    String.fromCharCode(view.getUint8(0)) +
+    String.fromCharCode(view.getUint8(1)) +
+    String.fromCharCode(view.getUint8(2)) +
+    String.fromCharCode(view.getUint8(3));
+  const wave =
+    String.fromCharCode(view.getUint8(8)) +
+    String.fromCharCode(view.getUint8(9)) +
+    String.fromCharCode(view.getUint8(10)) +
+    String.fromCharCode(view.getUint8(11));
+  return riff === "RIFF" && wave === "WAVE";
+}
+
+function decodePcm16(buffer, audioContext, sampleRate) {
+  const pcm = new Int16Array(buffer);
+  const audioBuffer = audioContext.createBuffer(1, pcm.length, sampleRate);
+  const channel = audioBuffer.getChannelData(0);
+  for (let i = 0; i < pcm.length; i += 1) {
+    channel[i] = pcm[i] / 0x8000;
+  }
+  return audioBuffer;
+}
+
+export class AudioProcessor {
+  constructor({ sampleRate = DEFAULT_SAMPLE_RATE } = {}) {
+    this.sampleRate = sampleRate;
+    this.audioContext = null;
+    this.playingSources = [];
+    this.stopRequested = false;
+  }
+
+  initialize() {
+    if (!this.audioContext) {
+      this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
+      this.analyser = this.audioContext.createAnalyser();
+      this.analyser.fftSize = 256;
+    }
+  }
+
+  isReady() {
+    return this.audioContext && this.audioContext.state === "running";
+  }
+
+  async resume() {
+    this.initialize();
+    if (this.audioContext.state === "suspended") {
+      await this.audioContext.resume();
+    }
+    return this.isReady();
+  }
+
+  stop() {
+    this.stopRequested = true;
+    this.playingSources.forEach((source) => {
+      try {
+        source.stop();
+      } catch (error) {
+        // Ignore stop errors for ended sources.
+      }
+    });
+    this.playingSources = [];
+  }
+
+  getVolume() {
+    if (!this.analyser) return 0;
+    const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
+    this.analyser.getByteTimeDomainData(dataArray);
+    
+    let sum = 0;
+    for(let i = 0; i < dataArray.length; i++) {
+      const v = (dataArray[i] - 128) / 128;
+      sum += v * v;
+    }
+    return Math.sqrt(sum / dataArray.length);
+  }
+
+  async decodeBuffer(buffer) {
+    this.initialize();
+    const arrayBuffer = buffer instanceof ArrayBuffer ? buffer : buffer.buffer.slice(0);
+    if (isWavBuffer(arrayBuffer)) {
+      return this.audioContext.decodeAudioData(arrayBuffer.slice(0));
+    }
+    return decodePcm16(arrayBuffer, this.audioContext, this.sampleRate);
+  }
+
+  async getAudioDuration(buffer) {
+    const audioBuffer = await this.decodeBuffer(buffer);
+    return audioBuffer.duration;
+  }
+
+  async play(buffers, onEnd) {
+    if (!buffers || buffers.length === 0) {
+      return;
+    }
+    await this.resume();
+    this.stop();
+    this.stopRequested = false;
+
+    for (const buffer of buffers) {
+      if (this.stopRequested) {
+        break;
+      }
+      const audioBuffer = await this.decodeBuffer(buffer);
+      await this.playAudioBuffer(audioBuffer);
+    }
+
+    if (onEnd) {
+      onEnd();
+    }
+  }
+
+  playAudioBuffer(audioBuffer) {
+    return new Promise((resolve) => {
+      const source = this.audioContext.createBufferSource();
+      source.buffer = audioBuffer;
+      // Connect: Source -> Analyser -> Destination
+      source.connect(this.analyser);
+      this.analyser.connect(this.audioContext.destination);
+      
+      source.onended = () => {
+        this.playingSources = this.playingSources.filter((item) => item !== source);
+        resolve();
+      };
+      this.playingSources.push(source);
+      source.start();
+    });
+  }
+}

package/src/events/EventEmitter.js

@@ -0,0 +1,27 @@
+export class EventEmitter {
+  constructor() {
+    this.listeners = {};
+  }
+
+  on(event, handler) {
+    if (!this.listeners[event]) {
+      this.listeners[event] = new Set();
+    }
+    this.listeners[event].add(handler);
+  }
+
+  off(event, handler) {
+    const set = this.listeners[event];
+    if (set) {
+      set.delete(handler);
+    }
+  }
+
+  emit(event, payload) {
+    const set = this.listeners[event];
+    if (!set) {
+      return;
+    }
+    set.forEach((handler) => handler(payload));
+  }
+}

package/src/index.js

@@ -0,0 +1,4 @@
+export { VoiceAgentClient } from "./VoiceAgentClient.js";
+export { buildWebSocketUrl, createWebSocket } from "./ws/createWebSocket.js";
+export { AudioProcessor, toArrayBuffer, isNonEmptyBuffer } from "./audio/audioProcessor.js";
+export { EventEmitter } from "./events/EventEmitter.js";

package/src/ws/createWebSocket.js

@@ -0,0 +1,27 @@
+const DEFAULT_BASE_PATH = "/ws/audio";
+
+function normalizeBasePath(basePath) {
+  if (!basePath) {
+    return DEFAULT_BASE_PATH;
+  }
+  return basePath.startsWith("/") ? basePath : `/${basePath}`;
+}
+
+export function buildWebSocketUrl({ basePath, host, port } = {}) {
+  if (typeof location === "undefined") {
+    throw new Error("location is not available in this environment");
+  }
+
+  const protocol = location.protocol === "https:" ? "wss:" : "ws:";
+  const finalHost = host || location.hostname;
+  const path = normalizeBasePath(basePath);
+  if (port === undefined || port === null || port === "") {
+    return `${protocol}//${finalHost}${path}`;
+  }
+  return `${protocol}//${finalHost}:${port}${path}`;
+}
+
+export function createWebSocket({ url, basePath, host, port } = {}) {
+  const finalUrl = url ?? buildWebSocketUrl({ basePath, host, port });
+  return new WebSocket(finalUrl);
+}