@amaster.ai/asr-client 1.0.0-beta.7 → 1.0.0-beta.73

package/dist/index.js

@@ -1,207 +1,554 @@
 // src/asr-client.ts
 var ASR_PATH = "/api/proxy/builtin/platform/qwen-asr-realtime/api-ws/v1/realtime";
-async function listen(onTranscript) {
-  const ws = new WebSocket(ASR_PATH);
-  let mediaStream = null;
-  let audioContext = null;
+async function createRealtimeRecorder() {
+  let stream = null;
+  let ctx = null;
+  let source = null;
   let processor = null;
-  const stop = () => {
-    if (mediaStream) {
-      mediaStream.getTracks().forEach((t) => t.stop());
-      mediaStream = null;
-    }
-    if (processor) {
-      processor.disconnect();
+  if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+    throw new Error("\u672A\u627E\u5230\u9EA6\u514B\u98CE\u6216\u65E0\u6743\u9650");
+  }
+  return {
+    async start(onAudio) {
+      stream = await navigator.mediaDevices.getUserMedia({
+        audio: { sampleRate: 16e3, channelCount: 1, echoCancellation: true }
+      });
+      log("\u2705 \u9EA6\u514B\u98CE\u5DF2\u542F\u52A8", "success");
+      log("\u{1F4AC} \u8BF7\u5BF9\u7740\u9EA6\u514B\u98CE\u8BF4\u8BDD\uFF0C\u5B9E\u65F6\u8BC6\u522B\u4E2D...", "success");
+      ctx = new AudioContext({ sampleRate: 16e3 });
+      source = ctx.createMediaStreamSource(stream);
+      processor = ctx.createScriptProcessor(4096, 1, 1);
+      processor.onaudioprocess = (e) => {
+        const inputData = e.inputBuffer.getChannelData(0);
+        const pcm = new Int16Array(inputData.length);
+        for (let i = 0; i < inputData.length; i++) {
+          const s = Math.max(-1, Math.min(1, inputData[i] || 0));
+          pcm[i] = s < 0 ? s * 32768 : s * 32767;
+        }
+        const bytes = new Uint8Array(pcm.buffer);
+        let binary = "";
+        for (let i = 0; i < bytes.length; i++) {
+          binary += String.fromCharCode(bytes[i] || 0);
+        }
+        onAudio(btoa(binary));
+      };
+      source.connect(processor);
+      processor.connect(ctx.destination);
+    },
+    async stop() {
+      stream?.getTracks().forEach((t) => t.stop());
+      source?.disconnect();
+      processor?.disconnect();
+      if (ctx) {
+        await ctx.close();
+      }
+      stream = null;
+      ctx = null;
+      source = null;
       processor = null;
     }
-    if (audioContext) {
-      audioContext.close();
-      audioContext = null;
-    }
-    if (ws.readyState === WebSocket.OPEN) {
-      ws.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
-      ws.close();
-    }
   };
-  return new Promise((resolve, reject) => {
-    ws.onmessage = async (event) => {
-      const data = JSON.parse(event.data);
-      if (data.type === "session.created") {
-        try {
-          mediaStream = await navigator.mediaDevices.getUserMedia({
-            audio: { sampleRate: 16e3, channelCount: 1, echoCancellation: true }
-          });
-          audioContext = new AudioContext({ sampleRate: 16e3 });
-          const source = audioContext.createMediaStreamSource(mediaStream);
-          processor = audioContext.createScriptProcessor(4096, 1, 1);
-          processor.onaudioprocess = (e) => {
-            if (ws.readyState !== WebSocket.OPEN) return;
-            const input = e.inputBuffer.getChannelData(0);
-            const pcm = new Int16Array(input.length);
-            for (let i = 0; i < input.length; i++) {
-              const s = Math.max(-1, Math.min(1, input[i]));
-              pcm[i] = s < 0 ? s * 32768 : s * 32767;
-            }
-            const bytes = new Uint8Array(pcm.buffer);
-            let binary = "";
-            for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
-            ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: btoa(binary) }));
-          };
-          source.connect(processor);
-          processor.connect(audioContext.destination);
-          resolve(stop);
-        } catch (err) {
-          reject(err);
-        }
-      }
-      if (data.type === "conversation.item.input_audio_transcription.text") {
-        onTranscript(data.text || "", false);
-      }
-      if (data.type === "conversation.item.input_audio_transcription.completed") {
-        onTranscript(data.text || data.transcript || "", true);
-      }
-      if (data.type === "error") {
-        stop();
-        reject(new Error(data.error?.message || "ASR error"));
-      }
-    };
-    ws.onerror = () => reject(new Error("ASR connection failed"));
-  });
+}
+var log = (message, type = "") => {
+  console.log(`[${type}]`, message);
+};
+var eventIdCounter = 0;
+function generateEventId() {
+  return `event_${Date.now()}_${++eventIdCounter}`;
 }
 function createASRClient(config) {
   const {
-    // audioFormat = 'pcm16',
-    sampleRate = 16e3,
     onReady,
     onSpeechStart,
     onSpeechEnd,
     onTranscript,
-    onError
+    onAudioBufferCommitted,
+    onSessionFinished,
+    onError,
+    onClose,
+    getAccessToken,
+    audioFormat = "pcm",
+    sampleRate = 16e3,
+    language = "zh",
+    enableVAD = true,
+    vadThreshold = 0.2,
+    vadSilenceDurationMs = 400
   } = config;
   let ws = null;
-  let mediaStream = null;
-  let audioContext = null;
-  let processor = null;
+  let recorder = null;
+  let isRecordingFlag = false;
+  let isClosing = false;
+  const path = ASR_PATH;
+  function sendEvent(event) {
+    if (!ws || ws.readyState !== WebSocket.OPEN) {
+      throw new Error("WebSocket not connected");
+    }
+    ws.send(JSON.stringify(event));
+  }
+  function buildSessionConfig() {
+    const sessionConfig = {
+      input_audio_format: audioFormat,
+      sample_rate: sampleRate,
+      input_audio_transcription: {
+        language
+      }
+    };
+    if (enableVAD) {
+      sessionConfig.turn_detection = {
+        type: "server_vad",
+        threshold: vadThreshold,
+        silence_duration_ms: vadSilenceDurationMs
+      };
+    } else {
+      sessionConfig.turn_detection = null;
+    }
+    return sessionConfig;
+  }
+  function sendSessionUpdate() {
+    const event = {
+      event_id: generateEventId(),
+      type: "session.update",
+      session: buildSessionConfig()
+    };
+    sendEvent(event);
+  }
+  function sendAudioBufferAppend(audio) {
+    const event = {
+      event_id: generateEventId(),
+      type: "input_audio_buffer.append",
+      audio
+    };
+    sendEvent(event);
+  }
+  function sendAudioBufferCommit() {
+    const event = {
+      event_id: generateEventId(),
+      type: "input_audio_buffer.commit"
+    };
+    sendEvent(event);
+  }
+  function sendSessionFinish() {
+    const event = {
+      event_id: generateEventId(),
+      type: "session.finish"
+    };
+    sendEvent(event);
+  }
+  function handleServerEvent(data) {
+    switch (data.type) {
+      case "session.created":
+        try {
+          sendSessionUpdate();
+        } catch (err2) {
+          onError?.(
+            new Error(
+              "Failed to send session.update: " + (err2 instanceof Error ? err2.message : String(err2))
+            )
+          );
+        }
+        break;
+      case "session.updated":
+        onReady?.();
+        break;
+      case "input_audio_buffer.speech_started":
+        onSpeechStart?.();
+        break;
+      case "input_audio_buffer.speech_stopped":
+        onSpeechEnd?.();
+        break;
+      case "input_audio_buffer.committed":
+        onAudioBufferCommitted?.();
+        break;
+      case "conversation.item.input_audio_transcription.text":
+        onTranscript?.(data.text || data.stash || data.transcript || "", false);
+        break;
+      case "conversation.item.input_audio_transcription.completed":
+        onTranscript?.(data.text || data.transcript || "", true);
+        break;
+      case "session.finished":
+        onSessionFinished?.();
+        close();
+        break;
+      case "error":
+        const err = new Error(data.error?.message || "ASR error");
+        onError?.(err);
+        break;
+      default:
+        console.warn("[ASR] Unknown server event:", data.type);
+    }
+  }
   async function connect() {
+    let wsUrl = path;
+    if (getAccessToken) {
+      const token = getAccessToken();
+      if (token) {
+        const separator = path.includes("?") ? "&" : "?";
+        wsUrl = `${path}${separator}token=${encodeURIComponent(token)}`;
+      }
+    }
+    if (typeof window !== "undefined" && window.location) {
+      const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
+      if (!wsUrl.startsWith("ws://") && !wsUrl.startsWith("wss://")) {
+        wsUrl = `${protocol}//${window.location.host}${wsUrl}`;
+      }
+    }
+    ws = new WebSocket(wsUrl);
     return new Promise((resolve, reject) => {
-      ws = new WebSocket(ASR_PATH);
+      if (!ws) {
+        reject(new Error("Failed to create WebSocket"));
+        return;
+      }
       ws.onopen = () => {
+        log("WebSocket connected", "success");
       };
       ws.onmessage = (event) => {
-        const data = JSON.parse(event.data);
-        if (data.type === "session.created") {
-          onReady?.();
-          resolve();
-        }
-        if (data.type === "input_audio_buffer.speech_started") {
-          onSpeechStart?.();
-        }
-        if (data.type === "input_audio_buffer.speech_stopped") {
-          onSpeechEnd?.();
-        }
-        if (data.type === "conversation.item.input_audio_transcription.text") {
-          onTranscript?.(data.text || "", false);
-        }
-        if (data.type === "conversation.item.input_audio_transcription.completed") {
-          onTranscript?.(data.text || data.transcript || "", true);
-        }
-        if (data.type === "error") {
-          const err = new Error(data.error?.message || "Unknown error");
-          onError?.(err);
-          reject(err);
+        try {
+          const data = JSON.parse(event.data);
+          handleServerEvent(data);
+          if (data.type === "session.updated") {
+            resolve();
+          }
+        } catch (err) {
+          const error = new Error(
+            "Failed to parse server message: " + (err instanceof Error ? err.message : String(err))
+          );
+          onError?.(error);
+          reject(error);
         }
       };
-      ws.onerror = () => {
-        const err = new Error("WebSocket connection error");
+      ws.onerror = (error) => {
+        console.error("WebSocket error:", error);
+        const err = new Error("WebSocket error");
         onError?.(err);
         reject(err);
       };
       ws.onclose = () => {
+        isRecordingFlag = false;
         ws = null;
+        onClose?.();
       };
     });
   }
   async function startRecording() {
-    if (typeof window === "undefined") {
-      throw new Error("Recording only supported in browser");
-    }
     if (!ws || ws.readyState !== WebSocket.OPEN) {
       throw new Error("WebSocket not connected");
     }
+    if (isRecordingFlag) {
+      return;
+    }
     try {
-      mediaStream = await navigator.mediaDevices.getUserMedia({
-        audio: {
-          sampleRate,
-          channelCount: 1,
-          echoCancellation: true,
-          noiseSuppression: true
-        }
-      });
-      audioContext = new AudioContext({ sampleRate });
-      const source = audioContext.createMediaStreamSource(mediaStream);
-      processor = audioContext.createScriptProcessor(4096, 1, 1);
-      processor.onaudioprocess = (e) => {
+      recorder = await createRealtimeRecorder();
+      isRecordingFlag = true;
+      await recorder.start((audio) => {
         if (!ws || ws.readyState !== WebSocket.OPEN) return;
-        const inputData = e.inputBuffer.getChannelData(0);
-        const inputLen = inputData.length;
-        const pcm = new Int16Array(inputLen);
-        for (let i = 0; i < inputLen; i++) {
-          const s = Math.max(-1, Math.min(1, inputData[i]));
-          pcm[i] = s < 0 ? s * 32768 : s * 32767;
-        }
-        const bytes = new Uint8Array(pcm.buffer);
-        const len = bytes.length;
-        let binary = "";
-        for (let i = 0; i < len; i++) {
-          binary += String.fromCharCode(bytes[i]);
+        try {
+          sendAudioBufferAppend(audio);
+        } catch (err) {
+          console.error("[ASR] Failed to send audio:", err);
         }
-        const base64 = btoa(binary);
-        ws.send(JSON.stringify({
-          type: "input_audio_buffer.append",
-          audio: base64
-        }));
-      };
-      source.connect(processor);
-      processor.connect(audioContext.destination);
+      });
     } catch (err) {
-      onError?.(err);
+      console.error("[ASR] Failed to start recorder:", err);
+      onError?.(err instanceof Error ? err : new Error(String(err)));
       throw err;
     }
   }
-  function stopRecording() {
-    if (mediaStream) {
-      mediaStream.getTracks().forEach((track) => track.stop());
-      mediaStream = null;
-    }
-    if (processor) {
-      processor.disconnect();
-      processor = null;
+  async function stopRecording() {
+    if (!isRecordingFlag) {
+      return;
     }
-    if (audioContext) {
-      audioContext.close();
-      audioContext = null;
+    try {
+      await recorder?.stop();
+    } catch (err) {
+      console.error("[ASR] Error stopping recorder:", err);
     }
-    if (ws && ws.readyState === WebSocket.OPEN) {
-      ws.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
+    recorder = null;
+    isRecordingFlag = false;
+    if (!enableVAD && ws?.readyState === WebSocket.OPEN) {
+      try {
+        sendAudioBufferCommit();
+      } catch (err) {
+        console.error("[ASR] Failed to send commit:", err);
+      }
     }
   }
-  function close() {
-    stopRecording();
-    if (ws) {
-      ws.close();
-      ws = null;
+  async function close() {
+    if (isClosing) {
+      return;
+    }
+    isClosing = true;
+    await stopRecording();
+    if (ws?.readyState === WebSocket.OPEN) {
+      try {
+        sendSessionFinish();
+        await new Promise((resolve) => setTimeout(resolve, 1e3));
+      } catch (err) {
+        console.error("[ASR] Failed to send session.finish:", err);
+      }
     }
+    if (ws && ws?.readyState !== WebSocket.CLOSING && ws?.readyState !== WebSocket.CLOSED) {
+      ws?.close();
+    }
+    ws = null;
+    isClosing = false;
+  }
+  function isRecording() {
+    return isRecordingFlag;
+  }
+  function isConnected() {
+    return ws !== null && ws.readyState === WebSocket.OPEN;
   }
   return {
     connect,
     startRecording,
     stopRecording,
-    close
+    close,
+    isRecording,
+    isConnected
+  };
+}
+var asr_client_default = (authConfig) => (config) => createASRClient({ ...authConfig, ...config });
+
+// src/http-asr-client.ts
+import { createHttpClient } from "@amaster.ai/http-client";
+var ASR_HTTP_PATH = "/api/proxy/builtin/platform/qwen-asr/compatible-mode/v1/chat/completions";
+var RECORDER_WORKLET = `
+class RecorderProcessor extends AudioWorkletProcessor {
+  process(inputs) {
+    const input = inputs[0];
+    if (input && input[0]) {
+      this.port.postMessage(input[0].slice(0));
+    }
+    return true;
+  }
+}
+registerProcessor('recorder-processor', RecorderProcessor);
+`;
+async function createWebRecorder(props) {
+  let stream;
+  let ctx;
+  let node;
+  let source;
+  const chunks = [];
+  const cleanup = () => {
+    try {
+      source?.disconnect();
+      node?.disconnect();
+      stream?.getTracks().forEach((t) => t.stop());
+      ctx?.close();
+    } catch (e) {
+    }
+  };
+  return {
+    async start() {
+      try {
+        stream = await navigator.mediaDevices.getUserMedia({
+          audio: {
+            channelCount: 1,
+            echoCancellation: true,
+            noiseSuppression: true,
+            autoGainControl: true
+          }
+        });
+        ctx = new AudioContext();
+        const blob = new Blob([RECORDER_WORKLET], {
+          type: "application/javascript"
+        });
+        const url = URL.createObjectURL(blob);
+        await ctx.audioWorklet.addModule(url);
+        URL.revokeObjectURL(url);
+        source = ctx.createMediaStreamSource(stream);
+        node = new AudioWorkletNode(ctx, "recorder-processor");
+        node.port.onmessage = (e) => {
+          const input = e.data;
+          const pcm = new Int16Array(input.length);
+          for (let i = 0; i < input.length; i++) {
+            const s = Math.max(-1, Math.min(1, input[i] || 0));
+            pcm[i] = s < 0 ? s * 32768 : s * 32767;
+          }
+          chunks.push(pcm);
+        };
+        source.connect(node);
+        props?.onStart?.();
+      } catch (error) {
+        props?.onError?.(
+          error instanceof Error ? error : new Error(String(error))
+        );
+        cleanup();
+      }
+    },
+    async stop() {
+      cleanup();
+      const total = chunks.reduce((s, c) => s + c.length, 0);
+      const pcm = new Int16Array(total);
+      let offset = 0;
+      for (const c of chunks) {
+        pcm.set(c, offset);
+        offset += c.length;
+      }
+      const result = { pcm, sampleRate: ctx?.sampleRate ?? 16e3 };
+      const base64 = await blobToBase64(
+        pcmToWav(result.pcm, result.sampleRate)
+      );
+      props?.onStop?.(base64);
+    }
+  };
+}
+function pcmToWav(pcm, sampleRate) {
+  const buffer = new ArrayBuffer(44 + pcm.length * 2);
+  const view = new DataView(buffer);
+  const write = (o, s) => {
+    for (let i = 0; i < s.length; i++) view.setUint8(o + i, s.charCodeAt(i));
   };
+  write(0, "RIFF");
+  view.setUint32(4, 36 + pcm.length * 2, true);
+  write(8, "WAVE");
+  write(12, "fmt ");
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, 1, true);
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * 2, true);
+  view.setUint16(32, 2, true);
+  view.setUint16(34, 16, true);
+  write(36, "data");
+  view.setUint32(40, pcm.length * 2, true);
+  for (let i = 0; i < pcm.length; i++) {
+    view.setInt16(44 + i * 2, pcm[i] || 0, true);
+  }
+  return new Blob([buffer], { type: "audio/wav" });
+}
+function blobToBase64(blob) {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onloadend = () => {
+      const result = reader.result;
+      resolve(result.split(",")[1] || "");
+    };
+    reader.onerror = reject;
+    reader.readAsDataURL(blob);
+  });
+}
+var AsrHttpClient = class {
+  constructor(config, path) {
+    this.recorder = null;
+    this.path = "";
+    this.recognizing = false;
+    this.http = config.http ?? createHttpClient();
+    this.config = config;
+    this.path = path;
+  }
+  async startRecording() {
+    if (this.recorder) {
+      return;
+    }
+    const options = {
+      onStart: () => {
+        this.config.onRecordingStart?.();
+        this.config.onStatusChange?.("recording");
+      },
+      onStop: async (base64) => {
+        this.config.onStatusChange?.("recognizing");
+        const text = await this.recognizeFile(base64);
+        this.config.onResult?.(text);
+        this.config.onRecordingStop?.();
+        this.config.onStatusChange?.("idle");
+        this.recorder = null;
+      },
+      onError: (err) => {
+        this.config.onError?.(err);
+        this.config.onStatusChange?.("idle");
+        this.recorder = null;
+      }
+    };
+    this.recorder = await (this.config.createRecorder?.(options) ?? createWebRecorder(options));
+    await this.recorder.start();
+  }
+  async stopRecording() {
+    if (this.recorder) {
+      await this.recorder.stop();
+      this.recorder = null;
+    } else {
+      this.config.onResult?.("");
+      this.config.onRecordingStop?.();
+      this.config.onStatusChange?.("idle");
+    }
+  }
+  async recognizeFile(base64) {
+    if (this.recognizing) {
+      return "";
+    }
+    this.recognizing = true;
+    try {
+      const response = await this.http.request({
+        url: this.path,
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        data: JSON.stringify({
+          model: "qwen3-asr-flash",
+          messages: [
+            {
+              role: "user",
+              content: [
+                {
+                  type: "input_audio",
+                  input_audio: { data: `data:audio/wav;base64,${base64}` }
+                }
+              ]
+            }
+          ]
+        })
+      });
+      return response?.data?.choices?.[0]?.message?.content || "";
+    } catch (e) {
+      console.error("ASR recognition error:", e);
+      return "";
+    } finally {
+      this.recognizing = false;
+    }
+  }
+  async recordAndRecognize(ms) {
+    await this.startRecording();
+    await new Promise((r) => setTimeout(r, ms));
+    await this.stopRecording();
+  }
+  async recognizeUrl(url) {
+    try {
+      const res = await this.http.request({
+        url: this.path,
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        data: JSON.stringify({
+          model: "qwen3-asr-flash",
+          messages: [
+            {
+              role: "user",
+              content: [{ type: "input_audio", input_audio: { url } }]
+            }
+          ]
+        })
+      });
+      return res?.data?.choices?.[0]?.message?.content || "";
+    } catch (e) {
+      console.error("ASR recognition error:", e);
+      return "";
+    }
+  }
+};
+function createASRHttpClient(config) {
+  let path = ASR_HTTP_PATH;
+  if (config.getAccessToken) {
+    const token = config.getAccessToken();
+    if (token) {
+      const separator = path.includes("?") ? "&" : "?";
+      path = `${path}${separator}token=${encodeURIComponent(token)}`;
+    }
+  }
+  return new AsrHttpClient(config, path);
 }
+var http_asr_client_default = (authConfig) => (config) => createASRHttpClient({ ...authConfig, ...config });
 export {
-  createASRClient,
-  createASRClient as createAsrClient,
-  listen
+  asr_client_default as createASRClient,
+  http_asr_client_default as createASRHttpClient
 };
 //# sourceMappingURL=index.js.map