@fishaudio/agent-client 0.1.0 → 0.2.1

package/README.md

@@ -2,7 +2,7 @@
 
 Framework-agnostic JavaScript/TypeScript SDK for embedding [Fish Audio](https://fish.audio) voice agents into any website or web app: realtime voice over WebRTC, live transcripts, text input, and client tools — behind one small, event-driven API.
 
-Using React? [`@fishaudio/agent-react`](https://github.com/fishaudio/fish-agent-sdk-web/tree/main/packages/react) wraps this SDK in hooks and components.
+Using React? [`@fishaudio/agent-react`](https://www.npmjs.com/package/@fishaudio/agent-react) wraps this SDK in hooks and components.
 
 ## Installation
 
@@ -36,10 +36,10 @@ Starting a session requests microphone access; call `start()` from a user gestur
 
 ## Documentation
 
-- [Quickstart](https://github.com/fishaudio/fish-agent-sdk-web/blob/main/docs/quickstart.md) — first call, with or without a backend.
-- [Authentication](https://github.com/fishaudio/fish-agent-sdk-web/blob/main/docs/authentication.md) — server-created session tokens vs. public agents.
-- [Sessions](https://github.com/fishaudio/fish-agent-sdk-web/blob/main/docs/sessions.md) — `start()` options, the session API, lifecycle and reconnection.
-- [Events](https://github.com/fishaudio/fish-agent-sdk-web/blob/main/docs/events.md) — event reference and live transcripts.
-- [Client tools](https://github.com/fishaudio/fish-agent-sdk-web/blob/main/docs/client-tools.md) — let the agent call functions in the browser.
-- [Customization](https://github.com/fishaudio/fish-agent-sdk-web/blob/main/docs/customization.md) — per-session overrides and dynamic variables.
-- [Errors](https://github.com/fishaudio/fish-agent-sdk-web/blob/main/docs/errors.md) — `FishAgentError` codes.
+- [Quickstart](https://docs.fish.audio/agents/quickstart) — first call, with or without a backend.
+- [Authenticated sessions](https://docs.fish.audio/agents/deploy/authenticated-sessions) and [public agents](https://docs.fish.audio/agents/deploy/public-agents) — server-created session tokens vs. a public agent id.
+- [Web SDK reference](https://docs.fish.audio/agents/deploy/web-sdk) — `start()` options, the session API, lifecycle and reconnection.
+- [Events](https://docs.fish.audio/agents/deploy/web-sdk#events) — event reference and live transcripts.
+- [Client tools](https://docs.fish.audio/agents/build/client-tools) — let the agent call functions in the browser.
+- [Overrides](https://docs.fish.audio/agents/deploy/authenticated-sessions#overrides) and [dynamic variables](https://docs.fish.audio/agents/build/dynamic-variables) — per-session customization.
+- [Errors](https://docs.fish.audio/agents/deploy/web-sdk#errors) — `FishAgentError` codes.

package/dist/index.cjs

@@ -65,6 +65,14 @@ __export(livekit_exports, {
 function hasMessageType(value) {
   return typeof value === "object" && value !== null && typeof value.type === "string";
 }
+function captureMicrophone(inputDeviceId) {
+  return (0, import_livekit_client.createLocalAudioTrack)({
+    deviceId: inputDeviceId,
+    echoCancellation: true,
+    noiseSuppression: true,
+    autoGainControl: true
+  });
+}
 var import_livekit_client, import_agent_protocol2, LiveKitTransport;
 var init_livekit = __esm({
   "src/transport/livekit.ts"() {
@@ -77,6 +85,11 @@ var init_livekit = __esm({
         /** Local tracks already wired to re-emit their stream when livekit restarts them. */
         this.restartWiredTracks = /* @__PURE__ */ new WeakSet();
       }
+      prepareMicrophone(options) {
+        const pending = captureMicrophone(options.inputDeviceId);
+        pending.catch(() => void 0);
+        this.pendingMicTrack = pending;
+      }
       async connect(sessionToken, options) {
         this.callbacks = options.callbacks;
         const room = new import_livekit_client.Room({
@@ -166,6 +179,12 @@ var init_livekit = __esm({
             });
           }
         });
+        let micTrack;
+        if (options.microphone !== false) {
+          const pending = this.pendingMicTrack ?? captureMicrophone(options.inputDeviceId);
+          this.pendingMicTrack = pending;
+          micTrack = await pending;
+        }
         try {
           await room.connect(sessionToken.livekit_url, sessionToken.token);
         } catch (cause) {
@@ -179,8 +198,9 @@ var init_livekit = __esm({
             this.callbacks?.onAgentState(state);
           }
         }
-        if (options.microphone !== false) {
-          await room.localParticipant.setMicrophoneEnabled(true);
+        if (micTrack) {
+          await room.localParticipant.publishTrack(micTrack);
+          this.pendingMicTrack = void 0;
           this.publishInputStream();
         }
       }
@@ -189,6 +209,9 @@ var init_livekit = __esm({
         void room.disconnect();
       }
       async disconnect() {
+        const pendingMic = this.pendingMicTrack;
+        this.pendingMicTrack = void 0;
+        void pendingMic?.then((track) => track.stop()).catch(() => void 0);
         const room = this.room;
         this.room = void 0;
         this.callbacks = void 0;
@@ -263,7 +286,8 @@ var src_exports = {};
 __export(src_exports, {
   AgentSession: () => AgentSession,
   DEFAULT_SERVER_URL: () => DEFAULT_SERVER_URL,
-  FishAgentError: () => FishAgentError
+  FishAgentError: () => FishAgentError,
+  MAX_CLIENT_TOOL_RESULT_BYTES: () => MAX_CLIENT_TOOL_RESULT_BYTES
 });
 module.exports = __toCommonJS(src_exports);
 
@@ -399,6 +423,22 @@ var AudioStreamAnalyser = class {
 // src/session/agentSession.ts
 init_errors();
 
+// src/events.ts
+var AGENT_SESSION_EVENT_NAMES = [
+  "connect",
+  "disconnect",
+  "statusChange",
+  "modeChange",
+  "userTranscript",
+  "agentResponseDelta",
+  "agentResponse",
+  "message",
+  "toolCallStarted",
+  "toolCallCompleted",
+  "toolCallFailed",
+  "error"
+];
+
 // src/sessionToken.ts
 var import_agent_protocol = require("@fishaudio/agent-protocol");
 init_errors();
@@ -542,6 +582,8 @@ var TypedEmitter = class {
 // src/session/toolDispatcher.ts
 init_errors();
 var DEFAULT_CLIENT_TOOL_TIMEOUT_MS = 15e3;
+var MAX_CLIENT_TOOL_RESULT_BYTES = 6e4;
+var textEncoder = new TextEncoder();
 var ClientToolDispatcher = class {
   constructor(tools, timeoutMs, send, onError) {
     this.timeoutMs = timeoutMs;
@@ -586,14 +628,56 @@ var ClientToolDispatcher = class {
     }
   }
   async reply(callId, body) {
+    const message = { type: "client_tool.result", callId, ...body };
+    const rejected = this.undeliverable(message);
+    if (rejected !== void 0) {
+      this.onError(new FishAgentError("tool_failed", rejected));
+      await this.deliver({ type: "client_tool.result", callId, isError: true, result: rejected });
+      return;
+    }
     try {
-      await this.send({ type: "client_tool.result", callId, ...body });
+      await this.send(message);
     } catch (error) {
       this.onError(
         new FishAgentError("tool_failed", "Failed to deliver a client tool result", {
           cause: error
         })
       );
+      if (body.isError) {
+        return;
+      }
+      await this.deliver({
+        type: "client_tool.result",
+        callId,
+        isError: true,
+        result: `Client tool result could not be delivered: ${String(error)}`
+      });
+    }
+  }
+  /** Why the message cannot go over the wire as-is, or undefined if it can. */
+  undeliverable(message) {
+    let serialized;
+    try {
+      serialized = JSON.stringify(message);
+    } catch (error) {
+      return `Client tool result is not JSON-serializable: ${String(error)}`;
+    }
+    const bytes = textEncoder.encode(serialized).byteLength;
+    if (bytes > MAX_CLIENT_TOOL_RESULT_BYTES) {
+      return `Client tool result is too large to send (${bytes} bytes serialized; limit ${MAX_CLIENT_TOOL_RESULT_BYTES}). Return a summary or a reference instead.`;
+    }
+    return void 0;
+  }
+  /** Last-resort send of an error result; a failure here is reported, not retried. */
+  async deliver(message) {
+    try {
+      await this.send(message);
+    } catch (error) {
+      this.onError(
+        new FishAgentError("tool_failed", "Failed to deliver a client tool error result", {
+          cause: error
+        })
+      );
     }
   }
   withTimeout(promise, toolName) {
@@ -676,6 +760,29 @@ request_fn = async function() {
 
 // src/session/agentSession.ts
 var AGENT_JOIN_TIMEOUT_MS = 15e3;
+var EVENT_NAMES = new Set(AGENT_SESSION_EVENT_NAMES);
+function resolveCallbackSubscriptions(callbacks) {
+  const subscriptions = [];
+  for (const [key, callback] of Object.entries(callbacks ?? {})) {
+    if (callback === void 0) {
+      continue;
+    }
+    const event = /^on[A-Z]/.test(key) ? key.charAt(2).toLowerCase() + key.slice(3) : void 0;
+    if (event === void 0 || !EVENT_NAMES.has(event)) {
+      const expected = AGENT_SESSION_EVENT_NAMES.map(
+        (name) => `on${name.charAt(0).toUpperCase()}${name.slice(1)}`
+      ).join(", ");
+      throw new TypeError(
+        `Unknown callback "${key}". Callback keys are event names prefixed with "on": ${expected}.`
+      );
+    }
+    if (typeof callback !== "function") {
+      throw new TypeError(`Callback "${key}" must be a function, got ${typeof callback}.`);
+    }
+    subscriptions.push([event, callback]);
+  }
+  return subscriptions;
+}
 var startWith;
 var _status, _mode, _endReason, _sessionId, _micMuted, _endedByClient, _ending, _transport, _tools, _output, _outputAnalyser, _inputAnalyser, _wakeLock, _agentSegments, _transcript, _transcriptIndex, _typedMessageCount, _agentPresent, _pendingMessages, _joinTimer, _AgentSession_instances, endOnce_fn, _AgentSession_static, startWith_fn, asStartError_fn, asMicError_fn, asDeviceError_fn, sendClientEvent_fn, setStatus_fn, setMode_fn, handleAgentPresent_fn, abandonAgentWait_fn, handleConnectionState_fn, finalize_fn, handleAgentEvent_fn, recordSegment_fn, handleTranscription_fn, recordTypedUserMessage_fn;
 var _AgentSession = class _AgentSession extends TypedEmitter {
@@ -747,13 +854,13 @@ var _AgentSession = class _AgentSession extends TypedEmitter {
     return __privateGet(this, _transcript).map((segment) => ({ ...segment }));
   }
   // ---- text & control channel ----
-  /** `audio: false` asks the agent to answer this turn in text only (no TTS). */
+  /** Typed turns get a text-only reply by default (no TTS); pass `audio: true` to have the agent speak this turn. */
   sendUserMessage(text, options) {
     const trimmed = text.trim();
     if (!trimmed) {
       return;
     }
-    const message = options?.audio === false ? { type: "user.message", text: trimmed, audio: false } : { type: "user.message", text: trimmed };
+    const message = options?.audio === true ? { type: "user.message", text: trimmed } : { type: "user.message", text: trimmed, audio: false };
     if (__privateGet(this, _agentPresent)) {
       __privateMethod(this, _AgentSession_instances, sendClientEvent_fn).call(this, message);
     } else {
@@ -896,16 +1003,26 @@ startWith_fn = async function(options, createTransport) {
   if (options.audio?.outputDeviceId !== void 0) {
     assertOutputSelectionSupported();
   }
-  const sessionToken = await resolveSessionToken(options);
-  const session = new _AgentSession(sessionToken);
-  if (options.audio?.outputDeviceId !== void 0) {
-    await __privateGet(session, _output).setSinkId(options.audio.outputDeviceId);
+  const callbackSubscriptions = resolveCallbackSubscriptions(options.callbacks);
+  const transport = createTransport();
+  if (options.microphone !== false) {
+    transport.prepareMicrophone?.({ inputDeviceId: options.audio?.inputDeviceId });
+  }
+  let sessionToken;
+  let session;
+  try {
+    sessionToken = await resolveSessionToken(options);
+    session = new _AgentSession(sessionToken);
+    if (options.audio?.outputDeviceId !== void 0) {
+      await __privateGet(session, _output).setSinkId(options.audio.outputDeviceId);
+    }
+  } catch (error) {
+    await transport.disconnect().catch(() => void 0);
+    throw error;
   }
-  for (const [key, callback] of Object.entries(options.callbacks ?? {})) {
-    const event = key.charAt(2).toLowerCase() + key.slice(3);
+  for (const [event, callback] of callbackSubscriptions) {
     session.on(event, callback);
   }
-  const transport = createTransport(sessionToken);
   __privateSet(session, _transport, transport);
   if (options.microphone === false) {
     __privateSet(session, _micMuted, true);
@@ -1187,6 +1304,7 @@ init_errors();
 0 && (module.exports = {
   AgentSession,
   DEFAULT_SERVER_URL,
-  FishAgentError
+  FishAgentError,
+  MAX_CLIENT_TOOL_RESULT_BYTES
 });
 //# sourceMappingURL=index.cjs.map