@fishaudio/agent-client 0.1.0 → 0.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fishaudio/agent-client",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "Fish Audio voice-agent client SDK for browsers: sessions, realtime audio, transcripts, client tools.",
   "keywords": [
     "fish-audio",
@@ -17,7 +17,7 @@
     "url": "git+https://github.com/fishaudio/fish-agent-sdk-web.git",
     "directory": "packages/client"
   },
-  "homepage": "https://github.com/fishaudio/fish-agent-sdk-web/tree/main/packages/client#readme",
+  "homepage": "https://docs.fish.audio/agents/deploy/web-sdk",
   "bugs": "https://github.com/fishaudio/fish-agent-sdk-web/issues",
   "type": "module",
   "sideEffects": false,
@@ -49,7 +49,7 @@
   },
   "dependencies": {
     "livekit-client": "^2.20.0",
-    "@fishaudio/agent-protocol": "^0.1.0"
+    "@fishaudio/agent-protocol": "^0.2.1"
   },
   "devDependencies": {
     "tsup": "^8.5.0",

package/dist/livekit-B62YPROC.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/transport/livekit.ts"],"sourcesContent":["import {\n  ConnectionState,\n  DisconnectReason,\n  Room,\n  RoomEvent,\n  Track,\n  TrackEvent,\n  type RemoteTrack,\n} from \"livekit-client\";\nimport {\n  AGENT_EVENT_TOPIC,\n  CLIENT_EVENT_TOPIC,\n  type AgentSessionMessage,\n  type ClientSessionMessage,\n  type SessionToken,\n} from \"@fishaudio/agent-protocol\";\nimport { FishAgentError } from \"../errors.js\";\nimport type { Transport, TransportCallbacks, TransportConnectOptions } from \"./types.js\";\n\nfunction hasMessageType(value: unknown): value is { type: string } {\n  return (\n    typeof value === \"object\" &&\n    value !== null &&\n    typeof (value as { type?: unknown }).type === \"string\"\n  );\n}\n\n/** Sole production Transport. LiveKit vocabulary must not escape this file,\n * except the `Room` handed out through the `getRoom()` escape hatch. */\nexport class LiveKitTransport implements Transport {\n  private room?: Room;\n  private callbacks?: TransportCallbacks;\n\n  async connect(sessionToken: SessionToken, options: TransportConnectOptions): Promise<void> {\n    this.callbacks = options.callbacks;\n    const room = new Room({\n      audioCaptureDefaults: {\n        deviceId: options.inputDeviceId,\n        echoCancellation: true,\n        noiseSuppression: true,\n        autoGainControl: true,\n      },\n    });\n    this.room = room;\n\n    room.on(RoomEvent.DataReceived, (payload, _participant, _kind, topic) => {\n      if (topic !== AGENT_EVENT_TOPIC) {\n        return;\n      }\n      let message: unknown;\n      try {\n        message = JSON.parse(new TextDecoder().decode(payload));\n      } catch {\n        return;\n      }\n      if (hasMessageType(message)) {\n        this.callbacks?.onAgentEvent(message as AgentSessionMessage);\n      }\n    });\n\n    const AGENT_STATE_ATTRIBUTE = \"lk.agent.state\";\n    room.on(RoomEvent.ParticipantAttributesChanged, (changed, participant) => {\n      const state = changed[AGENT_STATE_ATTRIBUTE];\n      if (!participant.isLocal && state) {\n        this.callbacks?.onAgentState(state);\n      }\n    });\n\n    room.on(RoomEvent.TrackSubscribed, (track: RemoteTrack) => {\n      if (track.kind === Track.Kind.Audio && track.mediaStreamTrack) {\n        this.callbacks?.onOutputStream(new MediaStream([track.mediaStreamTrack]));\n      }\n    });\n\n    // Both sides' transcripts ride LiveKit's built-in transcription streams:\n    // agent segments are playout-synced delta streams (final on stream close —\n    // an interrupted segment closes containing only the spoken text); user STT\n    // segments are re-published per update with an explicit final attribute,\n    // sent on behalf of the user, so sender identity distinguishes the roles.\n    room.registerTextStreamHandler(\"lk.transcription\", async (reader, participantInfo) => {\n      const attributes = reader.info.attributes ?? {};\n      const segmentId = attributes[\"lk.segment_id\"];\n      if (!segmentId) {\n        return;\n      }\n      const role =\n        participantInfo.identity === room.localParticipant.identity ? \"user\" : \"agent\";\n      const finalAttribute = attributes[\"lk.transcription_final\"] === \"true\";\n      let text = \"\";\n      let aborted = false;\n      try {\n        for await (const delta of reader) {\n          text += delta;\n          this.callbacks?.onTranscription({ segmentId, role, text, final: false });\n        }\n      } catch {\n        aborted = true; // interrupted mid-stream; what arrived is what was spoken\n      }\n      if (text) {\n        const final = role === \"agent\" ? true : finalAttribute && !aborted;\n        this.callbacks?.onTranscription({ segmentId, role, text, final });\n      }\n    });\n\n    room.on(RoomEvent.Reconnecting, () => {\n      this.callbacks?.onConnectionState(\"reconnecting\");\n    });\n    room.on(RoomEvent.Reconnected, () => {\n      // A full reconnect replays the roster from the join response before this\n      // event; an agent still missing here hung up while we were away.\n      if (room.remoteParticipants.size === 0) {\n        this.handleAgentLeft(room);\n        return;\n      }\n      this.callbacks?.onConnectionState(\"connected\");\n    });\n    room.on(RoomEvent.Disconnected, (reason?: DisconnectReason) => {\n      this.callbacks?.onConnectionState(\n        \"disconnected\",\n        reason === undefined ? undefined : DisconnectReason[reason],\n      );\n      // Remote-initiated ends never go through disconnect(); drop the refs here\n      // so getRoom() honors its ended-means-undefined contract and the room\n      // can be collected.\n      if (this.room === room) {\n        this.room = undefined;\n        this.callbacks = undefined;\n      }\n    });\n    // An agent-side hangup ends the worker without closing the room; the\n    // agent participant leaving IS the end-of-call signal for the client.\n    // A full reconnect also clears the roster this way, and can do so while\n    // room.state still reads Connected (Reconnecting is only set right after),\n    // so defer one microtask and require a room that is genuinely connected.\n    room.on(RoomEvent.ParticipantDisconnected, () => {\n      if (room.remoteParticipants.size === 0) {\n        queueMicrotask(() => {\n          if (room.state === ConnectionState.Connected && room.remoteParticipants.size === 0) {\n            this.handleAgentLeft(room);\n          }\n        });\n      }\n    });\n\n    try {\n      await room.connect(sessionToken.livekit_url, sessionToken.token);\n    } catch (cause) {\n      throw new FishAgentError(\"connection_failed\", \"Could not establish the realtime session\", {\n        cause,\n      });\n    }\n    // The attribute is sticky: if the agent joined and set its state before us,\n    // no change event will fire — replay the current value once.\n    for (const participant of room.remoteParticipants.values()) {\n      const state = participant.attributes?.[AGENT_STATE_ATTRIBUTE];\n      if (state) {\n        this.callbacks?.onAgentState(state);\n      }\n    }\n    if (options.microphone !== false) {\n      await room.localParticipant.setMicrophoneEnabled(true);\n      this.publishInputStream();\n    }\n  }\n\n  private handleAgentLeft(room: Room): void {\n    this.callbacks?.onConnectionState(\"disconnected\", \"AGENT_LEFT\");\n    void room.disconnect();\n  }\n\n  async disconnect(): Promise<void> {\n    const room = this.room;\n    this.room = undefined;\n    this.callbacks = undefined;\n    await room?.disconnect();\n  }\n\n  getRoom(): Room | undefined {\n    return this.room;\n  }\n\n  async setMicEnabled(enabled: boolean): Promise<void> {\n    await this.room?.localParticipant.setMicrophoneEnabled(enabled);\n    if (enabled) {\n      // After a mic-less connect the first enable captures the track late;\n      // re-wiring an already-published track is a harmless replace.\n      this.publishInputStream();\n    }\n  }\n\n  async setInputDevice(deviceId: string): Promise<void> {\n    const room = this.room;\n    if (!room) {\n      return;\n    }\n    const previous = room.getActiveDevice(\"audioinput\");\n    let switched: boolean;\n    try {\n      // exact: a wrong id must fail loudly, not silently fall back to another mic.\n      switched = await room.switchActiveDevice(\"audioinput\", deviceId, true);\n    } catch (error) {\n      // livekit stops the current track before acquiring the new device, so a\n      // failed pick would otherwise leave the call silently mic-dead — switch\n      // back before surfacing the failure. livekit reports \"default\" until the\n      // real captured id is known; that pseudo-id only resolves on Chrome, so\n      // restore it non-exactly and let other browsers fall back to any mic.\n      if (previous !== undefined) {\n        await room\n          .switchActiveDevice(\"audioinput\", previous, previous !== \"default\")\n          .catch(() => undefined);\n        this.publishInputStream();\n      }\n      throw error;\n    }\n    if (!switched) {\n      throw new FishAgentError(\n        \"device_change_failed\",\n        \"Could not activate the requested microphone\",\n      );\n    }\n    // A live switch replaces the publication's MediaStreamTrack — re-wire\n    // analysis. While muted, livekit defers the swap to the next unmute and\n    // setMicEnabled re-wires then.\n    this.publishInputStream();\n  }\n\n  /** Local tracks already wired to re-emit their stream when livekit restarts them. */\n  private readonly restartWiredTracks = new WeakSet<object>();\n\n  private publishInputStream(): void {\n    const room = this.room;\n    if (!room) {\n      return;\n    }\n    for (const publication of room.localParticipant.audioTrackPublications.values()) {\n      const track = publication.track;\n      if (track && !this.restartWiredTracks.has(track)) {\n        this.restartWiredTracks.add(track);\n        // Restarts replace the MediaStreamTrack without any Room-level event —\n        // livekit's devicechange auto-recovery and getRoom() escape-hatch\n        // switches would otherwise leave input analysis on a dead track.\n        track.on(TrackEvent.Restarted, () => this.publishInputStream());\n      }\n      const mediaStreamTrack = track?.mediaStreamTrack;\n      if (mediaStreamTrack) {\n        this.callbacks?.onInputStream(new MediaStream([mediaStreamTrack]));\n      }\n    }\n  }\n\n  async sendClientEvent(message: ClientSessionMessage): Promise<void> {\n    if (!this.room) {\n      throw new FishAgentError(\"connection_failed\", \"Session is not connected\");\n    }\n    const payload = new TextEncoder().encode(JSON.stringify(message));\n    await this.room.localParticipant.publishData(payload, {\n      reliable: true,\n      topic: CLIENT_EVENT_TOPIC,\n    });\n  }\n}\n"],"mappings":";;;;;AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP;AAAA,EACE;AAAA,EACA;AAAA,OAIK;AAIP,SAAS,eAAe,OAA2C;AACjE,SACE,OAAO,UAAU,YACjB,UAAU,QACV,OAAQ,MAA6B,SAAS;AAElD;AAIO,IAAM,mBAAN,MAA4C;AAAA,EAA5C;AAsML;AAAA,SAAiB,qBAAqB,oBAAI,QAAgB;AAAA;AAAA,EAlM1D,MAAM,QAAQ,cAA4B,SAAiD;AACzF,SAAK,YAAY,QAAQ;AACzB,UAAM,OAAO,IAAI,KAAK;AAAA,MACpB,sBAAsB;AAAA,QACpB,UAAU,QAAQ;AAAA,QAClB,kBAAkB;AAAA,QAClB,kBAAkB;AAAA,QAClB,iBAAiB;AAAA,MACnB;AAAA,IACF,CAAC;AACD,SAAK,OAAO;AAEZ,SAAK,GAAG,UAAU,cAAc,CAAC,SAAS,cAAc,OAAO,UAAU;AACvE,UAAI,UAAU,mBAAmB;AAC/B;AAAA,MACF;AACA,UAAI;AACJ,UAAI;AACF,kBAAU,KAAK,MAAM,IAAI,YAAY,EAAE,OAAO,OAAO,CAAC;AAAA,MACxD,QAAQ;AACN;AAAA,MACF;AACA,UAAI,eAAe,OAAO,GAAG;AAC3B,aAAK,WAAW,aAAa,OAA8B;AAAA,MAC7D;AAAA,IACF,CAAC;AAED,UAAM,wBAAwB;AAC9B,SAAK,GAAG,UAAU,8BAA8B,CAAC,SAAS,gBAAgB;AACxE,YAAM,QAAQ,QAAQ,qBAAqB;AAC3C,UAAI,CAAC,YAAY,WAAW,OAAO;AACjC,aAAK,WAAW,aAAa,KAAK;AAAA,MACpC;AAAA,IACF,CAAC;AAED,SAAK,GAAG,UAAU,iBAAiB,CAAC,UAAuB;AACzD,UAAI,MAAM,SAAS,MAAM,KAAK,SAAS,MAAM,kBAAkB;AAC7D,aAAK,WAAW,eAAe,IAAI,YAAY,CAAC,MAAM,gBAAgB,CAAC,CAAC;AAAA,MAC1E;AAAA,IACF,CAAC;AAOD,SAAK,0BAA0B,oBAAoB,OAAO,QAAQ,oBAAoB;AACpF,YAAM,aAAa,OAAO,KAAK,cAAc,CAAC;AAC9C,YAAM,YAAY,WAAW,eAAe;AAC5C,UAAI,CAAC,WAAW;AACd;AAAA,MACF;AACA,YAAM,OACJ,gBAAgB,aAAa,KAAK,iBAAiB,WAAW,SAAS;AACzE,YAAM,iBAAiB,WAAW,wBAAwB,MAAM;AAChE,UAAI,OAAO;AACX,UAAI,UAAU;AACd,UAAI;AACF,yBAAiB,SAAS,QAAQ;AAChC,kBAAQ;AACR,eAAK,WAAW,gBAAgB,EAAE,WAAW,MAAM,MAAM,OAAO,MAAM,CAAC;AAAA,QACzE;AAAA,MACF,QAAQ;AACN,kBAAU;AAAA,MACZ;AACA,UAAI,MAAM;AACR,cAAM,QAAQ,SAAS,UAAU,OAAO,kBAAkB,CAAC;AAC3D,aAAK,WAAW,gBAAgB,EAAE,WAAW,MAAM,MAAM,MAAM,CAAC;AAAA,MAClE;AAAA,IACF,CAAC;AAED,SAAK,GAAG,UAAU,cAAc,MAAM;AACpC,WAAK,WAAW,kBAAkB,cAAc;AAAA,IAClD,CAAC;AACD,SAAK,GAAG,UAAU,aAAa,MAAM;AAGnC,UAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,aAAK,gBAAgB,IAAI;AACzB;AAAA,MACF;AACA,WAAK,WAAW,kBAAkB,WAAW;AAAA,IAC/C,CAAC;AACD,SAAK,GAAG,UAAU,cAAc,CAAC,WAA8B;AAC7D,WAAK,WAAW;AAAA,QACd;AAAA,QACA,WAAW,SAAY,SAAY,iBAAiB,MAAM;AAAA,MAC5D;AAIA,UAAI,KAAK,SAAS,MAAM;AACtB,aAAK,OAAO;AACZ,aAAK,YAAY;AAAA,MACnB;AAAA,IACF,CAAC;AAMD,SAAK,GAAG,UAAU,yBAAyB,MAAM;AAC/C,UAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,uBAAe,MAAM;AACnB,cAAI,KAAK,UAAU,gBAAgB,aAAa,KAAK,mBAAmB,SAAS,GAAG;AAClF,iBAAK,gBAAgB,IAAI;AAAA,UAC3B;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,QAAI;AACF,YAAM,KAAK,QAAQ,aAAa,aAAa,aAAa,KAAK;AAAA,IACjE,SAAS,OAAO;AACd,YAAM,IAAI,eAAe,qBAAqB,4CAA4C;AAAA,QACxF;AAAA,MACF,CAAC;AAAA,IACH;AAGA,eAAW,eAAe,KAAK,mBAAmB,OAAO,GAAG;AAC1D,YAAM,QAAQ,YAAY,aAAa,qBAAqB;AAC5D,UAAI,OAAO;AACT,aAAK,WAAW,aAAa,KAAK;AAAA,MACpC;AAAA,IACF;AACA,QAAI,QAAQ,eAAe,OAAO;AAChC,YAAM,KAAK,iBAAiB,qBAAqB,IAAI;AACrD,WAAK,mBAAmB;AAAA,IAC1B;AAAA,EACF;AAAA,EAEQ,gBAAgB,MAAkB;AACxC,SAAK,WAAW,kBAAkB,gBAAgB,YAAY;AAC9D,SAAK,KAAK,WAAW;AAAA,EACvB;AAAA,EAEA,MAAM,aAA4B;AAChC,UAAM,OAAO,KAAK;AAClB,SAAK,OAAO;AACZ,SAAK,YAAY;AACjB,UAAM,MAAM,WAAW;AAAA,EACzB;AAAA,EAEA,UAA4B;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,cAAc,SAAiC;AACnD,UAAM,KAAK,MAAM,iBAAiB,qBAAqB,OAAO;AAC9D,QAAI,SAAS;AAGX,WAAK,mBAAmB;AAAA,IAC1B;AAAA,EACF;AAAA,EAEA,MAAM,eAAe,UAAiC;AACpD,UAAM,OAAO,KAAK;AAClB,QAAI,CAAC,MAAM;AACT;AAAA,IACF;AACA,UAAM,WAAW,KAAK,gBAAgB,YAAY;AAClD,QAAI;AACJ,QAAI;AAEF,iBAAW,MAAM,KAAK,mBAAmB,cAAc,UAAU,IAAI;AAAA,IACvE,SAAS,OAAO;AAMd,UAAI,aAAa,QAAW;AAC1B,cAAM,KACH,mBAAmB,cAAc,UAAU,aAAa,SAAS,EACjE,MAAM,MAAM,MAAS;AACxB,aAAK,mBAAmB;AAAA,MAC1B;AACA,YAAM;AAAA,IACR;AACA,QAAI,CAAC,UAAU;AACb,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAIA,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAKQ,qBAA2B;AACjC,UAAM,OAAO,KAAK;AAClB,QAAI,CAAC,MAAM;AACT;AAAA,IACF;AACA,eAAW,eAAe,KAAK,iBAAiB,uBAAuB,OAAO,GAAG;AAC/E,YAAM,QAAQ,YAAY;AAC1B,UAAI,SAAS,CAAC,KAAK,mBAAmB,IAAI,KAAK,GAAG;AAChD,aAAK,mBAAmB,IAAI,KAAK;AAIjC,cAAM,GAAG,WAAW,WAAW,MAAM,KAAK,mBAAmB,CAAC;AAAA,MAChE;AACA,YAAM,mBAAmB,OAAO;AAChC,UAAI,kBAAkB;AACpB,aAAK,WAAW,cAAc,IAAI,YAAY,CAAC,gBAAgB,CAAC,CAAC;AAAA,MACnE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB,SAA8C;AAClE,QAAI,CAAC,KAAK,MAAM;AACd,YAAM,IAAI,eAAe,qBAAqB,0BAA0B;AAAA,IAC1E;AACA,UAAM,UAAU,IAAI,YAAY,EAAE,OAAO,KAAK,UAAU,OAAO,CAAC;AAChE,UAAM,KAAK,KAAK,iBAAiB,YAAY,SAAS;AAAA,MACpD,UAAU;AAAA,MACV,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AACF;","names":[]}