@talqing/sdk 0.1.0

package/src/browser.tsx

@@ -0,0 +1,675 @@
+"use client";
+
+import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import {
+  ParticipantKind,
+  Room,
+  RpcError,
+  TokenSource,
+  Track,
+  setLogLevel,
+  LogLevel,
+  type RoomOptions,
+  type RemoteParticipant,
+  type RpcInvocationData,
+} from "livekit-client";
+import {
+  RoomAudioRenderer,
+  SessionProvider,
+  VideoTrack,
+  useAgent,
+  useParticipantTracks,
+  useRemoteParticipants,
+  useSession,
+  useSessionContext,
+  useSessionMessages,
+  useTrackToggle,
+  useVoiceAssistant,
+  type ReceivedMessage,
+  type TrackReference,
+  type UseSessionReturn,
+} from "@livekit/components-react";
+import type { JsonObject, JsonValue, TalqingClient } from "./client.js";
+import type {
+  ScreenShareCaptureOptions,
+  TrackPublishOptions,
+} from "livekit-client";
+import type { CallTokenResponse, TokenRequest } from "./gen/types.gen.js";
+
+/** Everything about one web call except which agent runs it. Wire-shaped, like
+ *  every other request type in this SDK. */
+export type TalqingCallOptions = Omit<TokenRequest, "agent_id">;
+
+export interface TalqingBrowserSessionOptions {
+  client: TalqingClient;
+  agentId: string | null;
+  roomOptions?: RoomOptions;
+  callOptions?: TalqingCallOptions;
+  onTokenDetails?: (details: CallTokenResponse) => void;
+}
+
+export interface TalqingRpcInvocation {
+  payload: string;
+  responseTimeout?: number;
+  callerIdentity?: string;
+}
+
+export type TalqingRpcHandler = (
+  data: TalqingRpcInvocation,
+) => string | Promise<string>;
+export type TalqingRpcHandlers = Record<string, TalqingRpcHandler>;
+
+export interface TalqingFrontendRpcEnvelope {
+  method: string;
+  payload?: JsonValue;
+}
+
+export class TalqingRpcError extends Error {
+  readonly code: number;
+  readonly data?: string;
+
+  constructor(code: number, message: string, data?: string) {
+    super(message);
+    this.name = "TalqingRpcError";
+    this.code = code;
+    this.data = data;
+  }
+}
+
+export type TalqingSession = UseSessionReturn;
+export type TalqingMessage = ReceivedMessage;
+export type TalqingVideoTrackRef = TrackReference;
+
+export function createTalqingTokenSource(
+  client: TalqingClient,
+  agentId: string | null,
+  options: TalqingCallOptions = {},
+  onTokenDetails?: (details: CallTokenResponse) => void,
+) {
+  // Carry the contact across reconnects. POST /v1/calls/token mints a key when
+  // none is supplied, so re-running this without pinning it would file every
+  // reconnect as a different person.
+  let resolvedContactKey = options.contact_key ?? null;
+
+  // `useSession` fetches a token twice off one click — once to warm the
+  // connection, once to connect. Unserialised they race: both go out before
+  // either resolves, so neither carries a key, the API mints one per request,
+  // and a single call is filed against two different contacts. Concurrent
+  // callers share one request.
+  //
+  // This coalesces, it does not cache: the promise is dropped once it settles,
+  // so every later connect still mints a fresh token. That matters — a token
+  // bakes in {agent version, room}, so a reused one silently runs a stale
+  // version after a republish, and reconnects into a dead room.
+  let inFlight: Promise<{ serverUrl: string; participantToken: string }> | null = null;
+
+  return TokenSource.literal(async () => {
+    if (inFlight) return inFlight;
+    const request = (async () => {
+      const details = await client.calls.token({
+        tokenRequest: { ...options, agent_id: agentId, contact_key: resolvedContactKey },
+      });
+      resolvedContactKey = details.contact_key;
+      onTokenDetails?.(details);
+      return {
+        serverUrl: details.server_url,
+        participantToken: details.participant_token,
+      };
+    })();
+    inFlight = request;
+    try {
+      return await request;
+    } finally {
+      inFlight = null;
+    }
+  });
+}
+
+/* Echo cancellation is what stops the agent hearing its own speech through the
+ * caller's microphone and interrupting itself. It is a default rather than
+ * something each app remembers, because a call without it is broken in a way
+ * that sounds like a bad model. Pass `roomOptions.audioCaptureDefaults` to
+ * override. */
+export const TALQING_AUDIO_CAPTURE_DEFAULTS = {
+  echoCancellation: true,
+  noiseSuppression: true,
+  autoGainControl: true,
+} as const;
+
+export function useTalqingSession({
+  client,
+  agentId,
+  roomOptions,
+  callOptions,
+  onTokenDetails,
+}: TalqingBrowserSessionOptions): TalqingSession {
+  const tokenSource = useMemo(
+    () => createTalqingTokenSource(client, agentId, callOptions, onTokenDetails),
+    [client, agentId, callOptions, onTokenDetails],
+  );
+  const room = useMemo(
+    () =>
+      new Room({ audioCaptureDefaults: TALQING_AUDIO_CAPTURE_DEFAULTS, ...roomOptions }),
+    [roomOptions],
+  );
+  return useSession(tokenSource, { room });
+}
+
+export function TalqingSessionProvider({
+  session,
+  children,
+  audio = true,
+}: {
+  session: TalqingSession;
+  children: React.ReactNode;
+  audio?: boolean;
+}) {
+  return (
+    <SessionProvider session={session}>
+      {children}
+      {audio ? <RoomAudioRenderer /> : null}
+    </SessionProvider>
+  );
+}
+
+export function useTalqingConnection() {
+  const ctx = useSessionContext();
+  return {
+    isConnected: ctx.isConnected,
+    connectionState: ctx.connectionState,
+    room: ctx.room,
+    start: ctx.start,
+    end: ctx.end,
+  };
+}
+
+export async function setTalqingUserdata(
+  room: Room,
+  patch: JsonObject,
+  responseTimeout = 3,
+): Promise<{ ok: boolean; error?: string }> {
+  const agent = [...room.remoteParticipants.values()].find(
+    (participant: RemoteParticipant) => participant.kind === ParticipantKind.AGENT,
+  );
+  if (!agent) return { ok: false, error: "agent participant is not connected" };
+  const response = await room.localParticipant.performRpc({
+    destinationIdentity: agent.identity,
+    method: "talqing.client.userdata_set",
+    payload: JSON.stringify(patch),
+    responseTimeout,
+  });
+  return JSON.parse(response || "{}") as { ok: boolean; error?: string };
+}
+
+export async function getTalqingUserdata(
+  room: Room,
+  keys?: string[],
+  responseTimeout = 3,
+): Promise<JsonObject> {
+  const agent = [...room.remoteParticipants.values()].find(
+    (participant: RemoteParticipant) => participant.kind === ParticipantKind.AGENT,
+  );
+  if (!agent) throw new Error("agent participant is not connected");
+  const response = await room.localParticipant.performRpc({
+    destinationIdentity: agent.identity,
+    method: "talqing.client.userdata_get",
+    payload: JSON.stringify(keys ? { keys } : {}),
+    responseTimeout,
+  });
+  const parsed = JSON.parse(response || "{}") as JsonObject & { ok?: boolean; error?: string };
+  if (parsed.ok === false) {
+    throw new Error(String(parsed.error || "userdata RPC failed"));
+  }
+  return parsed;
+}
+
+export function useTalqingUserdata() {
+  const { room } = useTalqingConnection();
+  return useMemo(
+    () => ({
+      get: (keys?: string[], responseTimeout?: number) => {
+        if (!room) throw new Error("room is not connected");
+        return getTalqingUserdata(room, keys, responseTimeout);
+      },
+      set: (patch: JsonObject, responseTimeout?: number) => {
+        if (!room) throw new Error("room is not connected");
+        return setTalqingUserdata(room, patch, responseTimeout);
+      },
+    }),
+    [room],
+  );
+}
+
+/* ── images ───────────────────────────────────────────────────────────────
+ *
+ * A caller attaches a photo on a web voice or video call and the agent sees
+ * it. The bytes go over a LiveKit byte stream on the `talqing.images` topic,
+ * addressed to the agent; the agent answers on an RPC back, and the id
+ * `sendFile` returns is the id that ack carries, so there is nothing to
+ * correlate by hand.
+ *
+ * The agent deliberately does NOT speak when a photo lands — the caller is
+ * mid-conversation and about to say what it is for. The image is in the
+ * model's view for their next turn.
+ */
+
+export const TALQING_IMAGE_TOPIC = "talqing.images";
+/** Decoded size the server refuses above. The downscale below keeps a phone
+ *  photo far under it; this is the belt for a file that is not a photo. */
+export const TALQING_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
+/** Longest side the server stores. Downscaling to it before sending is a
+ *  courtesy to the data channel, which is ordered and reliable and will
+ *  happily spend a minute pushing 10 MB past a call's audio. */
+export const TALQING_IMAGE_MAX_EDGE_PX = 1568;
+const TALQING_IMAGE_TYPES = ["image/jpeg", "image/png", "image/webp"] as const;
+/** How long to wait for the agent's ack before saying we do not know. Without
+ *  it a thumbnail sits on "sending" for the rest of the call when the worker
+ *  dies mid-upload. */
+const TALQING_IMAGE_ACK_TIMEOUT_MS = 30_000;
+
+export type TalqingImageStatus = "sending" | "sent" | "failed" | "unknown";
+
+export interface TalqingSentImage {
+  /** The stream id `sendFile` returned, which the ack comes back under. */
+  streamId: string;
+  name: string;
+  /** An object URL for the downscaled image, for rendering the thumbnail.
+   *  Revoked when the hook unmounts. */
+  previewUrl: string;
+  status: TalqingImageStatus;
+  /** Set when `status` is "failed" or "unknown" — a sentence, shown as-is. */
+  error?: string;
+}
+
+/** Refuse what the browser can see is wrong, instantly and with no round trip.
+ *  Returns the sentence to show, or null when the file is fine. The server
+ *  re-validates everything regardless — this exists so the common mistakes cost
+ *  nothing to catch. */
+export function talqingImageRejection(file: File): string | null {
+  if (!(TALQING_IMAGE_TYPES as readonly string[]).includes(file.type)) {
+    return "that file is not a JPEG, PNG or WebP";
+  }
+  if (file.size > TALQING_IMAGE_MAX_BYTES) return "images are 10 MB or smaller";
+  return null;
+}
+
+/** Downscale to the longest edge the server keeps, re-encoding as JPEG or PNG.
+ *
+ *  Also what normalizes whatever the OS handed the file input — this is how a
+ *  Safari HEIC becomes a JPEG. Throws if the image cannot be decoded, which is
+ *  the third dropzone check. */
+export async function talqingDownscaleImage(file: File): Promise<Blob> {
+  const bitmap = await createImageBitmap(file);
+  try {
+    const longest = Math.max(bitmap.width, bitmap.height);
+    const scale = longest > TALQING_IMAGE_MAX_EDGE_PX ? TALQING_IMAGE_MAX_EDGE_PX / longest : 1;
+    const canvas = document.createElement("canvas");
+    canvas.width = Math.max(1, Math.round(bitmap.width * scale));
+    canvas.height = Math.max(1, Math.round(bitmap.height * scale));
+    const context = canvas.getContext("2d");
+    if (!context) throw new Error("this browser cannot resize images");
+    context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
+    // PNG keeps transparency; everything else becomes a JPEG, including the
+    // HEIC the OS decoded for us.
+    const type = file.type === "image/png" ? "image/png" : "image/jpeg";
+    const blob = await new Promise<Blob | null>((resolve) =>
+      canvas.toBlob(resolve, type, 0.85),
+    );
+    if (!blob) throw new Error("that image could not be read");
+    return blob;
+  } finally {
+    bitmap.close();
+  }
+}
+
+/** Validate and downscale a file into the `data:` URL a text message carries.
+ *
+ *  `createTextMessage` takes `images: [{data_url, filename}]`; this produces the
+ *  `data_url`. It doubles as the thumbnail's `src`, so there is no object URL to
+ *  create and revoke. Throws the sentence to show if the file is refused. */
+export async function talqingImageDataUrl(file: File): Promise<string> {
+  const rejection = talqingImageRejection(file);
+  if (rejection) throw new Error(rejection);
+  const blob = await talqingDownscaleImage(file);
+  return await new Promise<string>((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onerror = () => reject(new Error("that image could not be read"));
+    reader.onload = () => resolve(String(reader.result));
+    reader.readAsDataURL(blob);
+  });
+}
+
+/** Send images to the agent on this call, and follow what became of each.
+ *
+ *  `send` resolves once the bytes are on their way; the outcome arrives on the
+ *  returned list, which is what a thumbnail should render.
+ *
+ *  Takes the `Room` rather than reading it from the session context: a byte
+ *  stream and an RPC handler are room-level things, and passing the room keeps
+ *  this usable from a surface that manages its own connection —
+ *  `useTalqingConnection().room` inside a provider, or the Room you built. */
+export function useTalqingImages(room: Room | null | undefined) {
+  const [sent, setSent] = useState<TalqingSentImage[]>([]);
+  const timers = useRef(new Map<string, ReturnType<typeof setTimeout>>());
+
+  const settle = useCallback((streamId: string, patch: Partial<TalqingSentImage>) => {
+    const timer = timers.current.get(streamId);
+    if (timer) {
+      clearTimeout(timer);
+      timers.current.delete(streamId);
+    }
+    setSent((images) =>
+      images.map((image) => (image.streamId === streamId ? { ...image, ...patch } : image)),
+    );
+  }, []);
+
+  useEffect(() => {
+    if (!room) return;
+    room.registerRpcMethod("talqing.image_result", async (data: RpcInvocationData) => {
+      const { stream_id, ok, error } = JSON.parse(data.payload || "{}") as {
+        stream_id: string;
+        ok: boolean;
+        error?: string | null;
+      };
+      settle(stream_id, {
+        status: ok ? "sent" : "failed",
+        error: ok ? undefined : error || "that image was not accepted",
+      });
+      return "{}";
+    });
+    return () => room.unregisterRpcMethod("talqing.image_result");
+  }, [room, settle]);
+
+  const pending = timers.current;
+  useEffect(
+    () => () => {
+      pending.forEach(clearTimeout);
+      pending.clear();
+    },
+    [pending],
+  );
+
+  const send = useCallback(
+    async (file: File): Promise<TalqingSentImage> => {
+      if (!room) throw new Error("the call is not connected");
+      const agent = [...room.remoteParticipants.values()].find(
+        (participant: RemoteParticipant) => participant.kind === ParticipantKind.AGENT,
+      );
+      if (!agent) throw new Error("agent participant is not connected");
+      const rejection = talqingImageRejection(file);
+      if (rejection) throw new Error(rejection);
+
+      const blob = await talqingDownscaleImage(file);
+      // Addressed to the agent, never broadcast: on video the avatar worker is
+      // a participant too, and a broadcast sends the bytes twice.
+      const { id } = await room.localParticipant.sendFile(
+        new File([blob], file.name, { type: blob.type }),
+        {
+          topic: TALQING_IMAGE_TOPIC,
+          mimeType: blob.type,
+          destinationIdentities: [agent.identity],
+        },
+      );
+      const image: TalqingSentImage = {
+        streamId: id,
+        name: file.name,
+        previewUrl: URL.createObjectURL(blob),
+        status: "sending",
+      };
+      setSent((images) => [...images, image]);
+      timers.current.set(
+        id,
+        setTimeout(
+          () =>
+            settle(id, {
+              status: "unknown",
+              // Four states, not two: the one people forget is the agent going
+              // away mid-upload, where no ack is ever coming.
+              error: "we did not hear back — the agent may not have received it",
+            }),
+          TALQING_IMAGE_ACK_TIMEOUT_MS,
+        ),
+      );
+      return image;
+    },
+    [room, settle],
+  );
+
+  const clear = useCallback(() => {
+    setSent((images) => {
+      images.forEach((image) => URL.revokeObjectURL(image.previewUrl));
+      return [];
+    });
+  }, []);
+
+  return { send, sent, clear };
+}
+
+export function useTalqingAgent() {
+  const agent = useAgent();
+  return {
+    state: agent.state,
+    isFinished: agent.isFinished,
+  };
+}
+
+export function useTalqingMessages() {
+  const messages = useSessionMessages();
+  return {
+    messages: messages.messages,
+    send: messages.send,
+    isSending: messages.isSending,
+  };
+}
+
+/* Screen share, with the four settings that decide whether the agent can read
+ * what is on it.
+ *
+ * Only the first is obvious, and every one of them is upstream of anything we
+ * could do on our side:
+ *
+ *  - `contentHint: "text"` tells the encoder to preserve spatial detail. The
+ *    default for a screen share on an SVC codec is `motion`, which trades
+ *    exactly that away — the wrong trade for reading code, and no JPEG quality
+ *    setting downstream recovers it.
+ *  - `videoCodec: "h264"` is what keeps that hint. `livekit-client` forces
+ *    `contentHint = "motion"` on screen shares published as vp9 or av1,
+ *    together with `scalabilityMode: "L1T3"`. **Publishing on an SVC codec will
+ *    cost you legibility**, and it is the first thing to check if the agent
+ *    starts misreading identifiers.
+ *  - `screenShareEncoding.maxFramerate: 1` is the frame-rate cap that actually
+ *    binds. `resolution.frameRate` reaches `getDisplayMedia` as a bare number,
+ *    which WebRTC reads as `ideal` — a wish, not a limit. The default publish
+ *    ceiling is 15 fps, and every delivered frame is copied into the agent's
+ *    process before it can be dropped, so this is the single biggest lever on
+ *    what a call costs the worker. One frame a second is ample: the agent is
+ *    handed the newest frame at the end of each turn, and turns are seconds
+ *    apart.
+ *  - `resolution` is set explicitly rather than inherited. The default is 1080p
+ *    by `ideal`, which is a strong hint and not a guarantee.
+ *
+ * The picker cannot be opened for the person — `getDisplayMedia` requires a
+ * transient user activation — so `start()` must be called from a click. The
+ * agent asking out loud is what prompts it; there is no version of this where
+ * the agent turns sharing on.
+ */
+export const TALQING_SCREEN_SHARE_CAPTURE: ScreenShareCaptureOptions = {
+  resolution: { width: 1920, height: 1080, frameRate: 1 },
+  contentHint: "text",
+};
+
+export const TALQING_SCREEN_SHARE_PUBLISH: TrackPublishOptions = {
+  videoCodec: "h264",
+  screenShareEncoding: { maxBitrate: 1_000_000, maxFramerate: 1 },
+};
+
+export function useTalqingScreenShare() {
+  const { toggle, enabled, pending } = useTrackToggle({
+    source: Track.Source.ScreenShare,
+    captureOptions: TALQING_SCREEN_SHARE_CAPTURE,
+    publishOptions: TALQING_SCREEN_SHARE_PUBLISH,
+  });
+  return useMemo(
+    () => ({
+      /** Whether a screen is being shared right now. */
+      enabled,
+      /** True while the browser picker is open or the track is publishing. */
+      pending,
+      /** Opens the browser's share picker. Must be called from a user gesture. */
+      start: () => toggle(true),
+      /** Stops sharing. The agent is told it can no longer see on the next turn. */
+      stop: () => toggle(false),
+    }),
+    [enabled, pending, toggle],
+  );
+}
+
+export function useTalqingAvatarTrack(): TalqingVideoTrackRef | undefined {
+  const { agent } = useVoiceAssistant();
+  const remoteParticipants = useRemoteParticipants();
+  const worker = remoteParticipants.find(
+    (participant: RemoteParticipant) =>
+      participant.kind === ParticipantKind.AGENT &&
+      participant.attributes["lk.publish_on_behalf"] === agent?.identity,
+  );
+
+  const agentTracks = useParticipantTracks(
+    [Track.Source.Camera, Track.Source.ScreenShare],
+    agent?.identity,
+  );
+  const workerTracks = useParticipantTracks(
+    [Track.Source.Camera, Track.Source.ScreenShare],
+    worker?.identity,
+  );
+
+  return (
+    workerTracks.find(
+      (track: TrackReference) => track.source === Track.Source.Camera,
+    ) ??
+    workerTracks.find(
+      (track: TrackReference) => track.source === Track.Source.ScreenShare,
+    ) ??
+    agentTracks.find(
+      (track: TrackReference) => track.source === Track.Source.Camera,
+    ) ??
+    agentTracks.find(
+      (track: TrackReference) => track.source === Track.Source.ScreenShare,
+    )
+  );
+}
+
+export function TalqingVideoTrack({
+  track,
+  className,
+}: {
+  track: TalqingVideoTrackRef;
+  className?: string;
+}) {
+  return (
+    <VideoTrack
+      trackRef={track}
+      width={track.publication.dimensions?.width ?? 0}
+      height={track.publication.dimensions?.height ?? 0}
+      className={className}
+    />
+  );
+}
+
+export function useTalqingRpcHandlers(
+  handlers: TalqingRpcHandlers,
+  enabled = true,
+) {
+  const { room, isConnected } = useTalqingConnection();
+
+  useEffect(() => {
+    if (!enabled || !room || !isConnected) return;
+
+    room.registerRpcMethod(
+      "talqing.frontend_rpc",
+      async (data: RpcInvocationData) => {
+        try {
+          const envelope = JSON.parse(
+            data.payload || "{}",
+          ) as TalqingFrontendRpcEnvelope;
+          const isWildcard = handlers[envelope.method] === undefined;
+          const handler = handlers[envelope.method] ?? handlers["*"];
+          if (!handler) {
+            throw new TalqingRpcError(
+              1404,
+              `No frontend RPC handler registered for ${envelope.method}`,
+            );
+          }
+          return await handler({
+            payload: isWildcard
+              ? JSON.stringify(envelope)
+              : envelope.payload === undefined
+                ? "{}"
+                : JSON.stringify(envelope.payload),
+            responseTimeout: data.responseTimeout,
+            callerIdentity: data.callerIdentity,
+          });
+        } catch (error) {
+          if (error instanceof TalqingRpcError) {
+            throw new RpcError(error.code, error.message, error.data);
+          }
+          throw new RpcError(
+            1500,
+            error instanceof Error ? error.message : "RPC handler failed",
+          );
+        }
+      },
+    );
+
+    return () => {
+      room.unregisterRpcMethod("talqing.frontend_rpc");
+    };
+  }, [enabled, handlers, isConnected, room]);
+}
+
+export function useTalqingFrontendRpcs(enabled = true) {
+  const [actions, setActions] = useState<
+    Record<string, { payload?: JsonValue; at: number }>
+  >({});
+
+  const handlers = useMemo<TalqingRpcHandlers>(
+    () => ({
+      "*": async (data: TalqingRpcInvocation) => {
+        const payload = JSON.parse(data.payload || "{}") as JsonValue;
+        const method = String((payload as { method?: string }).method || "");
+        if (method) {
+          setActions(
+            (current: Record<string, { payload?: JsonValue; at: number }>) => ({
+              ...current,
+              [method]: {
+                payload: (payload as { payload?: JsonValue }).payload,
+                at: Date.now(),
+              },
+            }),
+          );
+        }
+        return "ok";
+      },
+    }),
+    [],
+  );
+
+  useTalqingRpcHandlers(handlers, enabled);
+
+  return {
+    actions,
+    clear: () => setActions({}),
+  };
+}
+
+export function setTalqingBrowserLogLevel(
+  level: "debug" | "info" | "warn" | "error" | "silent",
+) {
+  const levels = {
+    debug: LogLevel.debug,
+    info: LogLevel.info,
+    warn: LogLevel.warn,
+    error: LogLevel.error,
+    silent: LogLevel.silent,
+  };
+  setLogLevel(levels[level]);
+}