@talqing/react 0.4.0

package/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@talqing/react",
+  "version": "0.4.0",
+  "description": "React bindings for Talqing web calls: the session, images, screen share, userdata and frontend RPC.",
+  "license": "MIT",
+  "homepage": "https://talqing.com",
+  "bugs": {
+    "email": "hello@talqing.com"
+  },
+  "keywords": [
+    "talqing",
+    "ai",
+    "agents",
+    "voice",
+    "video",
+    "livekit",
+    "react"
+  ],
+  "type": "module",
+  "sideEffects": false,
+  "main": "./dist/cjs/index.js",
+  "module": "./dist/esm/index.js",
+  "types": "./dist/cjs/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/esm/index.d.ts",
+        "default": "./dist/esm/index.js"
+      },
+      "require": {
+        "types": "./dist/cjs/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json && tsc -p tsconfig.cjs.json && node ../../scripts/dist-markers.mjs",
+    "dev": "tsc -w -p tsconfig.json",
+    "typecheck": "tsc --noEmit -p tsconfig.typecheck.json",
+    "prepublishOnly": "npm run build"
+  },
+  "dependencies": {},
+  "peerDependencies": {
+    "@livekit/components-react": "^2.9.21",
+    "@talqing/client": "^0.4.0",
+    "@talqing/sdk": "^0.4.0",
+    "livekit-client": "^2.19.1",
+    "react": "^18.3.1 || ^19.0.0",
+    "react-dom": "^18.3.1 || ^19.0.0"
+  },
+  "devDependencies": {
+    "@livekit/components-react": "^2.9.21",
+    "@talqing/client": "^0.4.0",
+    "@talqing/sdk": "^0.4.0",
+    "@types/react": "^19.0.0",
+    "livekit-client": "^2.19.1",
+    "react": "^19.0.0",
+    "react-dom": "^19.0.0",
+    "typescript": "~5.9.3"
+  }
+}

package/src/images.ts

@@ -0,0 +1,97 @@
+"use client";
+
+import { useCallback, useEffect, useRef, useState } from "react";
+import type { Room } from "livekit-client";
+import {
+  onTalqingImageResult,
+  sendTalqingImage,
+  TALQING_IMAGE_ACK_TIMEOUT_MS,
+  type TalqingSentImage,
+} from "@talqing/client";
+
+/** Send images to the agent on this call, and follow what became of each.
+ *
+ *  `send` resolves once the bytes are on their way; the outcome arrives on the
+ *  returned list, which is what a thumbnail should render.
+ *
+ *  Takes the `Room` rather than reading it from the session context: a byte
+ *  stream and an RPC handler are room-level things, and passing the room keeps
+ *  this usable from a surface that manages its own connection —
+ *  `useTalqingConnection().room` inside a provider, or the Room you built. */
+export function useTalqingImages(room: Room | null | undefined) {
+  const [sent, setSent] = useState<TalqingSentImage[]>([]);
+  const timers = useRef(new Map<string, ReturnType<typeof setTimeout>>());
+
+  const settle = useCallback((streamId: string, patch: Partial<TalqingSentImage>) => {
+    const timer = timers.current.get(streamId);
+    if (timer) {
+      clearTimeout(timer);
+      timers.current.delete(streamId);
+    }
+    setSent((images) =>
+      images.map((image) => (image.streamId === streamId ? { ...image, ...patch } : image)),
+    );
+  }, []);
+
+  useEffect(() => {
+    if (!room) return;
+    return onTalqingImageResult(room, ({ streamId, ok, error }) =>
+      settle(streamId, { status: ok ? "sent" : "failed", error }),
+    );
+  }, [room, settle]);
+
+  const pending = timers.current;
+  useEffect(
+    () => () => {
+      pending.forEach(clearTimeout);
+      pending.clear();
+    },
+    [pending],
+  );
+
+  /* Each thumbnail is an object URL over the downscaled blob, and each pins that
+     blob in memory until it is revoked. This hook is mounted per call, so
+     without this every call's images would be held for the life of the tab. */
+  const previews = useRef<string[]>([]);
+  useEffect(
+    () => () => {
+      previews.current.forEach(URL.revokeObjectURL);
+      previews.current = [];
+    },
+    [],
+  );
+
+  const send = useCallback(
+    async (file: File): Promise<TalqingSentImage> => {
+      if (!room) throw new Error("the call is not connected");
+      const image = await sendTalqingImage(room, file);
+      previews.current.push(image.previewUrl);
+      setSent((images) => [...images, image]);
+      timers.current.set(
+        image.streamId,
+        setTimeout(
+          () =>
+            settle(image.streamId, {
+              status: "unknown",
+              // Four states, not two: the one people forget is the agent going
+              // away mid-upload, where no ack is ever coming.
+              error: "we did not hear back — the agent may not have received it",
+            }),
+          TALQING_IMAGE_ACK_TIMEOUT_MS,
+        ),
+      );
+      return image;
+    },
+    [room, settle],
+  );
+
+  const clear = useCallback(() => {
+    setSent((images) => {
+      images.forEach((image) => URL.revokeObjectURL(image.previewUrl));
+      previews.current = [];
+      return [];
+    });
+  }, []);
+
+  return { send, sent, clear };
+}

package/src/index.ts

@@ -0,0 +1,58 @@
+/* React bindings for the Talqing call protocol.
+ *
+ * Thin by design. Everything here is either the call lifecycle (`useTalqingSession`
+ * and the auth union it takes) or a hook over something `@talqing/client` does to
+ * a `Room`. Nothing here wraps LiveKit to rename it: the layer only ever ADDS, so
+ * anything LiveKit can do you can do — take `useTalqingConnection().room` and use
+ * `livekit-client` directly. We never remove a capability to make our own surface
+ * look tidier, which is also why `livekit-client` and `@livekit/components-react`
+ * are peer dependencies: the `Room` we hand you is the same class your own
+ * imports and `instanceof` resolve to.
+ */
+
+export {
+  TalqingSessionProvider,
+  useTalqingConnection,
+  useTalqingSession,
+} from "./session.js";
+export type {
+  TalqingCallOptions,
+  TalqingSession,
+  TalqingSessionOptions,
+} from "./session.js";
+
+export { useTalqingUserdata } from "./userdata.js";
+export { useTalqingImages } from "./images.js";
+export { useTalqingFrontendRpcs, useTalqingRpcHandlers } from "./rpc.js";
+export { useTalqingAvatarTrack, useTalqingScreenShare } from "./media.js";
+
+/* All of `@talqing/client`, so one install and one import cover a web call.
+ * Everything there works against any `Room`, with or without React — the token
+ * source, the userdata and image and frontend-RPC calls, and the measured
+ * capture constants. Re-exported wholesale rather than as a curated list
+ * because the whole package is meant to be reachable from here, and a list
+ * would quietly fall behind it. */
+export * from "@talqing/client";
+
+/* LiveKit's own, re-exported UNDER THEIR OWN NAMES so a tenant installs one
+ * package and imports one name.
+ *
+ * Re-exporting is not wrapping: there is no surface of ours here to drift, and
+ * nothing is subtracted — `useAgent` is the ten-state discriminated union
+ * LiveKit ships, with every derived boolean, every action and
+ * `failureReasons` intact. Keeping LiveKit's names is the point: a
+ * `useTalqing`-prefixed alias would claim these as ours and hide where to read
+ * their documentation. Import them from `@livekit/components-react` instead if
+ * you prefer — it is the same module. */
+export {
+  useAgent,
+  useSessionMessages,
+  VideoTrack,
+} from "@livekit/components-react";
+export type {
+  AgentState,
+  ReceivedMessage,
+  TrackReference,
+  UseAgentReturn,
+  UseSessionReturn,
+} from "@livekit/components-react";

package/src/media.ts

@@ -0,0 +1,77 @@
+"use client";
+
+import { useMemo } from "react";
+import { ParticipantKind, Track, type RemoteParticipant } from "livekit-client";
+import {
+  useParticipantTracks,
+  useRemoteParticipants,
+  useTrackToggle,
+  useVoiceAssistant,
+  type TrackReference,
+} from "@livekit/components-react";
+import {
+  TALQING_SCREEN_SHARE_CAPTURE,
+  TALQING_SCREEN_SHARE_PUBLISH,
+} from "@talqing/client";
+
+/** Share the caller's screen with the agent, on the settings that keep it
+ *  legible — see `TALQING_SCREEN_SHARE_CAPTURE` for why each one is what it is. */
+export function useTalqingScreenShare() {
+  const { toggle, enabled, pending } = useTrackToggle({
+    source: Track.Source.ScreenShare,
+    captureOptions: TALQING_SCREEN_SHARE_CAPTURE,
+    publishOptions: TALQING_SCREEN_SHARE_PUBLISH,
+  });
+  return useMemo(
+    () => ({
+      /** Whether a screen is being shared right now. */
+      enabled,
+      /** True while the browser picker is open or the track is publishing. */
+      pending,
+      /** Opens the browser's share picker. Must be called from a user gesture. */
+      start: () => toggle(true),
+      /** Stops sharing. The agent is told it can no longer see on the next turn. */
+      stop: () => toggle(false),
+    }),
+    [enabled, pending, toggle],
+  );
+}
+
+/** The video track to render for a video agent.
+ *
+ *  Prefers the avatar worker's camera — the worker publishes on the agent's
+ *  behalf, which is how it is identified — and falls back through the worker's
+ *  screen share and then the agent's own tracks. */
+export function useTalqingAvatarTrack(): TrackReference | undefined {
+  const { agent } = useVoiceAssistant();
+  const remoteParticipants = useRemoteParticipants();
+  const worker = remoteParticipants.find(
+    (participant: RemoteParticipant) =>
+      participant.kind === ParticipantKind.AGENT &&
+      participant.attributes["lk.publish_on_behalf"] === agent?.identity,
+  );
+
+  const agentTracks = useParticipantTracks(
+    [Track.Source.Camera, Track.Source.ScreenShare],
+    agent?.identity,
+  );
+  const workerTracks = useParticipantTracks(
+    [Track.Source.Camera, Track.Source.ScreenShare],
+    worker?.identity,
+  );
+
+  return (
+    workerTracks.find(
+      (track: TrackReference) => track.source === Track.Source.Camera,
+    ) ??
+    workerTracks.find(
+      (track: TrackReference) => track.source === Track.Source.ScreenShare,
+    ) ??
+    agentTracks.find(
+      (track: TrackReference) => track.source === Track.Source.Camera,
+    ) ??
+    agentTracks.find(
+      (track: TrackReference) => track.source === Track.Source.ScreenShare,
+    )
+  );
+}

package/src/rpc.ts

@@ -0,0 +1,61 @@
+"use client";
+
+import { useEffect, useMemo, useState } from "react";
+import {
+  registerTalqingRpcHandlers,
+  type JsonValue,
+  type TalqingRpcHandlers,
+  type TalqingRpcInvocation,
+} from "@talqing/client";
+import { useTalqingConnection } from "./session.js";
+
+/** Handle the agent's `frontend_rpc` calls into this page.
+ *
+ *  Registration waits for the room to connect and is undone on unmount. Pass
+ *  `enabled: false` to hold off. */
+export function useTalqingRpcHandlers(handlers: TalqingRpcHandlers, enabled = true) {
+  const { room, isConnected } = useTalqingConnection();
+
+  useEffect(() => {
+    if (!enabled || !room || !isConnected) return;
+    return registerTalqingRpcHandlers(room, handlers);
+  }, [enabled, handlers, isConnected, room]);
+}
+
+/** The declarative alternative: register nothing, render what arrived.
+ *
+ *  Keeps the latest payload per method name and always answers the agent "ok". */
+export function useTalqingFrontendRpcs(enabled = true) {
+  const [actions, setActions] = useState<
+    Record<string, { payload?: JsonValue; at: number }>
+  >({});
+
+  const handlers = useMemo<TalqingRpcHandlers>(
+    () => ({
+      "*": async (data: TalqingRpcInvocation) => {
+        const payload = JSON.parse(data.payload || "{}") as JsonValue;
+        const method = String((payload as { method?: string }).method || "");
+        if (method) {
+          setActions(
+            (current: Record<string, { payload?: JsonValue; at: number }>) => ({
+              ...current,
+              [method]: {
+                payload: (payload as { payload?: JsonValue }).payload,
+                at: Date.now(),
+              },
+            }),
+          );
+        }
+        return "ok";
+      },
+    }),
+    [],
+  );
+
+  useTalqingRpcHandlers(handlers, enabled);
+
+  return {
+    actions,
+    clear: () => setActions({}),
+  };
+}

package/src/session.tsx

@@ -0,0 +1,148 @@
+"use client";
+
+import React, { useCallback, useMemo, useState } from "react";
+import { Room, type RoomOptions } from "livekit-client";
+import {
+  RoomAudioRenderer,
+  SessionProvider,
+  useSession,
+  useSessionContext,
+  type UseSessionReturn,
+} from "@livekit/components-react";
+import {
+  createTalqingTokenSource,
+  TALQING_AUDIO_CAPTURE_DEFAULTS,
+  type TalqingCallCredentials,
+} from "@talqing/client";
+import type { TalqingClient, TokenRequest } from "@talqing/sdk";
+
+/** Everything about one web call except which agent runs it. Wire-shaped, like
+ *  every other request type in this SDK. */
+export type TalqingCallOptions = Omit<TokenRequest, "agent_id">;
+
+/* Three ways to authorize one call, and exactly one of them per call.
+ *
+ * The branches are mutually exclusive by construction rather than by a runtime
+ * precedence rule: every branch declares every other branch's fields as
+ * `?: never`, so passing two is a type error at the call site.
+ *
+ * The first two are what a tenant ships. `POST /v1/calls/token` needs the EDITOR
+ * role, and a Talqing credential that can mint a call token can also create and
+ * delete agents, read every transcript and spend the workspace's credits — so it
+ * belongs on your server and nowhere near a browser. Your server mints; the
+ * browser receives two strings and joins.
+ *
+ * That is not only about the credential. `contact_key` decides WHOSE history the
+ * call resumes, and `userdata` is merged onto that person's record. A browser
+ * that chooses its own would be choosing whose past conversation to continue.
+ * On these two branches it never can: those fields never leave your backend.
+ */
+type TalqingCallAuth =
+  /** Your server called `POST /v1/calls/token` and forwarded these two fields. */
+  | {
+      token: TalqingCallCredentials;
+      fetchToken?: never;
+      client?: never;
+      agentId?: never;
+      callOptions?: never;
+    }
+  /** Your endpoint mints one. Called once, when the call starts. */
+  | {
+      fetchToken: () => Promise<TalqingCallCredentials>;
+      token?: never;
+      client?: never;
+      agentId?: never;
+      callOptions?: never;
+    }
+  /** Mint from the browser with a `TalqingClient` that already holds a
+   *  credential. **Trusted first-party surfaces only** — an app that is
+   *  same-origin with the API and signed in as a real editor, which is what our
+   *  own dashboard is. If you are embedding a call widget for your customers,
+   *  this is not your branch. */
+  | {
+      client: TalqingClient;
+      agentId: string | null;
+      /** `contact_key`, `userdata`, `vars` and the per-call agent selection.
+       *  Only on this branch: on the other two the server passes them when it
+       *  mints, which is what keeps a browser from choosing whose history to
+       *  resume. */
+      callOptions?: TalqingCallOptions;
+      token?: never;
+      fetchToken?: never;
+    };
+
+export type TalqingSessionOptions = TalqingCallAuth & { roomOptions?: RoomOptions };
+
+/** LiveKit's session, plus what resolving this call's plan complained about. */
+export type TalqingSession = UseSessionReturn & {
+  /** An unreachable team member, a handoff into an agent that records when this
+   *  one does not — resolved against state at CALL time, so it is not the list
+   *  publishing returned. Empty until the token is minted.
+   *
+   *  Populated on the `client` branch, which is the one where this browser is
+   *  what minted. On the two server-minted branches your own server received
+   *  them in the token response and this stays empty. */
+  warnings: string[];
+};
+
+/** One session, one call — `start()` it when this mounts and `end()` it on the
+ *  way out, the lifecycle LiveKit's own session docs describe. A second call is
+ *  a second mount, so give the component holding this a `key` that changes per
+ *  call. Starting a finished session again would rejoin the room the last call
+ *  used, which the API refuses. */
+export function useTalqingSession(options: TalqingSessionOptions): TalqingSession {
+  const { token, fetchToken, client, agentId, callOptions, roomOptions } = options;
+  const [warnings, setWarnings] = useState<string[]>([]);
+
+  // Reads every branch's fields so the dependency list is the same length on
+  // each; all but one are `undefined` on any given call.
+  const mint = useCallback(async (): Promise<TalqingCallCredentials> => {
+    if (token) return token;
+    if (fetchToken) return await fetchToken();
+    if (!client) throw new Error("useTalqingSession needs one of token, fetchToken or client");
+    const minted = await client.calls.token({
+      tokenRequest: { ...callOptions, agent_id: agentId ?? null },
+    });
+    setWarnings(minted.warnings);
+    return { server_url: minted.server_url, participant_token: minted.participant_token };
+  }, [token, fetchToken, client, agentId, callOptions]);
+
+  const tokenSource = useMemo(() => createTalqingTokenSource(mint), [mint]);
+  const room = useMemo(
+    () =>
+      new Room({ audioCaptureDefaults: TALQING_AUDIO_CAPTURE_DEFAULTS, ...roomOptions }),
+    [roomOptions],
+  );
+  const session = useSession(tokenSource, { room });
+  return useMemo(() => ({ ...session, warnings }), [session, warnings]);
+}
+
+export function TalqingSessionProvider({
+  session,
+  children,
+  audio = true,
+}: {
+  session: TalqingSession;
+  children: React.ReactNode;
+  audio?: boolean;
+}) {
+  return (
+    <SessionProvider session={session}>
+      {children}
+      {audio ? <RoomAudioRenderer /> : null}
+    </SessionProvider>
+  );
+}
+
+/** The call itself, read from context — and `room`, which is the supported way
+ *  down to raw `livekit-client` for anything this layer does not cover. */
+export function useTalqingConnection() {
+  const ctx = useSessionContext();
+  return {
+    isConnected: ctx.isConnected,
+    connectionState: ctx.connectionState,
+    room: ctx.room,
+    start: ctx.start,
+    end: ctx.end,
+  };
+}

package/src/userdata.ts

@@ -0,0 +1,27 @@
+"use client";
+
+import { useMemo } from "react";
+import {
+  getTalqingUserdata,
+  setTalqingUserdata,
+  type JsonObject,
+} from "@talqing/client";
+import { useTalqingConnection } from "./session.js";
+
+/** Read and write the live session's userdata from inside a provider. */
+export function useTalqingUserdata() {
+  const { room } = useTalqingConnection();
+  return useMemo(
+    () => ({
+      get: (keys?: string[], responseTimeout?: number) => {
+        if (!room) throw new Error("room is not connected");
+        return getTalqingUserdata(room, keys, responseTimeout);
+      },
+      set: (patch: JsonObject, responseTimeout?: number) => {
+        if (!room) throw new Error("room is not connected");
+        return setTalqingUserdata(room, patch, responseTimeout);
+      },
+    }),
+    [room],
+  );
+}