@sable-ai/sdk-core 0.1.0

package/src/session/index.ts

@@ -0,0 +1,375 @@
+/**
+ * Session lifecycle.
+ *
+ * `Session` is the glue layer: it fetches connection details, dynamically
+ * imports `livekit-client`, connects, publishes the mic, registers the
+ * runtime + browser-bridge RPCs, and — if vision is enabled — starts the
+ * frame source + video publisher and mounts the debug panel. `start()`
+ * returns once the room is live and mic is publishing; events are emitted
+ * via the `SableEventEmitter`.
+ *
+ * Only one session is allowed at a time. `start()` throws if a session is
+ * already active; callers must `stop()` first. `stop()` is idempotent.
+ *
+ * `livekit-client` is imported dynamically (not statically) so the IIFE
+ * entry bundle stays small — the heavy client only loads when a customer
+ * actually calls `start()`.
+ */
+
+import {
+  fetchConnectionDetails,
+  DEFAULT_API_URL,
+} from "../connection";
+import { SableEventEmitter } from "../events";
+import { installRuntime } from "../runtime";
+import { registerBrowserHandlers } from "../browser-bridge";
+import {
+  startVision,
+  type LiveKitPublishLib,
+  type VisionHandle,
+} from "../vision";
+import type {
+  SableAPI,
+  SableEventHandler,
+  SableEvents,
+  StartOptions,
+} from "../types";
+import { VERSION } from "../version";
+import { mountDebugPanel, shouldShowDebugPanel } from "./debug-panel";
+
+// ── livekit-client structural types ───────────────────────────────────────
+//
+// The client is dynamically imported so we can't use its types at the top
+// level. These mirror only the subset we call.
+
+interface LiveKitRoom {
+  connect(url: string, token: string): Promise<unknown>;
+  disconnect(): Promise<void>;
+  on(event: string, handler: (...args: unknown[]) => void): unknown;
+  registerRpcMethod(
+    method: string,
+    handler: (data: { payload: string }) => Promise<string>,
+  ): void;
+  localParticipant: {
+    identity?: string;
+    setMicrophoneEnabled(enabled: boolean): Promise<unknown>;
+    performRpc(opts: {
+      destinationIdentity: string;
+      method: string;
+      payload: string;
+    }): Promise<string>;
+    publishTrack(
+      track: unknown,
+      options?: { source?: unknown; name?: string },
+    ): Promise<{ trackSid?: string }>;
+    unpublishTrack(track: unknown, stopOnUnpublish?: boolean): Promise<unknown>;
+  };
+  remoteParticipants?: Map<
+    string,
+    { identity?: string; trackPublications?: Map<string, unknown> }
+  >;
+}
+
+// ── Agent handshake ───────────────────────────────────────────────────────
+//
+// Agents emit `agentReady` after joining and wait for a `uiReady` reply
+// before generating their greeting. If we don't reply within ~10s the agent
+// gives up and never publishes audio.
+// Reference: parley/src/features/agent/hooks/useAgentConnection.ts.
+
+const UI_READY_RETRY_ATTEMPTS = 5;
+const UI_READY_RETRY_DELAY_MS = 500;
+
+function findAgentIdentity(room: LiveKitRoom): string | null {
+  const remotes = room.remoteParticipants
+    ? Array.from(room.remoteParticipants.values())
+    : [];
+  const agent = remotes.find(
+    (p) => typeof p.identity === "string" && p.identity.startsWith("agent"),
+  );
+  return agent?.identity ?? remotes[0]?.identity ?? null;
+}
+
+async function sendUiReady(room: LiveKitRoom): Promise<void> {
+  for (let attempt = 1; attempt <= UI_READY_RETRY_ATTEMPTS; attempt++) {
+    const identity = findAgentIdentity(room);
+    if (!identity) {
+      console.warn("[Sable] sendUiReady: no agent participant yet", { attempt });
+      await new Promise((r) => setTimeout(r, UI_READY_RETRY_DELAY_MS));
+      continue;
+    }
+    try {
+      await room.localParticipant.performRpc({
+        destinationIdentity: identity,
+        method: "uiReady",
+        payload: JSON.stringify({ timestamp: Date.now() }),
+      });
+      console.log("[Sable] uiReady sent", { identity, attempt });
+      return;
+    } catch (err) {
+      console.warn("[Sable] uiReady RPC failed", { attempt, err });
+      await new Promise((r) => setTimeout(r, UI_READY_RETRY_DELAY_MS));
+    }
+  }
+  console.error("[Sable] uiReady: exhausted retries — agent will not greet");
+}
+
+// ── Session class ─────────────────────────────────────────────────────────
+
+/**
+ * One active session at a time. The class is internal — customers interact
+ * with the `SableAPI` singleton installed on `window.Sable` (see `global.ts`).
+ * Keeping a class here (rather than a bag of module-level vars) makes the
+ * state ownership explicit and the teardown path easier to reason about.
+ */
+export class Session implements SableAPI {
+  readonly version = VERSION;
+  private readonly emitter = new SableEventEmitter();
+  private activeRoom: LiveKitRoom | null = null;
+  private visionHandle: VisionHandle | null = null;
+  private unmountDebugPanel: (() => void) | null = null;
+
+  on<E extends keyof SableEvents>(
+    event: E,
+    handler: SableEventHandler<E>,
+  ): () => void {
+    return this.emitter.on(event, handler);
+  }
+
+  async start(opts: StartOptions): Promise<void> {
+    if (this.activeRoom) {
+      throw new Error("Sable already started; call stop() first");
+    }
+
+    // Public key resolution. `publicKey` wins when both are passed, but
+    // `agentPublicId` remains supported during beta so customers upgrading
+    // from an earlier build don't have to rename the field the same day
+    // they update the package.
+    const publicKey = opts.publicKey ?? opts.agentPublicId;
+    if (!publicKey) {
+      throw new Error("Sable.start: `publicKey` is required");
+    }
+
+    const apiUrl = opts.apiUrl ?? DEFAULT_API_URL;
+    console.log("[Sable] fetching connection details", { apiUrl });
+    const details = await fetchConnectionDetails({ apiUrl, publicKey });
+    console.log("[Sable] connection details received", {
+      roomName: details.roomName,
+      participantName: details.participantName,
+    });
+
+    // Dynamic import keeps the IIFE entry small; livekit-client is ~200KB
+    // minified and only needed once a session actually starts.
+    const livekit = await import("livekit-client");
+    const { Room, RoomEvent, LocalVideoTrack, Track } = livekit;
+    const room = new Room() as unknown as LiveKitRoom;
+
+    const publishLib: LiveKitPublishLib = {
+      LocalVideoTrack: LocalVideoTrack as unknown as LiveKitPublishLib["LocalVideoTrack"],
+      Track: Track as unknown as LiveKitPublishLib["Track"],
+    };
+
+    // Handshake handler MUST be registered before room.connect() so it's
+    // ready when the first RPC arrives.
+    room.registerRpcMethod("agentReady", async () => {
+      console.log("[Sable] RPC agentReady received");
+      void sendUiReady(room);
+      return JSON.stringify({ success: true });
+    });
+
+    // Browser bridge: 6 browser.* handlers. Safe to register even for
+    // voice-only sessions — the agent just won't call them.
+    registerBrowserHandlers(room);
+
+    // Runtime: default methods (clipboard, switchView) + customer overrides
+    // and extensions passed via `opts.runtime`.
+    installRuntime(room, opts.runtime);
+
+    this.wireRoomEvents(room, RoomEvent);
+
+    await room.connect(details.serverUrl, details.participantToken);
+    await room.localParticipant.setMicrophoneEnabled(true);
+
+    this.activeRoom = room;
+
+    // Vision is off by default. Opt in with `vision: { enabled: true }`.
+    if (opts.vision?.enabled) {
+      try {
+        this.visionHandle = await startVision({
+          room: room as unknown as Parameters<typeof startVision>[0]["room"],
+          lib: publishLib,
+          options: opts.vision,
+        });
+        if (shouldShowDebugPanel(opts.debug)) {
+          this.unmountDebugPanel = mountDebugPanel(this.visionHandle.canvas);
+        }
+      } catch (e) {
+        console.warn("[Sable] failed to start vision", e);
+      }
+    }
+
+    console.log("[Sable] session live", {
+      roomName: details.roomName,
+      participantName: details.participantName,
+    });
+    this.emitter.emit("session:started", {
+      roomName: details.roomName,
+      participantName: details.participantName,
+    });
+
+    // Watchdog: warn loudly if no remote audio track shows up within 10s.
+    setTimeout(() => {
+      if (this.activeRoom !== room) return;
+      const r = room as unknown as {
+        remoteParticipants?: Map<
+          string,
+          {
+            identity?: string;
+            trackPublications?: Map<
+              string,
+              { kind?: string; isSubscribed?: boolean }
+            >;
+          }
+        >;
+      };
+      const remotes = r.remoteParticipants
+        ? Array.from(r.remoteParticipants.values())
+        : [];
+      const summary = remotes.map((p) => ({
+        identity: p.identity,
+        tracks: p.trackPublications
+          ? Array.from(p.trackPublications.values()).map((t) => ({
+              kind: t.kind,
+              subscribed: t.isSubscribed,
+            }))
+          : [],
+      }));
+      const anyAudio = summary.some((p) =>
+        p.tracks.some((t) => t.kind === "audio"),
+      );
+      if (!anyAudio) {
+        console.warn(
+          "[Sable] no remote audio track after 10s — agent worker probably failed to publish. Remote participants:",
+          summary,
+        );
+      }
+    }, 10000);
+  }
+
+  async stop(): Promise<void> {
+    const room = this.activeRoom;
+    if (!room) return;
+    this.activeRoom = null;
+
+    if (this.unmountDebugPanel) {
+      try {
+        this.unmountDebugPanel();
+      } catch (e) {
+        console.warn("[Sable] debug panel unmount failed", e);
+      }
+      this.unmountDebugPanel = null;
+    }
+
+    if (this.visionHandle) {
+      try {
+        await this.visionHandle.stop();
+      } catch (e) {
+        console.warn("[Sable] vision stop failed", e);
+      }
+      this.visionHandle = null;
+    }
+
+    try {
+      await room.localParticipant.setMicrophoneEnabled(false);
+    } catch (err) {
+      console.warn("[Sable] setMicrophoneEnabled(false) failed", err);
+    }
+    await room.disconnect();
+    console.log("[Sable] session ended");
+    this.emitter.emit("session:ended", {});
+  }
+
+  /**
+   * Subscribe to LiveKit room events and translate the interesting ones into
+   * `SableEvents`. Keeps the Session → customer event surface decoupled from
+   * the LiveKit event names so we can swap the transport later without
+   * breaking subscribers.
+   */
+  private wireRoomEvents(
+    room: LiveKitRoom,
+    RoomEvent: Record<string, string>,
+  ): void {
+    room.on(RoomEvent.ConnectionStateChanged, (state: unknown) => {
+      console.log("[Sable] ConnectionStateChanged", state);
+    });
+    room.on(RoomEvent.Disconnected, (reason: unknown) => {
+      console.log("[Sable] Disconnected", reason);
+      // Forward to stop() so cleanup runs exactly once regardless of who
+      // initiated the disconnect (customer call vs. server drop).
+      if (this.activeRoom === room) {
+        void this.stop().catch((e) =>
+          console.warn("[Sable] stop on disconnect failed", e),
+        );
+      }
+    });
+    room.on(RoomEvent.ParticipantConnected, (participant: unknown) => {
+      const p = participant as {
+        identity?: string;
+        sid?: string;
+        metadata?: string;
+      };
+      console.log("[Sable] ParticipantConnected", {
+        identity: p.identity,
+        sid: p.sid,
+      });
+    });
+    room.on(RoomEvent.ParticipantDisconnected, (participant: unknown) => {
+      const p = participant as { identity?: string };
+      console.warn("[Sable] ParticipantDisconnected", { identity: p.identity });
+    });
+    room.on(
+      RoomEvent.TrackSubscribed,
+      (track: unknown, _pub: unknown, participant: unknown) => {
+        const t = track as {
+          kind?: string;
+          attach?: () => HTMLMediaElement;
+        };
+        const p = participant as { identity?: string };
+        console.log("[Sable] TrackSubscribed", {
+          kind: t.kind,
+          participant: p.identity,
+        });
+        // Auto-attach remote audio so the agent's voice plays without the
+        // customer needing to wire up an <audio> element themselves.
+        if (t.kind === "audio" && typeof t.attach === "function") {
+          const el = t.attach();
+          el.setAttribute("data-sable", "1");
+          el.setAttribute("playsinline", "");
+          el.autoplay = true;
+          document.body.appendChild(el);
+          console.log("[Sable] attached remote audio element");
+        }
+      },
+    );
+    room.on(RoomEvent.TrackUnsubscribed, (track: unknown) => {
+      const t = track as { detach?: () => HTMLMediaElement[] };
+      if (typeof t.detach === "function") {
+        for (const el of t.detach()) {
+          el.remove();
+        }
+      }
+    });
+    room.on(RoomEvent.ActiveSpeakersChanged, (speakers: unknown) => {
+      const list = (speakers as Array<{ identity?: string }>) ?? [];
+      const agentTalking = list.some(
+        (s) => typeof s.identity === "string" && s.identity.startsWith("agent"),
+      );
+      const userTalking = list.some(
+        (s) =>
+          typeof s.identity === "string" && !s.identity.startsWith("agent"),
+      );
+      this.emitter.emit("agent:speaking", agentTalking);
+      this.emitter.emit("user:speaking", userTalking);
+    });
+  }
+}

package/src/types/index.ts

@@ -0,0 +1,176 @@
+/**
+ * Public type surface for @sable-ai/sdk-core.
+ *
+ * Everything a consumer can touch lives here. Internal types stay in their
+ * respective modules so we never accidentally ship them in the published
+ * `.d.ts` bundle.
+ */
+
+// ── Frame sources ──────────────────────────────────────────────────────────
+//
+// Discriminated union — `type` is the tag. Adding a new source (e.g. `video`,
+// `webgl`) means adding a new variant here and a new case in
+// `vision/frame-source.ts`. The public API stays stable.
+
+export interface WireframeFrameSource {
+  type: "wireframe";
+  /** Capture rate in frames per second. Default: 2. */
+  rate?: number;
+  features?: {
+    /**
+     * Include rendered images in the wireframe (instead of placeholder boxes).
+     * Slightly higher CPU + bandwidth. Default: false.
+     */
+    includeImages?: boolean;
+  };
+}
+
+export interface FnFrameSource {
+  type: "fn";
+  /** Capture rate in frames per second. Default: 2. */
+  rate?: number;
+  /**
+   * Called at `rate` Hz. Return an `HTMLCanvasElement` or `ImageBitmap` that
+   * the SDK will publish to the agent as a video track. Useful for feeding
+   * custom sources like a 3D scene, a `<video>` element, or a WebGL surface
+   * that the DOM walker can't introspect.
+   */
+  captureFn: () => HTMLCanvasElement | ImageBitmap;
+}
+
+export type FrameSource = WireframeFrameSource | FnFrameSource;
+
+// ── Vision ─────────────────────────────────────────────────────────────────
+
+export interface VisionOptions {
+  /**
+   * Whether to publish a video track of the page to the agent. Default: false.
+   * Turn this on for agents that should be able to *see* the user's screen
+   * in addition to hearing them.
+   */
+  enabled?: boolean;
+  /**
+   * Where video frames come from. Defaults to the built-in wireframe renderer
+   * at 2 fps with images disabled.
+   */
+  frameSource?: FrameSource;
+}
+
+// ── Runtime (agent → page RPC surface) ─────────────────────────────────────
+//
+// The agent can call methods on the page over LiveKit RPC. `sdk-core` ships
+// default implementations for a known set of methods (clipboard copy, video
+// overlay, and no-op placeholders for host-UI-specific methods). Customers
+// override any of them by passing matching keys in `Sable.start({ runtime })`,
+// and can add new methods specific to their app that become callable by the
+// agent. One unified surface: no distinction between "SDK methods" and
+// "customer methods" from the agent's perspective.
+
+export type RuntimeMethod = (
+  payload: Record<string, unknown>,
+) => unknown | Promise<unknown>;
+
+/**
+ * Map of method name → handler. Used both for the user-provided overrides
+ * passed to `Sable.start({ runtime })` and for the SDK's internal defaults.
+ */
+export interface RuntimeMethods {
+  [method: string]: RuntimeMethod;
+}
+
+// ── Start options ──────────────────────────────────────────────────────────
+
+export interface StartOptions {
+  /**
+   * Publishable key for the agent (from platform.withsable.com → your agent
+   * → Web SDK → Public key). Safe to ship in client-side code — the security
+   * boundary is the allowed-domains list configured alongside the key.
+   *
+   * During beta, raw agent IDs (e.g. `agt_...`) are accepted here too.
+   */
+  publicKey?: string;
+
+  /**
+   * @deprecated Use `publicKey` instead. Accepted as an alias during beta and
+   * will be removed before 1.0. If both are set, `publicKey` wins.
+   */
+  agentPublicId?: string;
+
+  /**
+   * What the agent can see. Off by default — opt in for vision-enabled agents.
+   */
+  vision?: VisionOptions;
+
+  /**
+   * Overrides + extensions for methods the agent can RPC into the page.
+   * Unspecified methods fall back to the SDK's default implementations. New
+   * methods become callable by the agent as-is.
+   */
+  runtime?: RuntimeMethods;
+
+  /**
+   * Arbitrary metadata forwarded to the agent at session start. Surfaces
+   * verbatim in the agent's initial prompt.
+   */
+  context?: Record<string, unknown>;
+
+  /**
+   * Dev-only: mount a floating preview panel showing the exact wireframe
+   * canvas being published to the agent. Can also be enabled via
+   * `?sable-debug=1` or `localStorage["sable:debug"]="1"`.
+   */
+  debug?: boolean;
+
+  /**
+   * Override the sable-api base URL. Dev/test only. Defaults to the
+   * production gateway.
+   * @internal
+   */
+  apiUrl?: string;
+}
+
+// ── Events ─────────────────────────────────────────────────────────────────
+//
+// Fire-and-forget — the SDK does not care whether customers subscribe.
+
+export interface SableEvents {
+  /** Fired once the room is connected, mic is live, and handshake is done. */
+  "session:started": { roomName: string; participantName: string };
+  /** Fired once when the session ends for any reason. */
+  "session:ended": { reason?: string };
+  /** Fired whenever the agent starts or stops speaking. */
+  "agent:speaking": boolean;
+  /** Fired whenever the local user starts or stops speaking. */
+  "user:speaking": boolean;
+  /** Fired for any non-fatal error during the session. */
+  error: Error;
+}
+
+export type SableEventHandler<E extends keyof SableEvents> = (
+  payload: SableEvents[E],
+) => void;
+
+// ── Public API surface ─────────────────────────────────────────────────────
+
+export interface SableAPI {
+  /** SDK version string, matches the npm package version. */
+  version: string;
+  /** Start a voice (and optionally vision) session with the agent. */
+  start(opts: StartOptions): Promise<void>;
+  /** Tear down the active session. No-op if none. */
+  stop(): Promise<void>;
+  /**
+   * Subscribe to a session event. Returns an unsubscribe function. Fire-and-
+   * forget — the SDK does not care whether you subscribe.
+   */
+  on<E extends keyof SableEvents>(
+    event: E,
+    handler: SableEventHandler<E>,
+  ): () => void;
+}
+
+declare global {
+  interface Window {
+    Sable?: SableAPI;
+  }
+}

package/src/version.ts

@@ -0,0 +1,8 @@
+/**
+ * Single source of truth for the SDK version string.
+ *
+ * Kept in a standalone file so build tooling (GitHub Actions release workflow)
+ * can replace it at publish time without touching anything else.
+ */
+
+export const VERSION = "0.1.0";

package/src/vision/frame-source.ts

@@ -0,0 +1,111 @@
+/**
+ * FrameSource dispatcher.
+ *
+ * The public API lets customers choose what the agent sees via
+ * `vision: { frameSource: { type: "wireframe" | "fn", ... } }`. This module
+ * hides the strategy behind one entrypoint — `startFrameSource` — which
+ * returns a stop function. The target canvas is passed in so the caller
+ * (vision/index.ts) can hand the same canvas to `canvas.captureStream()`.
+ *
+ * Two built-in sources:
+ *
+ *   1. `{ type: "wireframe", rate, features: { includeImages } }`
+ *      — Runs the Wireframe library against `document.body` at `rate` Hz.
+ *        This is the default, tuned for low bandwidth + agent-readable
+ *        structure. `includeImages: true` fetches cover photos/avatars/
+ *        thumbnails via CORS and draws real pixels; otherwise the agent
+ *        gets labelled placeholder boxes.
+ *
+ *   2. `{ type: "fn", rate, captureFn }`
+ *      — The user supplies a function that returns a canvas or ImageBitmap
+ *        on each tick. Useful for custom sources the DOM walker can't
+ *        introspect: `<video>` elements, WebGL/3D scenes, off-screen canvases.
+ *
+ * Adding a new source (e.g. `{ type: "video" }`) is a matter of:
+ *   - adding a variant in `types.ts`
+ *   - adding a case here.
+ */
+
+import type { FrameSource } from "../types";
+import { getWireframeCtor } from "./wireframe";
+
+const DEFAULT_RATE_HZ = 2;
+
+function intervalMs(rate: number | undefined): number {
+  const r = typeof rate === "number" && rate > 0 ? rate : DEFAULT_RATE_HZ;
+  return Math.max(1, Math.round(1000 / r));
+}
+
+/**
+ * Resize `canvas` to match the current viewport if needed. Called on every
+ * tick so window resizes are picked up live without a separate listener.
+ */
+function syncCanvasSize(canvas: HTMLCanvasElement): void {
+  const w = Math.max(1, window.innerWidth);
+  const h = Math.max(1, window.innerHeight);
+  if (canvas.width !== w || canvas.height !== h) {
+    canvas.width = w;
+    canvas.height = h;
+  }
+}
+
+/**
+ * Start capturing frames from `source` into `canvas`. Returns a stop function
+ * that halts the loop. Errors inside a single tick are logged and skipped —
+ * one bad frame shouldn't kill vision for the rest of the session.
+ */
+export function startFrameSource(
+  source: FrameSource,
+  canvas: HTMLCanvasElement,
+): () => void {
+  const ctx = canvas.getContext("2d", { alpha: false });
+  if (!ctx) {
+    console.warn("[Sable] frame source: 2d context unavailable");
+    return () => {};
+  }
+
+  const delayMs = intervalMs(source.rate);
+
+  let stopped = false;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let inFlight = false;
+
+  const tick = async (): Promise<void> => {
+    if (stopped) return;
+    if (!inFlight) {
+      inFlight = true;
+      try {
+        syncCanvasSize(canvas);
+        if (source.type === "wireframe") {
+          const includeImages = source.features?.includeImages === true;
+          const Wireframe = getWireframeCtor();
+          const wf = new Wireframe(document.body, { images: includeImages });
+          const { canvas: src } = await wf.capture();
+          ctx.fillStyle = "#ffffff";
+          ctx.fillRect(0, 0, canvas.width, canvas.height);
+          ctx.drawImage(src, 0, 0, canvas.width, canvas.height);
+        } else if (source.type === "fn") {
+          const frame = source.captureFn();
+          ctx.fillStyle = "#ffffff";
+          ctx.fillRect(0, 0, canvas.width, canvas.height);
+          // Both HTMLCanvasElement and ImageBitmap are valid drawImage sources.
+          ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
+        }
+      } catch (e) {
+        console.warn("[Sable] frame source tick failed", e);
+      } finally {
+        inFlight = false;
+      }
+    }
+    if (!stopped) {
+      timer = setTimeout(tick, delayMs);
+    }
+  };
+
+  void tick();
+
+  return () => {
+    stopped = true;
+    if (timer !== undefined) clearTimeout(timer);
+  };
+}