@sable-ai/sdk-core 0.1.0

package/src/runtime/index.ts

@@ -0,0 +1,138 @@
+/**
+ * Runtime: the set of methods the agent can RPC into the page.
+ *
+ * Historically these were called "UI stubs" because they originated as
+ * no-op placeholders for methods parley implements in its call overlay.
+ * That framing no longer fits: half of them now do real work, and the
+ * public API lets customers replace any of them AND add new ones through
+ * the same `Sable.start({ runtime })` surface.
+ *
+ * The shape:
+ *
+ *   1. `DEFAULT_RUNTIME` — built-in implementations shipped with the SDK.
+ *      A few do real work (clipboard copy, video overlay); the rest are
+ *      no-ops for methods that only make sense in a host-app call UI.
+ *
+ *   2. `installRuntime(room, userRuntime)` — merges `userRuntime` over the
+ *      defaults and registers every entry as a LiveKit RPC handler on
+ *      `room`. Agent RPC calls → run the matching method → return a
+ *      JSON-encoded result.
+ *
+ * Customers extend the runtime by passing new keys in `userRuntime`:
+ * anything you put in becomes callable by the agent as-is. This means
+ * the same surface handles both "override a built-in" and "expose a
+ * business-logic tool" — one concept, not two.
+ */
+
+import { safeParse, type RpcRoom } from "../rpc";
+import type { RuntimeMethod, RuntimeMethods } from "../types";
+import { handleCopyable } from "./clipboard";
+import { mountVideoOverlay, removeViewOverlay } from "./video-overlay";
+
+// ── Default runtime ────────────────────────────────────────────────────────
+//
+// Methods that do meaningful work out of the box (clipboard, video overlay)
+// plus no-ops for the set of call-UI methods the agent can call. The no-ops
+// are here so agent tool calls always succeed even if the host app hasn't
+// overridden them — without these, the agent's builtin tools raise
+// "Method not supported at destination" and the conversation derails.
+
+function noop(): Promise<{ success: true }> {
+  return Promise.resolve({ success: true });
+}
+
+export const DEFAULT_RUNTIME: RuntimeMethods = {
+  // Real defaults
+  sendToolMessage: (payload) => handleCopyable("sendToolMessage", payload),
+  sendCopyableText: (payload) => handleCopyable("sendCopyableText", payload),
+  switchView: async (payload) => {
+    const mode = typeof payload.mode === "string" ? payload.mode : "";
+    const url = typeof payload.url === "string" ? payload.url : "";
+    if (mode === "video" && url) {
+      try {
+        mountVideoOverlay(url);
+        return { success: true };
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        console.warn("[Sable] switchView: failed to mount video overlay", msg);
+        return { success: false, error: msg };
+      }
+    }
+    removeViewOverlay();
+    return { success: true };
+  },
+
+  // No-ops: host-UI-specific methods. Host apps override to render into
+  // their own call surface; the SDK's default answer is "ack, did nothing".
+  setCallControlsEnabled: noop,
+  setUserInputEnabled: noop,
+  setAgentInControl: noop,
+  showSuggestedReplies: noop,
+  hideSuggestedReplies: noop,
+  highlightHangup: noop,
+  hideVideo: noop,
+  showVideo: noop,
+  stopScreenShare: noop,
+  showSlide: noop,
+  hideSlide: noop,
+  responseFailed: noop,
+  requestContinue: noop,
+  greetingComplete: noop,
+  speechComplete: noop,
+  enableMicrophone: noop,
+  requestDisconnect: noop,
+  setNickelSession: noop,
+};
+
+// ── Registration ───────────────────────────────────────────────────────────
+
+/**
+ * Wrap a runtime method in an RPC-compatible handler: decode JSON payload,
+ * run the method, re-encode the result. Errors become `{ error: message }`
+ * responses — never thrown back to the caller, since LiveKit RPC propagates
+ * exceptions to the agent and derails the conversation.
+ */
+function toRpcHandler(
+  name: string,
+  method: RuntimeMethod,
+): (data: { payload: string }) => Promise<string> {
+  return async (data) => {
+    const payload = safeParse(data.payload);
+    try {
+      const result = await method(payload);
+      return JSON.stringify(result ?? { success: true });
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[Sable] runtime method "${name}" threw`, msg);
+      return JSON.stringify({ success: false, error: msg });
+    }
+  };
+}
+
+/**
+ * Merge the user-provided runtime over `DEFAULT_RUNTIME` and register every
+ * entry as a LiveKit RPC handler on `room`. Later keys win — so passing
+ * `{ switchView: myImpl }` replaces the default video-overlay behaviour,
+ * while passing `{ activateTrial: ... }` exposes a new method the agent
+ * can call without touching the built-ins.
+ */
+export function installRuntime(
+  room: RpcRoom,
+  userRuntime: RuntimeMethods = {},
+): void {
+  const merged: RuntimeMethods = { ...DEFAULT_RUNTIME, ...userRuntime };
+  for (const [name, method] of Object.entries(merged)) {
+    room.registerRpcMethod(name, toRpcHandler(name, method));
+  }
+  const overrides = Object.keys(userRuntime).filter(
+    (k) => k in DEFAULT_RUNTIME,
+  );
+  const extensions = Object.keys(userRuntime).filter(
+    (k) => !(k in DEFAULT_RUNTIME),
+  );
+  console.log("[Sable] runtime installed", {
+    total: Object.keys(merged).length,
+    overrides,
+    extensions,
+  });
+}

package/src/runtime/video-overlay.ts

@@ -0,0 +1,94 @@
+/**
+ * Default `switchView({ mode: "video", url })` implementation.
+ *
+ * Mounts a centred floating video clip in the page. Host apps with their own
+ * call UI would override `switchView` in `runtime` to render into their own
+ * surface; the standalone SDK uses this simple overlay as the built-in
+ * default so agents that call `switchView` Just Work out of the box.
+ *
+ * Module-level state (`activeViewOverlay`) keeps at most one overlay mounted
+ * — calling `mountVideoOverlay` while another is showing tears the old one
+ * down first.
+ */
+
+let activeViewOverlay: HTMLDivElement | null = null;
+
+export function removeViewOverlay(): void {
+  if (activeViewOverlay) {
+    activeViewOverlay.remove();
+    activeViewOverlay = null;
+  }
+}
+
+export function mountVideoOverlay(url: string): void {
+  removeViewOverlay();
+
+  const overlay = document.createElement("div");
+  overlay.setAttribute("data-sable", "view-overlay");
+  Object.assign(overlay.style, {
+    position: "fixed",
+    top: "50%",
+    left: "50%",
+    transform: "translate(-50%, -50%)",
+    zIndex: "2147483646",
+    background: "rgba(0, 0, 0, 0.85)",
+    borderRadius: "12px",
+    padding: "8px",
+    boxShadow: "0 10px 40px rgba(0, 0, 0, 0.5)",
+    maxWidth: "min(80vw, 960px)",
+    maxHeight: "80vh",
+    display: "flex",
+    flexDirection: "column",
+    gap: "8px",
+  } as Partial<CSSStyleDeclaration>);
+
+  const closeBtn = document.createElement("button");
+  closeBtn.textContent = "✕";
+  closeBtn.setAttribute("aria-label", "Close video");
+  Object.assign(closeBtn.style, {
+    alignSelf: "flex-end",
+    background: "rgba(255,255,255,0.15)",
+    color: "white",
+    border: "none",
+    borderRadius: "999px",
+    width: "28px",
+    height: "28px",
+    cursor: "pointer",
+    fontSize: "14px",
+    lineHeight: "1",
+  } as Partial<CSSStyleDeclaration>);
+  closeBtn.addEventListener("click", removeViewOverlay);
+
+  const video = document.createElement("video");
+  video.src = url;
+  video.controls = false;
+  video.autoplay = true;
+  video.playsInline = true;
+  video.disablePictureInPicture = true;
+  video.setAttribute(
+    "controlslist",
+    "nodownload nofullscreen noremoteplayback noplaybackrate",
+  );
+  Object.assign(video.style, {
+    maxWidth: "100%",
+    maxHeight: "70vh",
+    borderRadius: "8px",
+    display: "block",
+  } as Partial<CSSStyleDeclaration>);
+
+  const onEnded = (): void => {
+    if (activeViewOverlay === overlay) {
+      removeViewOverlay();
+    }
+  };
+  video.addEventListener("ended", onEnded);
+
+  overlay.appendChild(closeBtn);
+  overlay.appendChild(video);
+  document.body.appendChild(overlay);
+  activeViewOverlay = overlay;
+
+  video.play().catch((e) => {
+    console.warn("[Sable] switchView video autoplay blocked", e);
+  });
+}

package/src/session/debug-panel.ts

@@ -0,0 +1,254 @@
+/**
+ * Floating debug panel — "what the agent sees".
+ *
+ * When debug is on, we mount the vision capture canvas as a draggable
+ * preview in the host page. The panel renders the *exact* pixels that get
+ * encoded into the LiveKit video track, so "what you see in the panel" is
+ * literally "what the agent sees". Position + minimized state persist in
+ * `localStorage` so customers don't have to re-place the panel every
+ * reload.
+ *
+ * Opt-in signals (any of these enables the panel):
+ *   - `Sable.start({ debug: true })`
+ *   - `?sable-debug=1` anywhere in the page URL
+ *   - `localStorage.setItem('sable:debug', '1')`
+ */
+
+interface DebugPanelState {
+  left?: number;
+  top?: number;
+  minimized?: boolean;
+}
+
+const DEBUG_PANEL_STATE_KEY = "sable:debug:panel";
+
+function loadState(): DebugPanelState {
+  try {
+    const raw = window.localStorage?.getItem(DEBUG_PANEL_STATE_KEY);
+    if (!raw) return {};
+    const parsed = JSON.parse(raw) as DebugPanelState;
+    return typeof parsed === "object" && parsed !== null ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+function saveState(state: DebugPanelState): void {
+  try {
+    window.localStorage?.setItem(DEBUG_PANEL_STATE_KEY, JSON.stringify(state));
+  } catch {
+    /* ignore */
+  }
+}
+
+/**
+ * Opt-in check: does ANY of the debug signals say we should show the panel?
+ * Called by the session before deciding whether to mount.
+ */
+export function shouldShowDebugPanel(debugOpt: boolean | undefined): boolean {
+  if (debugOpt) return true;
+  try {
+    if (new URL(window.location.href).searchParams.get("sable-debug") === "1") {
+      return true;
+    }
+  } catch {
+    /* ignore */
+  }
+  try {
+    if (window.localStorage?.getItem("sable:debug") === "1") return true;
+  } catch {
+    /* ignore */
+  }
+  return false;
+}
+
+/**
+ * Mount `canvas` as a floating preview in the page. Returns a teardown that
+ * removes the wrapper and detaches listeners.
+ *
+ * The wrapper is pointer-events:none by default; only the header bar and
+ * its minimize button re-enable pointer events, so the panel never blocks
+ * clicks on the underlying page.
+ */
+export function mountDebugPanel(canvas: HTMLCanvasElement): () => void {
+  const state = loadState();
+
+  const wrap = document.createElement("div");
+  wrap.setAttribute("data-sable-debug", "vision");
+  Object.assign(wrap.style, {
+    position: "fixed",
+    width: "240px",
+    zIndex: "2147483647",
+    background: "#111",
+    color: "#ddd",
+    border: "1px solid #444",
+    borderRadius: "8px",
+    font: "11px/1.3 system-ui, sans-serif",
+    boxShadow: "0 8px 24px rgba(0,0,0,.4)",
+    pointerEvents: "none",
+    userSelect: "none",
+    overflow: "hidden",
+  } as Partial<CSSStyleDeclaration>);
+
+  // Initial placement: restored from localStorage, else default top-right.
+  if (typeof state.left === "number" && typeof state.top === "number") {
+    wrap.style.left = `${state.left}px`;
+    wrap.style.top = `${state.top}px`;
+  } else {
+    wrap.style.right = "12px";
+    wrap.style.top = "12px";
+  }
+
+  // ── Header: drag handle + minimize button ────────────────────────────
+  const header = document.createElement("div");
+  Object.assign(header.style, {
+    display: "flex",
+    alignItems: "center",
+    justifyContent: "space-between",
+    gap: "6px",
+    padding: "6px 8px",
+    background: "#1a1a1a",
+    borderBottom: "1px solid #333",
+    cursor: "move",
+    pointerEvents: "auto",
+  } as Partial<CSSStyleDeclaration>);
+
+  const title = document.createElement("div");
+  title.textContent = "sable: agent vision";
+  Object.assign(title.style, {
+    opacity: "0.75",
+    fontWeight: "600",
+    flex: "1",
+    pointerEvents: "none",
+  } as Partial<CSSStyleDeclaration>);
+  header.appendChild(title);
+
+  const minBtn = document.createElement("button");
+  minBtn.setAttribute("aria-label", "Minimize vision panel");
+  Object.assign(minBtn.style, {
+    background: "transparent",
+    color: "#ddd",
+    border: "1px solid #444",
+    borderRadius: "4px",
+    width: "20px",
+    height: "20px",
+    cursor: "pointer",
+    fontSize: "12px",
+    lineHeight: "1",
+    padding: "0",
+    pointerEvents: "auto",
+  } as Partial<CSSStyleDeclaration>);
+  header.appendChild(minBtn);
+
+  wrap.appendChild(header);
+
+  // ── Body: the capture canvas (click-through) ─────────────────────────
+  const body = document.createElement("div");
+  Object.assign(body.style, {
+    padding: "6px",
+    background: "#111",
+    pointerEvents: "none",
+  } as Partial<CSSStyleDeclaration>);
+
+  canvas.style.width = "100%";
+  canvas.style.height = "auto";
+  canvas.style.display = "block";
+  canvas.style.background = "#fff";
+  canvas.style.borderRadius = "4px";
+  canvas.style.pointerEvents = "none";
+  body.appendChild(canvas);
+
+  wrap.appendChild(body);
+
+  // ── Minimize state ───────────────────────────────────────────────────
+  let minimized = !!state.minimized;
+  const applyMinimized = (): void => {
+    body.style.display = minimized ? "none" : "block";
+    minBtn.textContent = minimized ? "▢" : "–";
+    minBtn.setAttribute(
+      "aria-label",
+      minimized ? "Restore vision panel" : "Minimize vision panel",
+    );
+  };
+  applyMinimized();
+
+  minBtn.addEventListener("click", (ev) => {
+    ev.stopPropagation();
+    minimized = !minimized;
+    applyMinimized();
+    saveState({ ...loadState(), minimized });
+  });
+
+  // ── Drag behaviour ───────────────────────────────────────────────────
+  let dragState: { offsetX: number; offsetY: number } | null = null;
+
+  const onPointerDown = (ev: PointerEvent): void => {
+    if (ev.target instanceof HTMLElement && ev.target.closest("button")) {
+      return;
+    }
+    const rect = wrap.getBoundingClientRect();
+    dragState = {
+      offsetX: ev.clientX - rect.left,
+      offsetY: ev.clientY - rect.top,
+    };
+    // Convert to left/top anchoring on first drag.
+    wrap.style.left = `${rect.left}px`;
+    wrap.style.top = `${rect.top}px`;
+    wrap.style.right = "auto";
+    wrap.style.bottom = "auto";
+    header.setPointerCapture(ev.pointerId);
+    ev.preventDefault();
+  };
+
+  const onPointerMove = (ev: PointerEvent): void => {
+    if (!dragState) return;
+    // Clamp so the panel can't be dragged fully off-screen (keep 24px of
+    // the header visible on every edge so the user can always grab it).
+    const vw = window.innerWidth;
+    const vh = window.innerHeight;
+    const w = wrap.offsetWidth;
+    let left = ev.clientX - dragState.offsetX;
+    let top = ev.clientY - dragState.offsetY;
+    left = Math.min(Math.max(left, -w + 48), vw - 48);
+    top = Math.min(Math.max(top, 0), vh - 24);
+    wrap.style.left = `${left}px`;
+    wrap.style.top = `${top}px`;
+  };
+
+  const onPointerUp = (ev: PointerEvent): void => {
+    if (!dragState) return;
+    dragState = null;
+    try {
+      header.releasePointerCapture(ev.pointerId);
+    } catch {
+      /* ignore */
+    }
+    const left = parseFloat(wrap.style.left) || 0;
+    const top = parseFloat(wrap.style.top) || 0;
+    saveState({ ...loadState(), left, top });
+  };
+
+  header.addEventListener("pointerdown", onPointerDown);
+  header.addEventListener("pointermove", onPointerMove);
+  header.addEventListener("pointerup", onPointerUp);
+  header.addEventListener("pointercancel", onPointerUp);
+
+  document.body.appendChild(wrap);
+  console.log("[Sable] debug vision panel mounted", {
+    minimized,
+    restoredPosition:
+      typeof state.left === "number" && typeof state.top === "number",
+  });
+
+  return () => {
+    try {
+      header.removeEventListener("pointerdown", onPointerDown);
+      header.removeEventListener("pointermove", onPointerMove);
+      header.removeEventListener("pointerup", onPointerUp);
+      header.removeEventListener("pointercancel", onPointerUp);
+      wrap.remove();
+    } catch {
+      /* ignore */
+    }
+  };
+}