@sable-ai/sdk-core 0.1.0

package/src/browser-bridge/actions.ts

@@ -0,0 +1,161 @@
+/**
+ * Action dispatcher for `browser.execute_action`.
+ *
+ * The canonical wire contract lives in the Python bridge
+ * (`sable_agentkit/components/browser/bridges/wire.py`). The `kind` tag and
+ * payload shape of each variant must stay in lock-step with it — if a new
+ * action lands on the Python side, mirror it here.
+ *
+ * Target resolution: actions can target an element either by CSS selector
+ * string or by `{ x, y }` coordinates (for vision-driven clicks where the
+ * agent only knows pixel positions). See `resolveTarget`.
+ */
+
+interface Coordinates {
+  x: number;
+  y: number;
+}
+
+function isCoordinates(p: unknown): p is Coordinates {
+  return (
+    typeof p === "object" &&
+    p !== null &&
+    typeof (p as Coordinates).x === "number" &&
+    typeof (p as Coordinates).y === "number"
+  );
+}
+
+/**
+ * Resolve an Action.payload to a target element.
+ *   - `{ x, y }` → `document.elementFromPoint`
+ *   - selector string → `document.querySelector`
+ */
+function resolveTarget(payload: unknown): Element | null {
+  if (isCoordinates(payload)) {
+    return document.elementFromPoint(payload.x, payload.y);
+  }
+  if (typeof payload === "string") {
+    try {
+      return document.querySelector(payload);
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}
+
+export interface ActionEnvelope {
+  kind: string;
+  payload?: unknown;
+  // common variants
+  button?: string;
+  key?: string;
+  text?: string;
+  delay?: number;
+  replace?: boolean;
+  url?: string;
+  expression?: string;
+  start?: unknown;
+  end?: unknown;
+  steps?: number;
+}
+
+export async function dispatchAction(action: ActionEnvelope): Promise<void> {
+  switch (action.kind) {
+    case "click": {
+      const el = resolveTarget(action.payload);
+      if (!el) throw new Error(`click: target not found`);
+      (el as HTMLElement).scrollIntoView({ block: "center", inline: "center" });
+      (el as HTMLElement).click();
+      return;
+    }
+    case "hover": {
+      const el = resolveTarget(action.payload);
+      if (!el) return;
+      el.dispatchEvent(
+        new MouseEvent("mouseover", { bubbles: true, cancelable: true }),
+      );
+      return;
+    }
+    case "type": {
+      const el = resolveTarget(action.payload);
+      if (!el) throw new Error(`type: target not found`);
+      const input = el as HTMLInputElement | HTMLTextAreaElement;
+      input.focus();
+      if (action.replace) {
+        input.value = "";
+      }
+      const text = action.text ?? "";
+      // Use the native setter so React/Vue controlled inputs see the change.
+      const proto =
+        input instanceof HTMLTextAreaElement
+          ? HTMLTextAreaElement.prototype
+          : HTMLInputElement.prototype;
+      const setter = Object.getOwnPropertyDescriptor(proto, "value")?.set;
+      if (setter) {
+        setter.call(input, (input.value ?? "") + text);
+      } else {
+        input.value = (input.value ?? "") + text;
+      }
+      input.dispatchEvent(new Event("input", { bubbles: true }));
+      input.dispatchEvent(new Event("change", { bubbles: true }));
+      return;
+    }
+    case "key": {
+      const target = (document.activeElement ?? document.body) as HTMLElement;
+      const key = action.key ?? "";
+      target.dispatchEvent(
+        new KeyboardEvent("keydown", { key, bubbles: true, cancelable: true }),
+      );
+      target.dispatchEvent(
+        new KeyboardEvent("keyup", { key, bubbles: true, cancelable: true }),
+      );
+      return;
+    }
+    case "clear": {
+      const el = document.activeElement as
+        | HTMLInputElement
+        | HTMLTextAreaElement
+        | null;
+      if (!el || !("value" in el)) return;
+      el.value = "";
+      el.dispatchEvent(new Event("input", { bubbles: true }));
+      el.dispatchEvent(new Event("change", { bubbles: true }));
+      return;
+    }
+    case "navigate": {
+      const url = action.url ?? "";
+      if (!url) return;
+      // Same-URL is a no-op so the agent can re-issue navigate cheaply.
+      if (url === window.location.href) return;
+      // Full-document navigation tears down the SDK; for v0 the SDK is
+      // expected to live inside the destination page already, so log
+      // and best-effort assign. The extension/host must re-inject after
+      // the new document loads.
+      console.warn(
+        "[Sable] browser.navigate will reload the page; SDK must be re-injected on the new document",
+        { url },
+      );
+      window.location.assign(url);
+      return;
+    }
+    case "evaluate": {
+      const expr = action.expression ?? "";
+      // eslint-disable-next-line @typescript-eslint/no-implied-eval
+      (0, eval)(expr);
+      return;
+    }
+    // Visual-only actions are no-ops for v0; they exist on Nickel for the
+    // server-rendered demo recordings, not for an SDK-driven user browser.
+    case "highlight_box":
+    case "highlight_text":
+    case "select_text":
+    case "center_scroll":
+    case "drag":
+    case "hide_cursor":
+    case "show_cursor":
+      return;
+    default:
+      throw new Error(`unsupported action kind: ${action.kind}`);
+  }
+}

package/src/browser-bridge/dom-state.ts

@@ -0,0 +1,103 @@
+/**
+ * DOM-state capture for `browser.get_dom_state`.
+ *
+ * The agent calls `browser.get_dom_state` when it needs a fresh snapshot of
+ * the page before deciding on the next action. The response carries three
+ * things:
+ *
+ *   - `screenshot_jpeg_b64` — a wireframe-rendered image of `document.body`
+ *     (the field name is historical; the bytes are PNG — Northstar treats
+ *     it as an opaque image and doesn't enforce the codec)
+ *   - `elements` — the visible-element list produced by `visible-dom.js`,
+ *     an agent-friendly structured summary of the interactive DOM
+ *   - `viewport` + `url` — so the agent can reason about pixel coordinates
+ *     and the current page identity
+ *
+ * `visible-dom.js` is shipped as a text asset and eval'd once on first use.
+ * `settle()` is also here — it's a mutation-observer quiet-period wait used
+ * by the `browser.settle` RPC to let animations/transitions finish before
+ * the agent reads DOM state again.
+ */
+
+import visibleDomJs from "../assets/visible-dom.js.txt";
+import { getWireframeCtor } from "../vision/wireframe";
+
+let visibleDomFn: (() => unknown) | null = null;
+
+function getVisibleDomFn(): () => unknown {
+  if (!visibleDomFn) {
+    // The text starts with `() => { ... }` — wrap in parens so eval
+    // returns the function expression.
+    visibleDomFn = (0, eval)(`(${visibleDomJs})`) as () => unknown;
+  }
+  return visibleDomFn;
+}
+
+export interface DomStateResponse {
+  screenshot_jpeg_b64: string;
+  elements: unknown;
+  viewport: { width: number; height: number };
+  url: string;
+}
+
+export async function captureDomState(): Promise<DomStateResponse> {
+  const elements = getVisibleDomFn()();
+
+  const Wireframe = getWireframeCtor();
+  const wf = new Wireframe(document.body, {});
+  const dataUrl = await wf.toDataURL();
+  // Strip the `data:image/png;base64,` prefix.
+  const b64 = dataUrl.replace(/^data:[^,]+,/, "");
+
+  return {
+    screenshot_jpeg_b64: b64,
+    elements,
+    viewport: { width: window.innerWidth, height: window.innerHeight },
+    url: window.location.href,
+  };
+}
+
+/**
+ * Mutation-observer quiet-period wait. Mirrors `visible_dom.py`'s settle —
+ * return as soon as the DOM has been quiet for `QUIET_MS`, or after
+ * `MAX_MS`, whichever comes first. Bookended by two double-rAFs so any
+ * in-flight layout/paint work gets flushed before and after the wait.
+ */
+export async function settle(): Promise<void> {
+  const raf2 = (): Promise<void> =>
+    new Promise<void>((r) =>
+      requestAnimationFrame(() => requestAnimationFrame(() => r())),
+    );
+  await raf2();
+  await new Promise<void>((resolve) => {
+    const QUIET_MS = 30;
+    const MAX_MS = 30;
+    const start = performance.now();
+    let lastMut = performance.now();
+    const obs = new MutationObserver(() => {
+      lastMut = performance.now();
+    });
+    obs.observe(document.documentElement, {
+      subtree: true,
+      childList: true,
+      attributes: true,
+      characterData: true,
+    });
+    const tick = (): void => {
+      const now = performance.now();
+      if (now - lastMut >= QUIET_MS) {
+        obs.disconnect();
+        resolve();
+        return;
+      }
+      if (now - start >= MAX_MS) {
+        obs.disconnect();
+        resolve();
+        return;
+      }
+      requestAnimationFrame(tick);
+    };
+    requestAnimationFrame(tick);
+  });
+  await raf2();
+}

package/src/browser-bridge/index.ts

@@ -0,0 +1,99 @@
+/**
+ * SDK side of the Sable browser bridge.
+ *
+ * Registers six LiveKit RPC handlers that the agent's UserBrowserBridge
+ * (`sable-agentkit/components/browser/bridges/user.py`) calls into:
+ *
+ *   browser.execute_action  → dispatches an Action variant against the page
+ *   browser.get_dom_state   → wireframe screenshot + visible-element list
+ *   browser.get_url         → window.location.href
+ *   browser.get_viewport    → window.innerWidth/innerHeight
+ *   browser.verify_selector → !!document.querySelector(selector)
+ *   browser.settle          → mutation-observer quiet-period wait
+ *
+ * The wire contract is the canonical Python implementation in
+ * `sable_agentkit/components/browser/bridges/wire.py` — every field shape
+ * and Action `kind` tag must match it exactly.
+ */
+
+import type { RpcRoom } from "../rpc";
+import { dispatchAction, type ActionEnvelope } from "./actions";
+import { captureDomState, settle } from "./dom-state";
+
+function makeHandler(
+  name: string,
+  body: (req: Record<string, unknown>) => Promise<unknown>,
+): (data: { payload: string }) => Promise<string> {
+  return async (data) => {
+    let req: Record<string, unknown> = {};
+    try {
+      req = data.payload ? JSON.parse(data.payload) : {};
+    } catch (e) {
+      console.warn(`[Sable] ${name}: bad JSON payload`, e);
+    }
+    try {
+      const result = await body(req);
+      return JSON.stringify(result ?? {});
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[Sable] ${name}: handler error`, msg);
+      return JSON.stringify({ error: msg });
+    }
+  };
+}
+
+export function registerBrowserHandlers(room: RpcRoom): void {
+  room.registerRpcMethod(
+    "browser.execute_action",
+    makeHandler("browser.execute_action", async (req) => {
+      const action = req.action as ActionEnvelope | undefined;
+      if (!action || typeof action !== "object") {
+        throw new Error("execute_action: missing action");
+      }
+      await dispatchAction(action);
+      return {};
+    }),
+  );
+
+  room.registerRpcMethod(
+    "browser.get_dom_state",
+    makeHandler("browser.get_dom_state", async () => captureDomState()),
+  );
+
+  room.registerRpcMethod(
+    "browser.get_url",
+    makeHandler("browser.get_url", async () => ({ url: window.location.href })),
+  );
+
+  room.registerRpcMethod(
+    "browser.get_viewport",
+    makeHandler("browser.get_viewport", async () => ({
+      width: window.innerWidth,
+      height: window.innerHeight,
+    })),
+  );
+
+  room.registerRpcMethod(
+    "browser.verify_selector",
+    makeHandler("browser.verify_selector", async (req) => {
+      const selector = typeof req.selector === "string" ? req.selector : "";
+      let matches = false;
+      try {
+        matches = !!document.querySelector(selector);
+      } catch {
+        matches = false;
+      }
+      return { matches };
+    }),
+  );
+
+  room.registerRpcMethod(
+    "browser.settle",
+    makeHandler("browser.settle", async () => {
+      await settle();
+      return {};
+    }),
+  );
+
+  console.log("[Sable] browser bridge RPCs registered");
+}

package/src/connection/index.ts

@@ -0,0 +1,49 @@
+/**
+ * sable-api `/connection-details` fetch.
+ *
+ * Isolated in its own file because this is the ONE thing that changes when
+ * the backend flips from raw agent IDs (`?agentPublicId=...`) to publishable
+ * keys (`?publicKey=pk_live_...` + per-agent allowed-domains origin check).
+ * When that lands, the diff is contained to this file.
+ *
+ * Until then: we accept `publicKey` from the customer and pass it through as
+ * `?agentPublicId=...` on the wire, which keeps the current sable-api
+ * contract working while the public-facing option name is already the one
+ * we want long-term.
+ */
+
+export const DEFAULT_API_URL = "https://sable-api-gateway-9dfmhij9.wl.gateway.dev";
+
+export interface ConnectionDetails {
+  serverUrl: string;
+  roomName: string;
+  participantToken: string;
+  participantName: string;
+}
+
+export interface FetchConnectionDetailsInput {
+  apiUrl: string;
+  /** Either a `pk_live_...` publishable key or a raw `agt_...` agent ID. */
+  publicKey: string;
+}
+
+export async function fetchConnectionDetails(
+  input: FetchConnectionDetailsInput,
+): Promise<ConnectionDetails> {
+  const url = new URL("/connection-details", input.apiUrl);
+  // During beta the backend still reads this as `agentPublicId`. When pk_live
+  // keys land, rename here (and only here).
+  url.searchParams.set("agentPublicId", input.publicKey);
+  // The SDK is, by definition, the "agent drives the user's browser" path.
+  // There is no nickel-backed SDK mode — that's what the virtual browser
+  // product handles — so we hardcode the bridge attribute here instead of
+  // leaking it into the public API.
+  url.searchParams.set("bridge", "user");
+
+  const res = await fetch(url.toString());
+  if (!res.ok) {
+    const body = await res.text().catch(() => "");
+    throw new Error(`connection-details failed: ${res.status} ${body}`);
+  }
+  return (await res.json()) as ConnectionDetails;
+}

package/src/events/index.ts

@@ -0,0 +1,50 @@
+/**
+ * Typed event emitter for the SDK's public event surface.
+ *
+ * Intentionally tiny — one map, one loop, no dependency on any EventTarget
+ * polyfill. We keep fire-and-forget semantics: handler exceptions are caught
+ * and logged so one misbehaving subscriber can't break the session.
+ */
+
+import type { SableEventHandler, SableEvents } from "../types";
+
+type HandlerSet<E extends keyof SableEvents> = Set<SableEventHandler<E>>;
+
+export class SableEventEmitter {
+  // Map of event name → set of handlers. Any-typed because TS can't express
+  // "Map<K, Set<Handler<K>>>" cleanly; we gate access through the typed
+  // `on`/`emit` methods below.
+  private readonly listeners = new Map<keyof SableEvents, Set<unknown>>();
+
+  on<E extends keyof SableEvents>(
+    event: E,
+    handler: SableEventHandler<E>,
+  ): () => void {
+    let set = this.listeners.get(event) as HandlerSet<E> | undefined;
+    if (!set) {
+      set = new Set<SableEventHandler<E>>();
+      this.listeners.set(event, set as unknown as Set<unknown>);
+    }
+    set.add(handler);
+    return () => {
+      set?.delete(handler);
+    };
+  }
+
+  emit<E extends keyof SableEvents>(event: E, payload: SableEvents[E]): void {
+    const set = this.listeners.get(event) as HandlerSet<E> | undefined;
+    if (!set || set.size === 0) return;
+    for (const handler of set) {
+      try {
+        handler(payload);
+      } catch (err) {
+        console.warn(`[Sable] event handler for "${String(event)}" threw`, err);
+      }
+    }
+  }
+
+  /** Drop every handler. Called on session teardown to avoid leaks. */
+  clear(): void {
+    this.listeners.clear();
+  }
+}

package/src/global.ts

@@ -0,0 +1,35 @@
+/**
+ * `window.Sable` installer.
+ *
+ * The SDK ships in two distribution formats that MUST present the same
+ * runtime singleton:
+ *
+ *   1. IIFE bundle (`<script src="https://sdk.withsable.com/v1/sable.js">`)
+ *      — auto-installs `window.Sable` on load.
+ *
+ *   2. npm ESM (`import Sable from "@sable-ai/sdk-core"`) — the `index.ts`
+ *      barrel calls `installGlobal()` in addition to exporting `Sable` as
+ *      its default export. If the customer also loaded the IIFE in the
+ *      same page, the second install is a no-op (first-write-wins), so
+ *      framework apps and script-tag users see the exact same session
+ *      object. This is the Stripe/Intercom pattern: one global, multiple
+ *      ways to reach it.
+ */
+
+import { Session } from "./session";
+import type { SableAPI } from "./types";
+
+/** The process-wide Sable singleton. Used by both `index.ts` and `installGlobal`. */
+export const Sable: SableAPI = new Session();
+
+/**
+ * Attach `Sable` to `window.Sable`, unless something already claimed that
+ * slot. First-write-wins so mixed script-tag + npm usage doesn't swap the
+ * singleton mid-session.
+ */
+export function installGlobal(): void {
+  if (typeof window === "undefined") return;
+  if (window.Sable) return;
+  window.Sable = Sable;
+  console.log("[Sable] SDK loaded", Sable.version);
+}

package/src/index.test.ts

@@ -0,0 +1,6 @@
+import { test, expect } from "bun:test";
+import { VERSION } from "./index";
+
+test("VERSION is exported as 0.1.0", () => {
+  expect(VERSION).toBe("0.1.0");
+});

package/src/index.ts

@@ -0,0 +1,43 @@
+/**
+ * @sable-ai/sdk-core — public entry point.
+ *
+ * Two ways to use this package, both backed by the same singleton:
+ *
+ *   1. Script tag (IIFE bundle):
+ *      `<script src="https://sdk.withsable.com/v1/sable.js"></script>`
+ *      — auto-installs `window.Sable`, good for no-build sites and the
+ *        Chrome extension's inject script.
+ *
+ *   2. npm package (ESM):
+ *      `import Sable from "@sable-ai/sdk-core"`
+ *      — good for framework apps. Importing also installs `window.Sable`
+ *        so mixed usage (one page, two entry points) stays coherent.
+ *
+ * This file is a barrel: all real code lives in sibling folders. Keep it
+ * that way — the build output is what customers see, and a lean entry
+ * module minimises tree-shake surprises.
+ */
+
+import { Sable, installGlobal } from "./global";
+
+// Auto-install on import. Script-tag consumers get it via the IIFE
+// wrapper's initialiser; ESM consumers get it here.
+installGlobal();
+
+// Re-exports for framework/ESM consumers who want named access to the
+// public type surface (e.g. for building typed wrappers or adapters).
+export { VERSION } from "./version";
+export type {
+  SableAPI,
+  SableEvents,
+  SableEventHandler,
+  StartOptions,
+  VisionOptions,
+  FrameSource,
+  WireframeFrameSource,
+  FnFrameSource,
+  RuntimeMethod,
+  RuntimeMethods,
+} from "./types";
+
+export default Sable;

package/src/rpc.ts

@@ -0,0 +1,31 @@
+/**
+ * Shared LiveKit RPC primitives.
+ *
+ * Both `runtime/` (agent → page method calls) and `browser-bridge/` (agent
+ * driving the user's browser) register handlers on the room via
+ * `registerRpcMethod`. They don't need the full LiveKit `Room` type — just the
+ * single method — so we describe the minimum shape here. This keeps the heavy
+ * `livekit-client` import dynamic and out of the IIFE entry bundle.
+ */
+
+export interface RpcRoom {
+  registerRpcMethod(
+    method: string,
+    handler: (data: { payload: string }) => Promise<string>,
+  ): void;
+}
+
+/**
+ * Parse an RPC payload string into a plain object. RPC payloads are JSON but
+ * we don't want a single malformed call from the agent to throw inside a
+ * handler — LiveKit RPC propagates exceptions back to the caller and the
+ * agent's tool use logic treats that as a hard error that can derail the
+ * conversation. Soft-failing to `{}` lets the handler decide what to do.
+ */
+export function safeParse(payload: string): Record<string, unknown> {
+  try {
+    return payload ? (JSON.parse(payload) as Record<string, unknown>) : {};
+  } catch {
+    return {};
+  }
+}

package/src/runtime/clipboard.ts

@@ -0,0 +1,47 @@
+/**
+ * Clipboard runtime methods.
+ *
+ * `sendToolMessage` and its legacy alias `sendCopyableText` carry text the
+ * user is supposed to act on (URLs, code snippets, prompts). Parley renders
+ * them as chat bubbles; the standalone SDK has no chat surface, so we copy
+ * to the clipboard so the user can ⌘V into whatever the agent is guiding
+ * them through. URL wins over message when both are present — agents put
+ * explanatory text in `message` and the actual thing-to-copy in `url`.
+ */
+
+export async function handleCopyable(
+  rpcName: string,
+  payload: Record<string, unknown>,
+): Promise<{ success: boolean; error?: string }> {
+  const message = typeof payload.message === "string" ? payload.message : "";
+  const url = typeof payload.url === "string" ? payload.url : "";
+  const toCopy = url || message;
+
+  if (!toCopy) {
+    console.warn(`[Sable] ${rpcName}: empty payload, nothing to copy`);
+    return { success: false, error: "empty payload" };
+  }
+
+  try {
+    if (navigator.clipboard?.writeText) {
+      await navigator.clipboard.writeText(toCopy);
+      return { success: true };
+    }
+    // execCommand fallback for contexts without async clipboard API
+    // (e.g. insecure origins, older webviews).
+    const ta = document.createElement("textarea");
+    ta.value = toCopy;
+    ta.style.position = "fixed";
+    ta.style.opacity = "0";
+    document.body.appendChild(ta);
+    ta.select();
+    const ok = document.execCommand("copy");
+    document.body.removeChild(ta);
+    if (!ok) throw new Error("execCommand copy returned false");
+    return { success: true };
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    console.warn(`[Sable] ${rpcName}: copy failed`, msg);
+    return { success: false, error: msg };
+  }
+}