@sable-ai/sdk-core 0.1.0

package/src/vision/index.ts

@@ -0,0 +1,70 @@
+/**
+ * Vision entry point.
+ *
+ * `startVision` owns the full lifecycle of "what the agent sees":
+ *
+ *   1. create a capture canvas sized to the viewport
+ *   2. start the configured frame source, drawing into that canvas
+ *   3. publish the canvas as a LiveKit screenshare video track
+ *
+ * Returns both the canvas (so the session can hand it to the debug panel,
+ * which just renders the exact pixels we publish) and a combined async stop
+ * function. Everything is off by default — callers pass `vision: { enabled:
+ * true }` in `Sable.start({ ... })` to opt in.
+ */
+
+import type { FrameSource, VisionOptions } from "../types";
+import { startFrameSource } from "./frame-source";
+import {
+  publishCanvasAsVideoTrack,
+  type LiveKitPublishLib,
+  type PublishCapableRoom,
+} from "./publisher";
+
+export type { LiveKitPublishLib, PublishCapableRoom } from "./publisher";
+
+const DEFAULT_FRAME_SOURCE: FrameSource = {
+  type: "wireframe",
+  rate: 2,
+  features: { includeImages: false },
+};
+
+export interface StartVisionArgs {
+  room: PublishCapableRoom;
+  lib: LiveKitPublishLib;
+  options: VisionOptions;
+}
+
+export interface VisionHandle {
+  /** The canvas being published. Useful for the debug panel. */
+  canvas: HTMLCanvasElement;
+  /** Stop the frame loop and unpublish the track. */
+  stop: () => Promise<void>;
+}
+
+export async function startVision(args: StartVisionArgs): Promise<VisionHandle> {
+  const source: FrameSource = args.options.frameSource ?? DEFAULT_FRAME_SOURCE;
+  const fps = typeof source.rate === "number" && source.rate > 0 ? source.rate : 2;
+
+  const canvas = document.createElement("canvas");
+  canvas.width = Math.max(1, window.innerWidth);
+  canvas.height = Math.max(1, window.innerHeight);
+
+  const stopFrameSource = startFrameSource(source, canvas);
+
+  let stopPublish: () => Promise<void>;
+  try {
+    stopPublish = await publishCanvasAsVideoTrack(args.room, args.lib, canvas, fps);
+  } catch (err) {
+    stopFrameSource();
+    throw err;
+  }
+
+  return {
+    canvas,
+    stop: async () => {
+      stopFrameSource();
+      await stopPublish();
+    },
+  };
+}

package/src/vision/publisher.ts

@@ -0,0 +1,106 @@
+/**
+ * Publish a canvas as a LiveKit screenshare video track.
+ *
+ * Vision is delivered as a regular LiveKit video track rather than raw
+ * bytes: we draw each frame into a persistent canvas (see `frame-source.ts`)
+ * and hand that canvas to `canvas.captureStream(fps)`. The resulting
+ * MediaStreamTrack gets wrapped in a LocalVideoTrack and published as
+ * `Track.Source.ScreenShare`, so the agent subscribes to it the same way
+ * it would subscribe to any screenshare — no custom byte-stream handler,
+ * no per-frame PNG decode, and the codec delta-compresses mostly-static
+ * pages so bandwidth stays low.
+ *
+ * The fps passed to `captureStream` should match the frame source's render
+ * rate; a mismatch either drops frames (encoder faster than source) or
+ * wastes bandwidth (encoder slower than source). `vision/index.ts` owns
+ * the pairing.
+ */
+
+// ── livekit-client structural types ───────────────────────────────────────
+//
+// sdk-core does NOT statically import `livekit-client` — the heavy runtime
+// lives behind a dynamic import so the IIFE entry bundle stays small. We
+// describe only the minimum surface the publisher actually touches here.
+
+interface LocalTrackPublication {
+  trackSid?: string;
+}
+
+export interface PublishCapableRoom {
+  localParticipant: {
+    publishTrack(
+      track: unknown,
+      options?: { source?: unknown; name?: string },
+    ): Promise<LocalTrackPublication>;
+    unpublishTrack(track: unknown, stopOnUnpublish?: boolean): Promise<unknown>;
+  };
+}
+
+export interface LiveKitPublishLib {
+  LocalVideoTrack: new (
+    mediaStreamTrack: MediaStreamTrack,
+    constraints?: unknown,
+    userProvidedTrack?: boolean,
+  ) => unknown;
+  Track: {
+    Source: {
+      ScreenShare: unknown;
+    };
+  };
+}
+
+const BROWSER_TRACK_NAME = "browser";
+
+/**
+ * Publish `canvas` as a screenshare track at `fps` frames per second.
+ * Returns an async teardown that unpublishes the track and stops the
+ * underlying MediaStreamTrack.
+ */
+export async function publishCanvasAsVideoTrack(
+  room: PublishCapableRoom,
+  lib: LiveKitPublishLib,
+  canvas: HTMLCanvasElement,
+  fps: number,
+): Promise<() => Promise<void>> {
+  const mediaStream = (canvas as unknown as {
+    captureStream: (fps?: number) => MediaStream;
+  }).captureStream(fps);
+
+  const videoTracks = mediaStream.getVideoTracks();
+  if (videoTracks.length === 0) {
+    throw new Error("canvas.captureStream produced no video tracks");
+  }
+  const mediaStreamTrack = videoTracks[0];
+
+  // userProvidedTrack=true → livekit-client won't try to restart the
+  // track (which would fail for a canvas-backed MediaStreamTrack).
+  const localTrack = new lib.LocalVideoTrack(
+    mediaStreamTrack,
+    undefined,
+    true,
+  );
+
+  const publication = await room.localParticipant.publishTrack(localTrack, {
+    source: lib.Track.Source.ScreenShare,
+    name: BROWSER_TRACK_NAME,
+  });
+
+  console.log("[Sable] vision track published", {
+    trackSid: publication.trackSid,
+    fps,
+  });
+
+  return async () => {
+    try {
+      await room.localParticipant.unpublishTrack(localTrack, true);
+    } catch (e) {
+      console.warn("[Sable] vision unpublishTrack failed", e);
+    }
+    try {
+      mediaStreamTrack.stop();
+    } catch {
+      /* ignore */
+    }
+    console.log("[Sable] vision track stopped");
+  };
+}

package/src/vision/wireframe.ts

@@ -0,0 +1,43 @@
+/**
+ * Lazy bootstrap for the Wireframe class.
+ *
+ * `wireframe.js` is shipped as a text asset (see `assets/wireframe.js.txt`)
+ * and eval'd once at first use. Evaluating inside an IIFE with a shadowed
+ * `console` keeps the library's per-capture "[wireframe] drew N elements"
+ * log out of the host page's devtools — at 1 fps it would flood the console.
+ * `.warn`/`.error` still go through so real problems surface.
+ *
+ * Used by:
+ *   - `vision/frame-source.ts` — to render the wireframe canvas at `rate` Hz
+ *   - `browser-bridge/dom-state.ts` — to produce the `screenshot_jpeg_b64`
+ *     field returned by `browser.get_dom_state`
+ */
+
+import wireframeJs from "../assets/wireframe.js.txt";
+
+export type WireframeInstance = {
+  toDataURL(): Promise<string>;
+  capture: () => Promise<{ canvas: HTMLCanvasElement }>;
+};
+
+export type WireframeCtor = new (
+  root?: Element,
+  opts?: Record<string, unknown>,
+) => WireframeInstance;
+
+let wireframeCtor: WireframeCtor | null = null;
+
+export function getWireframeCtor(): WireframeCtor {
+  if (!wireframeCtor) {
+    wireframeCtor = (0, eval)(
+      `(function(){
+        var console = Object.assign({}, globalThis.console, {
+          log: function(){}, info: function(){}, debug: function(){}
+        });
+        ${wireframeJs};
+        return Wireframe;
+      })()`,
+    ) as WireframeCtor;
+  }
+  return wireframeCtor;
+}