cursor-buddy 0.0.2 → 0.0.4

package/dist/client-DJRU6dKB.d.mts

@@ -0,0 +1,462 @@
+//#region src/core/utils/elements.d.ts
+/**
+ * Element discovery for annotated screenshots.
+ * Finds visible interactive elements and assigns marker IDs.
+ */
+/**
+ * Element marker with reference to actual DOM element.
+ */
+interface ElementMarker {
+  /** Sequential marker ID (1, 2, 3...) */
+  id: number;
+  /** Reference to the actual DOM element */
+  element: Element;
+  /** Bounding rect at time of capture */
+  rect: DOMRect;
+  /** Brief description for AI context */
+  description: string;
+}
+/**
+ * Map of marker ID to element marker.
+ */
+type MarkerMap = Map<number, ElementMarker>;
+//#endregion
+//#region src/core/types.d.ts
+/**
+ * Voice state machine states
+ */
+type VoiceState = "idle" | "listening" | "processing" | "responding";
+/**
+ * Events for the voice state machine
+ */
+type VoiceEvent = {
+  type: "HOTKEY_PRESSED";
+} | {
+  type: "HOTKEY_RELEASED";
+} | {
+  type: "RESPONSE_STARTED";
+} | {
+  type: "TTS_COMPLETE";
+} | {
+  type: "ERROR";
+  error: Error;
+};
+/**
+ * Point coordinates parsed from AI response.
+ * Supports two formats:
+ * - Marker-based: [POINT:5:label] - references a numbered marker
+ * - Coordinate-based: [POINT:640,360:label] - raw pixel coordinates
+ */
+interface PointingTarget {
+  /** X coordinate in viewport pixels (top-left origin) */
+  x: number;
+  /** Y coordinate in viewport pixels (top-left origin) */
+  y: number;
+  /** Label to display in speech bubble */
+  label: string;
+}
+/**
+ * 2D point
+ */
+interface Point {
+  x: number;
+  y: number;
+}
+/**
+ * Screenshot capture result
+ */
+interface ScreenshotResult {
+  /** Base64-encoded image data URL */
+  imageData: string;
+  /** Screenshot image width in pixels (after any downscaling) */
+  width: number;
+  /** Screenshot image height in pixels (after any downscaling) */
+  height: number;
+  /** Live browser viewport width in CSS pixels */
+  viewportWidth: number;
+  /** Live browser viewport height in CSS pixels */
+  viewportHeight: number;
+}
+/**
+ * Annotated screenshot result with marker map.
+ */
+interface AnnotatedScreenshotResult extends ScreenshotResult {
+  /** Map of marker ID to element reference */
+  markerMap: MarkerMap;
+  /** Text description of markers for AI context */
+  markerContext: string;
+}
+/**
+ * Selects where media processing happens.
+ */
+type CursorBuddyMediaMode = "auto" | "browser" | "server";
+/**
+ * Controls how user speech is transcribed before it is sent to the chat model.
+ */
+interface CursorBuddyTranscriptionConfig {
+  /**
+   * Selects where transcription happens.
+   *
+   * - "auto": Try browser speech recognition first, then fall back to server
+   *   transcription if browser recognition is unavailable, fails, or does not
+   *   produce a final transcript.
+   * - "browser": Require browser speech recognition. If it is unavailable or
+   *   fails, the turn errors and no server fallback is attempted.
+   * - "server": Skip browser speech recognition and always use server
+   *   transcription.
+   *
+   * @default "auto"
+   */
+  mode?: CursorBuddyMediaMode;
+}
+/**
+ * Controls how assistant speech is synthesized before it is played back.
+ */
+interface CursorBuddySpeechConfig {
+  /**
+   * Selects where speech synthesis happens.
+   *
+   * - "auto": Try browser speech synthesis first, then fall back to server
+   *   synthesis if browser speech is unavailable or fails.
+   * - "browser": Require browser speech synthesis. If it is unavailable or
+   *   fails, the turn errors and no server fallback is attempted.
+   * - "server": Skip browser speech synthesis and always use server TTS.
+   *
+   * @default "server"
+   */
+  mode?: CursorBuddyMediaMode;
+  /**
+   * Whether speech may start before the full chat response is available.
+   *
+   * When enabled, completed sentence segments are spoken as soon as they are
+   * ready. When disabled, speech waits for the full chat response first.
+   *
+   * @default false
+   */
+  allowStreaming?: boolean;
+}
+/**
+ * Public contract for voice capture used by the core client.
+ */
+interface VoiceCapturePort {
+  start(): Promise<void>;
+  stop(): Promise<Blob>;
+  onLevel(callback: (level: number) => void): void;
+  dispose(): void;
+}
+/**
+ * Public contract for audio playback used by the core client.
+ */
+interface AudioPlaybackPort {
+  play(blob: Blob, signal?: AbortSignal): Promise<void>;
+  stop(): void;
+}
+/**
+ * Public contract for browser-side live transcription.
+ */
+interface LiveTranscriptionPort {
+  isAvailable(): boolean;
+  start(): Promise<void>;
+  stop(): Promise<string>;
+  onPartial(callback: (text: string) => void): void;
+  dispose(): void;
+}
+/**
+ * Public contract for browser-side speech synthesis.
+ */
+interface BrowserSpeechPort {
+  isAvailable(): boolean;
+  speak(text: string, signal?: AbortSignal): Promise<void>;
+  stop(): void;
+}
+/**
+ * Public contract for screenshot capture used by the core client.
+ */
+interface ScreenCapturePort {
+  capture(): Promise<ScreenshotResult>;
+  captureAnnotated(): Promise<AnnotatedScreenshotResult>;
+}
+/**
+ * Public contract for pointer control used by the core client.
+ */
+interface PointerControllerPort {
+  pointAt(target: PointingTarget): void;
+  release(): void;
+  isPointing(): boolean;
+  subscribe(listener: () => void): () => void;
+  updateFollowPosition(): void;
+}
+/**
+ * Internal services interface for dependency injection.
+ */
+interface CursorBuddyServices {
+  voiceCapture?: VoiceCapturePort;
+  audioPlayback?: AudioPlaybackPort;
+  liveTranscription?: LiveTranscriptionPort;
+  browserSpeech?: BrowserSpeechPort;
+  screenCapture?: ScreenCapturePort;
+  pointerController?: PointerControllerPort;
+}
+/**
+ * Cursor render props passed to custom cursor components
+ */
+interface CursorRenderProps {
+  /** Current voice state */
+  state: VoiceState;
+  /** Whether cursor is currently engaged with a pointing target */
+  isPointing: boolean;
+  /** Rotation in radians (direction of travel during pointing) */
+  rotation: number;
+  /** Scale factor (1.0 normal, up to 1.3 during flight) */
+  scale: number;
+}
+/**
+ * Speech bubble render props
+ */
+interface SpeechBubbleRenderProps {
+  /** Text to display */
+  text: string;
+  /** Whether bubble is visible */
+  isVisible: boolean;
+  /** Called when the bubble should be dismissed */
+  onClick?: () => void;
+}
+/**
+ * Waveform render props
+ */
+interface WaveformRenderProps {
+  /** Current audio level (0-1) */
+  audioLevel: number;
+  /** Whether currently listening */
+  isListening: boolean;
+}
+/**
+ * Configuration options for CursorBuddyClient
+ */
+interface CursorBuddyClientOptions {
+  /**
+   * Transcription configuration.
+   *
+   * If omitted, Cursor Buddy uses `{ mode: "auto" }`.
+   */
+  transcription?: CursorBuddyTranscriptionConfig;
+  /**
+   * Speech configuration.
+   *
+   * If omitted, Cursor Buddy uses
+   * `{ mode: "server", allowStreaming: false }`.
+   */
+  speech?: CursorBuddySpeechConfig;
+  /** Callback when transcript is ready */
+  onTranscript?: (text: string) => void;
+  /** Callback when AI responds */
+  onResponse?: (text: string) => void;
+  /** Callback when pointing at element */
+  onPoint?: (target: PointingTarget) => void;
+  /** Callback when state changes */
+  onStateChange?: (state: VoiceState) => void;
+  /** Callback when error occurs */
+  onError?: (error: Error) => void;
+}
+/**
+ * Client snapshot for React's useSyncExternalStore
+ */
+interface CursorBuddySnapshot {
+  /** Current voice state */
+  state: VoiceState;
+  /**
+   * In-progress transcript while the user is speaking.
+   * Populated only when browser transcription is active.
+   */
+  liveTranscript: string;
+  /** Latest transcribed user speech */
+  transcript: string;
+  /** Latest AI response (stripped of POINT tags) */
+  response: string;
+  /** Current error (null if none) */
+  error: Error | null;
+  /** Whether currently engaged with a pointing target */
+  isPointing: boolean;
+  /** Whether the buddy is enabled */
+  isEnabled: boolean;
+}
+//#endregion
+//#region src/core/client.d.ts
+/**
+ * Framework-agnostic client for cursor buddy voice interactions.
+ *
+ * Manages the complete voice interaction flow:
+ * idle -> listening -> processing -> responding -> idle
+ *
+ * Supports interruption: pressing hotkey during any state aborts
+ * in-flight work and immediately transitions to listening.
+ */
+declare class CursorBuddyClient {
+  private endpoint;
+  private options;
+  private voiceCapture;
+  private audioPlayback;
+  private browserSpeech;
+  private liveTranscription;
+  private screenCapture;
+  private pointerController;
+  private stateMachine;
+  private liveTranscript;
+  private transcript;
+  private response;
+  private error;
+  private abortController;
+  private historyCommittedForTurn;
+  private speechProviderForTurn;
+  private cachedSnapshot;
+  private listeners;
+  constructor(endpoint: string, options?: CursorBuddyClientOptions, services?: CursorBuddyServices);
+  /**
+   * Start listening for voice input.
+   * Aborts any in-flight work from previous session.
+   */
+  startListening(): void;
+  /**
+   * Stop listening and process the voice input.
+   */
+  stopListening(): Promise<void>;
+  /**
+   * Enable or disable the buddy.
+   */
+  setEnabled(enabled: boolean): void;
+  /**
+   * Manually point at coordinates.
+   */
+  pointAt(x: number, y: number, label: string): void;
+  /**
+   * Dismiss the current pointing target.
+   */
+  dismissPointing(): void;
+  /**
+   * Reset to idle state and stop any in-progress work.
+   */
+  reset(): void;
+  /**
+   * Update buddy position to follow cursor.
+   * Call this on cursor position changes.
+   */
+  updateCursorPosition(): void;
+  /**
+   * Subscribe to state changes.
+   */
+  subscribe(listener: () => void): () => void;
+  /**
+   * Get current state snapshot for React's useSyncExternalStore.
+   * Returns a cached object to ensure referential stability.
+   */
+  getSnapshot(): CursorBuddySnapshot;
+  /**
+   * Build a new snapshot object.
+   */
+  private buildSnapshot;
+  private abort;
+  /**
+   * Commit partial turn to history when interrupted.
+   * Only commits if we have both transcript and response,
+   * and haven't already committed for this turn.
+   */
+  private commitPartialHistory;
+  private transcribe;
+  /**
+   * Stream the chat response, keep the visible text updated, and feed complete
+   * speech segments into the TTS queue as soon as they are ready.
+   */
+  private chatAndSpeak;
+  /**
+   * Request server-side TTS audio for one text segment.
+   */
+  private synthesizeSpeech;
+  /**
+   * Resolve the initial speech provider for this turn.
+   *
+   * Decision tree:
+   * 1. In `server` mode, always synthesize on the server.
+   * 2. In `browser` mode, require browser speech support up front.
+   * 3. In `auto` mode, prefer browser speech when available and keep that
+   *    choice cached so later segments stay on the same provider unless a
+   *    browser failure forces a one-way fallback to the server.
+   */
+  private prepareSpeechMode;
+  /**
+   * Prepare a playback task for one text segment.
+   *
+   * The queue calls this eagerly so server synthesis can overlap with the
+   * currently playing segment, but the returned task is still executed in the
+   * original enqueue order.
+   */
+  private prepareSpeechSegment;
+  /**
+   * Synthesize server audio immediately and return a playback task that reuses
+   * the prepared blob later.
+   */
+  private prepareServerSpeechTask;
+  /**
+   * Return a browser playback task for one text segment.
+   */
+  private prepareBrowserSpeechTask;
+  /**
+   * Prepare a playback task for `auto` mode.
+   *
+   * We prefer the browser for low latency, but if browser speech fails for any
+   * segment we permanently switch the remainder of the turn to server TTS so
+   * later segments do not keep retrying the failing browser path.
+   */
+  private prepareAutoSpeechTask;
+  /**
+   * Read the current provider choice for `auto` mode, lazily defaulting to the
+   * browser when supported and the server otherwise.
+   */
+  private getAutoSpeechProvider;
+  private handleError;
+  /**
+   * Resolve the effective transcription mode for the current client.
+   */
+  private getTranscriptionMode;
+  /**
+   * Resolve the effective speech mode for the current client.
+   */
+  private getSpeechMode;
+  /**
+   * Decide whether speech should start before the full chat response is ready.
+   */
+  private isSpeechStreamingEnabled;
+  /**
+   * Decide whether this turn should attempt browser speech recognition.
+   */
+  private shouldAttemptBrowserTranscription;
+  /**
+   * Decide whether browser speech recognition is mandatory for this turn.
+   */
+  private isBrowserTranscriptionRequired;
+  /**
+   * Start the recorder and browser speech recognition together.
+   *
+   * The recorder always runs so we keep waveform updates and preserve a raw
+   * audio backup for server fallback in `auto` mode.
+   */
+  private beginListeningSession;
+  /**
+   * Stop browser speech recognition and return the best final transcript it
+   * produced for this turn.
+   */
+  private stopLiveTranscription;
+  /**
+   * Choose the transcript that should drive the turn.
+   *
+   * Decision tree:
+   * 1. Use the browser transcript when it is available.
+   * 2. In browser-only mode, fail if the browser produced nothing usable.
+   * 3. In auto/server modes, fall back to the recorded audio upload.
+   */
+  private resolveTranscript;
+  private updateResponse;
+  private notify;
+}
+//#endregion
+export { CursorBuddySnapshot as a, CursorRenderProps as c, SpeechBubbleRenderProps as d, VoiceEvent as f, CursorBuddyMediaMode as i, Point as l, WaveformRenderProps as m, BrowserSpeechPort as n, CursorBuddySpeechConfig as o, VoiceState as p, CursorBuddyClientOptions as r, CursorBuddyTranscriptionConfig as s, CursorBuddyClient as t, PointingTarget as u };
+//# sourceMappingURL=client-DJRU6dKB.d.mts.map

package/dist/client-DJRU6dKB.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client-DJRU6dKB.d.mts","names":[],"sources":["../src/core/utils/elements.ts","../src/core/types.ts","../src/core/client.ts"],"mappings":";;AAgEA;;;;;;UAAiB,aAAA;EAMf;EAJA,EAAA;EAMA;EAJA,OAAA,EAAS,OAAA;EAIE;EAFX,IAAA,EAAM,OAAA;EAQa;EANnB,WAAA;AAAA;;;;KAMU,SAAA,GAAY,GAAA,SAAY,aAAA;;;;AAdpC;;KC7DY,UAAA;;;;KAKA,UAAA;EACN,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;EAAe,KAAA,EAAO,KAAA;AAAA;;;AAV5B;;;;UAkBiB,cAAA;EAbL;EAeV,CAAA;;EAEA,CAAA;EAhBI;EAkBJ,KAAA;AAAA;;;AAaF;UAAiB,KAAA;EACf,CAAA;EACA,CAAA;AAAA;AAMF;;;AAAA,UAAiB,gBAAA;EAEf;EAAA,SAAA;EAIA;EAFA,KAAA;EAMA;EAJA,MAAA;EAIc;EAFd,aAAA;EAWyC;EATzC,cAAA;AAAA;;;;UASe,yBAAA,SAAkC,gBAAA;EAIpC;EAFb,SAAA,EAFyC,SAAA;EAkBX;EAd9B,aAAA;AAAA;;;;KAcU,oBAAA;;;;UAKK,8BAAA;EAgDA;;;;;;;;;;;;;EAlCf,IAAA,GAAO,oBAAA;AAAA;;;;UAMQ,uBAAA;EAsCA;;;;;;;;;;;EA1Bf,IAAA,GAAO,oBAAA;EA2BU;;;;;AAOnB;;;EAxBE,cAAA;AAAA;;;;UAMe,gBAAA;EACf,KAAA,IAAS,OAAA;EACT,IAAA,IAAQ,OAAA,CAAQ,IAAA;EAChB,OAAA,CAAQ,QAAA,GAAW,KAAA;EACnB,OAAA;AAAA;;AAyBF;;UAnBiB,iBAAA;EACf,IAAA,CAAK,IAAA,EAAM,IAAA,EAAM,MAAA,GAAS,WAAA,GAAc,OAAA;EACxC,IAAA;AAAA;;;;UAMe,qBAAA;EACf,WAAA;EACA,KAAA,IAAS,OAAA;EACT,IAAA,IAAQ,OAAA;EACR,SAAA,CAAU,QAAA,GAAW,IAAA;EACrB,OAAA;AAAA;;;;UAMe,iBAAA;EACf,WAAA;EACA,KAAA,CAAM,IAAA,UAAc,MAAA,GAAS,WAAA,GAAc,OAAA;EAC3C,IAAA;AAAA;;;;UAMe,iBAAA;EACf,OAAA,IAAW,OAAA,CAAQ,gBAAA;EACnB,gBAAA,IAAoB,OAAA,CAAQ,yBAAA;AAAA;;;;UAMb,qBAAA;EACf,OAAA,CAAQ,MAAA,EAAQ,cAAA;EAChB,OAAA;EACA,UAAA;EACA,SAAA,CAAU,QAAA;EACV,oBAAA;AAAA;;;;UAMe,mBAAA;EACf,YAAA,GAAe,gBAAA;EACf,aAAA,GAAgB,iBAAA;EAChB,iBAAA,GAAoB,qBAAA;EACpB,aAAA,GAAgB,iBAAA;EAChB,aAAA,GAAgB,iBAAA;EAChB,iBAAA,GAAoB,qBAAA;AAAA;;;;UAML,iBAAA;EAXA;EAaf,KAAA,EAAO,UAAA;EAZS;EAchB,UAAA;EAboB;EAepB,QAAA;EAdgB;EAgBhB,KAAA;AAAA;;;;UAMe,uBAAA;EAdA;EAgBf,IAAA;;EAEA,SAAA;EAhBA;EAkBA,OAAA;AAAA;;;;UAMe,mBAAA;EAZA;EAcf,UAAA;;EAEA,WAAA;AAAA;;;;UAMe,wBAAA;EAVA;;;;;EAgBf,aAAA,GAAgB,8BAAA;EANuB;;;;;;EAavC,MAAA,GAAS,uBAAA;EAUc;EARvB,YAAA,IAAgB,IAAA;EAThB;EAWA,UAAA,IAAc,IAAA;EAJd;EAMA,OAAA,IAAW,MAAA,EAAQ,cAAA;EAJnB;EAMA,aAAA,IAAiB,KAAA,EAAO,UAAA;EAJxB;EAMA,OAAA,IAAW,KAAA,EAAO,KAAA;AAAA;;;;UAMH,mBAAA;EARE;EAUjB,KAAA,EAAO,UAAA;EARW;;;;EAalB,cAAA;EAPkC;EASlC,UAAA;EAIY;EAFZ,QAAA;EATO;EAWP,KAAA,EAAO,KAAA;EAJP;EAMA,UAAA;EAFA;EAIA,SAAA;AAAA;;;;;;;;;;;;cC9LW,iBAAA;EAAA,QACH,QAAA;EAAA,QACA,OAAA;EAAA,QAGA,YAAA;EAAA,QACA,aAAA;EAAA,QACA,aAAA;EAAA,QACA,iBAAA;EAAA,QACA,aAAA;EAAA,QACA,iBAAA;EAAA,QACA,YAAA;EAAA,QAGA,cAAA;EAAA,QACA,UAAA;EAAA,QACA,QAAA;EAAA,QACA,KAAA;EAAA,QACA,eAAA;EAAA,QACA,uBAAA;EAAA,QACA,qBAAA;EAAA,QAGA,cAAA;EAAA,QAGA,SAAA;cAGN,QAAA,UACA,OAAA,GAAS,wBAAA,EACT,QAAA,GAAU,mBAAA;EDxHR;;;;ECmKJ,cAAA,CAAA;EDhK0B;;;ECiMpB,aAAA,CAAA,GAAiB,OAAA;EDzLM;;;ECiT7B,UAAA,CAAW,OAAA;ED7SX;;;ECqTA,OAAA,CAAQ,CAAA,UAAW,CAAA,UAAW,KAAA;EDtSf;;;EC6Sf,eAAA,CAAA;ED3SC;AAMH;;EC4SE,KAAA,CAAA;ED5S+B;;;;EC4T/B,oBAAA,CAAA;EDlTA;;;ECyTA,SAAA,CAAU,QAAA;EDhT+B;;;;ECyTzC,WAAA,CAAA,GAAe,mBAAA;EDvTN;;;EAAA,QC8TD,aAAA;EAAA,QAcA,KAAA;ED5TsB;;;;AAKhC;EALgC,QCgVtB,oBAAA;EAAA,QAcM,UAAA;ED3Ud;;AAMF;;EANE,QCiWc,YAAA;ED/Ua;;;EAAA,QCsbb,gBAAA;ED5aA;;AAMhB;;;;;;;;EANgB,QCwcN,iBAAA;EDjcC;;;;;;;EAAA,QC8dK,oBAAA;ED3dP;;AAMT;;EANS,QC6eO,uBAAA;EDteH;;;EAAA,QCkfG,wBAAA;EDlfiC;;;;;;;EAAA,QCggBjC,qBAAA;ED/fV;;AAMN;;EANM,QCwiBI,qBAAA;EAAA,QAYA,WAAA;ED7iBR;;;EAAA,QCwjBQ,oBAAA;EDtjBA;;;EAAA,QC6jBA,aAAA;ED3jBR;;;EAAA,QCkkBQ,wBAAA;ED5jBwB;;;EAAA,QCmkBxB,iCAAA;EDjkBR;;;EAAA,QCwkBQ,8BAAA;EDxkBmC;;;;AAO7C;;EAP6C,QCklB7B,qBAAA;ED1kBK;;;;EAAA,QCwnBL,qBAAA;EDvnBa;;;;;;;;EAAA,QCopBb,iBAAA;EAAA,QAmBN,cAAA;EAAA,QAOA,MAAA;AAAA"}