cursor-buddy 0.0.3 → 0.0.4

package/dist/{client-CPQnk2_x.d.mts → client-DJRU6dKB.d.mts}

@@ -34,11 +34,7 @@ type VoiceEvent = {
 } | {
   type: "HOTKEY_RELEASED";
 } | {
-  type: "TRANSCRIPTION_COMPLETE";
-  transcript: string;
-} | {
-  type: "AI_RESPONSE_COMPLETE";
-  response: string;
+  type: "RESPONSE_STARTED";
 } | {
   type: "TTS_COMPLETE";
 } | {
@@ -90,6 +86,55 @@ interface AnnotatedScreenshotResult extends ScreenshotResult {
   /** Text description of markers for AI context */
   markerContext: string;
 }
+/**
+ * Selects where media processing happens.
+ */
+type CursorBuddyMediaMode = "auto" | "browser" | "server";
+/**
+ * Controls how user speech is transcribed before it is sent to the chat model.
+ */
+interface CursorBuddyTranscriptionConfig {
+  /**
+   * Selects where transcription happens.
+   *
+   * - "auto": Try browser speech recognition first, then fall back to server
+   *   transcription if browser recognition is unavailable, fails, or does not
+   *   produce a final transcript.
+   * - "browser": Require browser speech recognition. If it is unavailable or
+   *   fails, the turn errors and no server fallback is attempted.
+   * - "server": Skip browser speech recognition and always use server
+   *   transcription.
+   *
+   * @default "auto"
+   */
+  mode?: CursorBuddyMediaMode;
+}
+/**
+ * Controls how assistant speech is synthesized before it is played back.
+ */
+interface CursorBuddySpeechConfig {
+  /**
+   * Selects where speech synthesis happens.
+   *
+   * - "auto": Try browser speech synthesis first, then fall back to server
+   *   synthesis if browser speech is unavailable or fails.
+   * - "browser": Require browser speech synthesis. If it is unavailable or
+   *   fails, the turn errors and no server fallback is attempted.
+   * - "server": Skip browser speech synthesis and always use server TTS.
+   *
+   * @default "server"
+   */
+  mode?: CursorBuddyMediaMode;
+  /**
+   * Whether speech may start before the full chat response is available.
+   *
+   * When enabled, completed sentence segments are spoken as soon as they are
+   * ready. When disabled, speech waits for the full chat response first.
+   *
+   * @default false
+   */
+  allowStreaming?: boolean;
+}
 /**
  * Public contract for voice capture used by the core client.
  */
@@ -106,6 +151,24 @@ interface AudioPlaybackPort {
   play(blob: Blob, signal?: AbortSignal): Promise<void>;
   stop(): void;
 }
+/**
+ * Public contract for browser-side live transcription.
+ */
+interface LiveTranscriptionPort {
+  isAvailable(): boolean;
+  start(): Promise<void>;
+  stop(): Promise<string>;
+  onPartial(callback: (text: string) => void): void;
+  dispose(): void;
+}
+/**
+ * Public contract for browser-side speech synthesis.
+ */
+interface BrowserSpeechPort {
+  isAvailable(): boolean;
+  speak(text: string, signal?: AbortSignal): Promise<void>;
+  stop(): void;
+}
 /**
  * Public contract for screenshot capture used by the core client.
  */
@@ -129,6 +192,8 @@ interface PointerControllerPort {
 interface CursorBuddyServices {
   voiceCapture?: VoiceCapturePort;
   audioPlayback?: AudioPlaybackPort;
+  liveTranscription?: LiveTranscriptionPort;
+  browserSpeech?: BrowserSpeechPort;
   screenCapture?: ScreenCapturePort;
   pointerController?: PointerControllerPort;
 }
@@ -169,6 +234,19 @@ interface WaveformRenderProps {
  * Configuration options for CursorBuddyClient
  */
 interface CursorBuddyClientOptions {
+  /**
+   * Transcription configuration.
+   *
+   * If omitted, Cursor Buddy uses `{ mode: "auto" }`.
+   */
+  transcription?: CursorBuddyTranscriptionConfig;
+  /**
+   * Speech configuration.
+   *
+   * If omitted, Cursor Buddy uses
+   * `{ mode: "server", allowStreaming: false }`.
+   */
+  speech?: CursorBuddySpeechConfig;
   /** Callback when transcript is ready */
   onTranscript?: (text: string) => void;
   /** Callback when AI responds */
@@ -186,6 +264,11 @@ interface CursorBuddyClientOptions {
 interface CursorBuddySnapshot {
   /** Current voice state */
   state: VoiceState;
+  /**
+   * In-progress transcript while the user is speaking.
+   * Populated only when browser transcription is active.
+   */
+  liveTranscript: string;
   /** Latest transcribed user speech */
   transcript: string;
   /** Latest AI response (stripped of POINT tags) */
@@ -213,14 +296,18 @@ declare class CursorBuddyClient {
   private options;
   private voiceCapture;
   private audioPlayback;
+  private browserSpeech;
+  private liveTranscription;
   private screenCapture;
   private pointerController;
   private stateMachine;
+  private liveTranscript;
   private transcript;
   private response;
   private error;
   private abortController;
   private historyCommittedForTurn;
+  private speechProviderForTurn;
   private cachedSnapshot;
   private listeners;
   constructor(endpoint: string, options?: CursorBuddyClientOptions, services?: CursorBuddyServices);
@@ -275,11 +362,101 @@ declare class CursorBuddyClient {
    */
   private commitPartialHistory;
   private transcribe;
-  private chat;
-  private speak;
+  /**
+   * Stream the chat response, keep the visible text updated, and feed complete
+   * speech segments into the TTS queue as soon as they are ready.
+   */
+  private chatAndSpeak;
+  /**
+   * Request server-side TTS audio for one text segment.
+   */
+  private synthesizeSpeech;
+  /**
+   * Resolve the initial speech provider for this turn.
+   *
+   * Decision tree:
+   * 1. In `server` mode, always synthesize on the server.
+   * 2. In `browser` mode, require browser speech support up front.
+   * 3. In `auto` mode, prefer browser speech when available and keep that
+   *    choice cached so later segments stay on the same provider unless a
+   *    browser failure forces a one-way fallback to the server.
+   */
+  private prepareSpeechMode;
+  /**
+   * Prepare a playback task for one text segment.
+   *
+   * The queue calls this eagerly so server synthesis can overlap with the
+   * currently playing segment, but the returned task is still executed in the
+   * original enqueue order.
+   */
+  private prepareSpeechSegment;
+  /**
+   * Synthesize server audio immediately and return a playback task that reuses
+   * the prepared blob later.
+   */
+  private prepareServerSpeechTask;
+  /**
+   * Return a browser playback task for one text segment.
+   */
+  private prepareBrowserSpeechTask;
+  /**
+   * Prepare a playback task for `auto` mode.
+   *
+   * We prefer the browser for low latency, but if browser speech fails for any
+   * segment we permanently switch the remainder of the turn to server TTS so
+   * later segments do not keep retrying the failing browser path.
+   */
+  private prepareAutoSpeechTask;
+  /**
+   * Read the current provider choice for `auto` mode, lazily defaulting to the
+   * browser when supported and the server otherwise.
+   */
+  private getAutoSpeechProvider;
   private handleError;
+  /**
+   * Resolve the effective transcription mode for the current client.
+   */
+  private getTranscriptionMode;
+  /**
+   * Resolve the effective speech mode for the current client.
+   */
+  private getSpeechMode;
+  /**
+   * Decide whether speech should start before the full chat response is ready.
+   */
+  private isSpeechStreamingEnabled;
+  /**
+   * Decide whether this turn should attempt browser speech recognition.
+   */
+  private shouldAttemptBrowserTranscription;
+  /**
+   * Decide whether browser speech recognition is mandatory for this turn.
+   */
+  private isBrowserTranscriptionRequired;
+  /**
+   * Start the recorder and browser speech recognition together.
+   *
+   * The recorder always runs so we keep waveform updates and preserve a raw
+   * audio backup for server fallback in `auto` mode.
+   */
+  private beginListeningSession;
+  /**
+   * Stop browser speech recognition and return the best final transcript it
+   * produced for this turn.
+   */
+  private stopLiveTranscription;
+  /**
+   * Choose the transcript that should drive the turn.
+   *
+   * Decision tree:
+   * 1. Use the browser transcript when it is available.
+   * 2. In browser-only mode, fail if the browser produced nothing usable.
+   * 3. In auto/server modes, fall back to the recorded audio upload.
+   */
+  private resolveTranscript;
+  private updateResponse;
   private notify;
 }
 //#endregion
-export { Point as a, VoiceEvent as c, CursorRenderProps as i, VoiceState as l, CursorBuddyClientOptions as n, PointingTarget as o, CursorBuddySnapshot as r, SpeechBubbleRenderProps as s, CursorBuddyClient as t, WaveformRenderProps as u };
-//# sourceMappingURL=client-CPQnk2_x.d.mts.map
+export { CursorBuddySnapshot as a, CursorRenderProps as c, SpeechBubbleRenderProps as d, VoiceEvent as f, CursorBuddyMediaMode as i, Point as l, WaveformRenderProps as m, BrowserSpeechPort as n, CursorBuddySpeechConfig as o, VoiceState as p, CursorBuddyClientOptions as r, CursorBuddyTranscriptionConfig as s, CursorBuddyClient as t, PointingTarget as u };
+//# sourceMappingURL=client-DJRU6dKB.d.mts.map

package/dist/client-DJRU6dKB.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client-DJRU6dKB.d.mts","names":[],"sources":["../src/core/utils/elements.ts","../src/core/types.ts","../src/core/client.ts"],"mappings":";;AAgEA;;;;;;UAAiB,aAAA;EAMf;EAJA,EAAA;EAMA;EAJA,OAAA,EAAS,OAAA;EAIE;EAFX,IAAA,EAAM,OAAA;EAQa;EANnB,WAAA;AAAA;;;;KAMU,SAAA,GAAY,GAAA,SAAY,aAAA;;;;AAdpC;;KC7DY,UAAA;;;;KAKA,UAAA;EACN,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;EAAe,KAAA,EAAO,KAAA;AAAA;;;AAV5B;;;;UAkBiB,cAAA;EAbL;EAeV,CAAA;;EAEA,CAAA;EAhBI;EAkBJ,KAAA;AAAA;;;AAaF;UAAiB,KAAA;EACf,CAAA;EACA,CAAA;AAAA;AAMF;;;AAAA,UAAiB,gBAAA;EAEf;EAAA,SAAA;EAIA;EAFA,KAAA;EAMA;EAJA,MAAA;EAIc;EAFd,aAAA;EAWyC;EATzC,cAAA;AAAA;;;;UASe,yBAAA,SAAkC,gBAAA;EAIpC;EAFb,SAAA,EAFyC,SAAA;EAkBX;EAd9B,aAAA;AAAA;;;;KAcU,oBAAA;;;;UAKK,8BAAA;EAgDA;;;;;;;;;;;;;EAlCf,IAAA,GAAO,oBAAA;AAAA;;;;UAMQ,uBAAA;EAsCA;;;;;;;;;;;EA1Bf,IAAA,GAAO,oBAAA;EA2BU;;;;;AAOnB;;;EAxBE,cAAA;AAAA;;;;UAMe,gBAAA;EACf,KAAA,IAAS,OAAA;EACT,IAAA,IAAQ,OAAA,CAAQ,IAAA;EAChB,OAAA,CAAQ,QAAA,GAAW,KAAA;EACnB,OAAA;AAAA;;AAyBF;;UAnBiB,iBAAA;EACf,IAAA,CAAK,IAAA,EAAM,IAAA,EAAM,MAAA,GAAS,WAAA,GAAc,OAAA;EACxC,IAAA;AAAA;;;;UAMe,qBAAA;EACf,WAAA;EACA,KAAA,IAAS,OAAA;EACT,IAAA,IAAQ,OAAA;EACR,SAAA,CAAU,QAAA,GAAW,IAAA;EACrB,OAAA;AAAA;;;;UAMe,iBAAA;EACf,WAAA;EACA,KAAA,CAAM,IAAA,UAAc,MAAA,GAAS,WAAA,GAAc,OAAA;EAC3C,IAAA;AAAA;;;;UAMe,iBAAA;EACf,OAAA,IAAW,OAAA,CAAQ,gBAAA;EACnB,gBAAA,IAAoB,OAAA,CAAQ,yBAAA;AAAA;;;;UAMb,qBAAA;EACf,OAAA,CAAQ,MAAA,EAAQ,cAAA;EAChB,OAAA;EACA,UAAA;EACA,SAAA,CAAU,QAAA;EACV,oBAAA;AAAA;;;;UAMe,mBAAA;EACf,YAAA,GAAe,gBAAA;EACf,aAAA,GAAgB,iBAAA;EAChB,iBAAA,GAAoB,qBAAA;EACpB,aAAA,GAAgB,iBAAA;EAChB,aAAA,GAAgB,iBAAA;EAChB,iBAAA,GAAoB,qBAAA;AAAA;;;;UAML,iBAAA;EAXA;EAaf,KAAA,EAAO,UAAA;EAZS;EAchB,UAAA;EAboB;EAepB,QAAA;EAdgB;EAgBhB,KAAA;AAAA;;;;UAMe,uBAAA;EAdA;EAgBf,IAAA;;EAEA,SAAA;EAhBA;EAkBA,OAAA;AAAA;;;;UAMe,mBAAA;EAZA;EAcf,UAAA;;EAEA,WAAA;AAAA;;;;UAMe,wBAAA;EAVA;;;;;EAgBf,aAAA,GAAgB,8BAAA;EANuB;;;;;;EAavC,MAAA,GAAS,uBAAA;EAUc;EARvB,YAAA,IAAgB,IAAA;EAThB;EAWA,UAAA,IAAc,IAAA;EAJd;EAMA,OAAA,IAAW,MAAA,EAAQ,cAAA;EAJnB;EAMA,aAAA,IAAiB,KAAA,EAAO,UAAA;EAJxB;EAMA,OAAA,IAAW,KAAA,EAAO,KAAA;AAAA;;;;UAMH,mBAAA;EARE;EAUjB,KAAA,EAAO,UAAA;EARW;;;;EAalB,cAAA;EAPkC;EASlC,UAAA;EAIY;EAFZ,QAAA;EATO;EAWP,KAAA,EAAO,KAAA;EAJP;EAMA,UAAA;EAFA;EAIA,SAAA;AAAA;;;;;;;;;;;;cC9LW,iBAAA;EAAA,QACH,QAAA;EAAA,QACA,OAAA;EAAA,QAGA,YAAA;EAAA,QACA,aAAA;EAAA,QACA,aAAA;EAAA,QACA,iBAAA;EAAA,QACA,aAAA;EAAA,QACA,iBAAA;EAAA,QACA,YAAA;EAAA,QAGA,cAAA;EAAA,QACA,UAAA;EAAA,QACA,QAAA;EAAA,QACA,KAAA;EAAA,QACA,eAAA;EAAA,QACA,uBAAA;EAAA,QACA,qBAAA;EAAA,QAGA,cAAA;EAAA,QAGA,SAAA;cAGN,QAAA,UACA,OAAA,GAAS,wBAAA,EACT,QAAA,GAAU,mBAAA;EDxHR;;;;ECmKJ,cAAA,CAAA;EDhK0B;;;ECiMpB,aAAA,CAAA,GAAiB,OAAA;EDzLM;;;ECiT7B,UAAA,CAAW,OAAA;ED7SX;;;ECqTA,OAAA,CAAQ,CAAA,UAAW,CAAA,UAAW,KAAA;EDtSf;;;EC6Sf,eAAA,CAAA;ED3SC;AAMH;;EC4SE,KAAA,CAAA;ED5S+B;;;;EC4T/B,oBAAA,CAAA;EDlTA;;;ECyTA,SAAA,CAAU,QAAA;EDhT+B;;;;ECyTzC,WAAA,CAAA,GAAe,mBAAA;EDvTN;;;EAAA,QC8TD,aAAA;EAAA,QAcA,KAAA;ED5TsB;;;;AAKhC;EALgC,QCgVtB,oBAAA;EAAA,QAcM,UAAA;ED3Ud;;AAMF;;EANE,QCiWc,YAAA;ED/Ua;;;EAAA,QCsbb,gBAAA;ED5aA;;AAMhB;;;;;;;;EANgB,QCwcN,iBAAA;EDjcC;;;;;;;EAAA,QC8dK,oBAAA;ED3dP;;AAMT;;EANS,QC6eO,uBAAA;EDteH;;;EAAA,QCkfG,wBAAA;EDlfiC;;;;;;;EAAA,QCggBjC,qBAAA;ED/fV;;AAMN;;EANM,QCwiBI,qBAAA;EAAA,QAYA,WAAA;ED7iBR;;;EAAA,QCwjBQ,oBAAA;EDtjBA;;;EAAA,QC6jBA,aAAA;ED3jBR;;;EAAA,QCkkBQ,wBAAA;ED5jBwB;;;EAAA,QCmkBxB,iCAAA;EDjkBR;;;EAAA,QCwkBQ,8BAAA;EDxkBmC;;;;AAO7C;;EAP6C,QCklB7B,qBAAA;ED1kBK;;;;EAAA,QCwnBL,qBAAA;EDvnBa;;;;;;;;EAAA,QCopBb,iBAAA;EAAA,QAmBN,cAAA;EAAA,QAOA,MAAA;AAAA"}