cursor-buddy 0.0.1 → 0.0.3

package/dist/client-CPQnk2_x.d.mts

@@ -0,0 +1,285 @@
+//#region src/core/utils/elements.d.ts
+/**
+ * Element discovery for annotated screenshots.
+ * Finds visible interactive elements and assigns marker IDs.
+ */
+/**
+ * Element marker with reference to actual DOM element.
+ */
+interface ElementMarker {
+  /** Sequential marker ID (1, 2, 3...) */
+  id: number;
+  /** Reference to the actual DOM element */
+  element: Element;
+  /** Bounding rect at time of capture */
+  rect: DOMRect;
+  /** Brief description for AI context */
+  description: string;
+}
+/**
+ * Map of marker ID to element marker.
+ */
+type MarkerMap = Map<number, ElementMarker>;
+//#endregion
+//#region src/core/types.d.ts
+/**
+ * Voice state machine states
+ */
+type VoiceState = "idle" | "listening" | "processing" | "responding";
+/**
+ * Events for the voice state machine
+ */
+type VoiceEvent = {
+  type: "HOTKEY_PRESSED";
+} | {
+  type: "HOTKEY_RELEASED";
+} | {
+  type: "TRANSCRIPTION_COMPLETE";
+  transcript: string;
+} | {
+  type: "AI_RESPONSE_COMPLETE";
+  response: string;
+} | {
+  type: "TTS_COMPLETE";
+} | {
+  type: "ERROR";
+  error: Error;
+};
+/**
+ * Point coordinates parsed from AI response.
+ * Supports two formats:
+ * - Marker-based: [POINT:5:label] - references a numbered marker
+ * - Coordinate-based: [POINT:640,360:label] - raw pixel coordinates
+ */
+interface PointingTarget {
+  /** X coordinate in viewport pixels (top-left origin) */
+  x: number;
+  /** Y coordinate in viewport pixels (top-left origin) */
+  y: number;
+  /** Label to display in speech bubble */
+  label: string;
+}
+/**
+ * 2D point
+ */
+interface Point {
+  x: number;
+  y: number;
+}
+/**
+ * Screenshot capture result
+ */
+interface ScreenshotResult {
+  /** Base64-encoded image data URL */
+  imageData: string;
+  /** Screenshot image width in pixels (after any downscaling) */
+  width: number;
+  /** Screenshot image height in pixels (after any downscaling) */
+  height: number;
+  /** Live browser viewport width in CSS pixels */
+  viewportWidth: number;
+  /** Live browser viewport height in CSS pixels */
+  viewportHeight: number;
+}
+/**
+ * Annotated screenshot result with marker map.
+ */
+interface AnnotatedScreenshotResult extends ScreenshotResult {
+  /** Map of marker ID to element reference */
+  markerMap: MarkerMap;
+  /** Text description of markers for AI context */
+  markerContext: string;
+}
+/**
+ * Public contract for voice capture used by the core client.
+ */
+interface VoiceCapturePort {
+  start(): Promise<void>;
+  stop(): Promise<Blob>;
+  onLevel(callback: (level: number) => void): void;
+  dispose(): void;
+}
+/**
+ * Public contract for audio playback used by the core client.
+ */
+interface AudioPlaybackPort {
+  play(blob: Blob, signal?: AbortSignal): Promise<void>;
+  stop(): void;
+}
+/**
+ * Public contract for screenshot capture used by the core client.
+ */
+interface ScreenCapturePort {
+  capture(): Promise<ScreenshotResult>;
+  captureAnnotated(): Promise<AnnotatedScreenshotResult>;
+}
+/**
+ * Public contract for pointer control used by the core client.
+ */
+interface PointerControllerPort {
+  pointAt(target: PointingTarget): void;
+  release(): void;
+  isPointing(): boolean;
+  subscribe(listener: () => void): () => void;
+  updateFollowPosition(): void;
+}
+/**
+ * Internal services interface for dependency injection.
+ */
+interface CursorBuddyServices {
+  voiceCapture?: VoiceCapturePort;
+  audioPlayback?: AudioPlaybackPort;
+  screenCapture?: ScreenCapturePort;
+  pointerController?: PointerControllerPort;
+}
+/**
+ * Cursor render props passed to custom cursor components
+ */
+interface CursorRenderProps {
+  /** Current voice state */
+  state: VoiceState;
+  /** Whether cursor is currently engaged with a pointing target */
+  isPointing: boolean;
+  /** Rotation in radians (direction of travel during pointing) */
+  rotation: number;
+  /** Scale factor (1.0 normal, up to 1.3 during flight) */
+  scale: number;
+}
+/**
+ * Speech bubble render props
+ */
+interface SpeechBubbleRenderProps {
+  /** Text to display */
+  text: string;
+  /** Whether bubble is visible */
+  isVisible: boolean;
+  /** Called when the bubble should be dismissed */
+  onClick?: () => void;
+}
+/**
+ * Waveform render props
+ */
+interface WaveformRenderProps {
+  /** Current audio level (0-1) */
+  audioLevel: number;
+  /** Whether currently listening */
+  isListening: boolean;
+}
+/**
+ * Configuration options for CursorBuddyClient
+ */
+interface CursorBuddyClientOptions {
+  /** Callback when transcript is ready */
+  onTranscript?: (text: string) => void;
+  /** Callback when AI responds */
+  onResponse?: (text: string) => void;
+  /** Callback when pointing at element */
+  onPoint?: (target: PointingTarget) => void;
+  /** Callback when state changes */
+  onStateChange?: (state: VoiceState) => void;
+  /** Callback when error occurs */
+  onError?: (error: Error) => void;
+}
+/**
+ * Client snapshot for React's useSyncExternalStore
+ */
+interface CursorBuddySnapshot {
+  /** Current voice state */
+  state: VoiceState;
+  /** Latest transcribed user speech */
+  transcript: string;
+  /** Latest AI response (stripped of POINT tags) */
+  response: string;
+  /** Current error (null if none) */
+  error: Error | null;
+  /** Whether currently engaged with a pointing target */
+  isPointing: boolean;
+  /** Whether the buddy is enabled */
+  isEnabled: boolean;
+}
+//#endregion
+//#region src/core/client.d.ts
+/**
+ * Framework-agnostic client for cursor buddy voice interactions.
+ *
+ * Manages the complete voice interaction flow:
+ * idle -> listening -> processing -> responding -> idle
+ *
+ * Supports interruption: pressing hotkey during any state aborts
+ * in-flight work and immediately transitions to listening.
+ */
+declare class CursorBuddyClient {
+  private endpoint;
+  private options;
+  private voiceCapture;
+  private audioPlayback;
+  private screenCapture;
+  private pointerController;
+  private stateMachine;
+  private transcript;
+  private response;
+  private error;
+  private abortController;
+  private historyCommittedForTurn;
+  private cachedSnapshot;
+  private listeners;
+  constructor(endpoint: string, options?: CursorBuddyClientOptions, services?: CursorBuddyServices);
+  /**
+   * Start listening for voice input.
+   * Aborts any in-flight work from previous session.
+   */
+  startListening(): void;
+  /**
+   * Stop listening and process the voice input.
+   */
+  stopListening(): Promise<void>;
+  /**
+   * Enable or disable the buddy.
+   */
+  setEnabled(enabled: boolean): void;
+  /**
+   * Manually point at coordinates.
+   */
+  pointAt(x: number, y: number, label: string): void;
+  /**
+   * Dismiss the current pointing target.
+   */
+  dismissPointing(): void;
+  /**
+   * Reset to idle state and stop any in-progress work.
+   */
+  reset(): void;
+  /**
+   * Update buddy position to follow cursor.
+   * Call this on cursor position changes.
+   */
+  updateCursorPosition(): void;
+  /**
+   * Subscribe to state changes.
+   */
+  subscribe(listener: () => void): () => void;
+  /**
+   * Get current state snapshot for React's useSyncExternalStore.
+   * Returns a cached object to ensure referential stability.
+   */
+  getSnapshot(): CursorBuddySnapshot;
+  /**
+   * Build a new snapshot object.
+   */
+  private buildSnapshot;
+  private abort;
+  /**
+   * Commit partial turn to history when interrupted.
+   * Only commits if we have both transcript and response,
+   * and haven't already committed for this turn.
+   */
+  private commitPartialHistory;
+  private transcribe;
+  private chat;
+  private speak;
+  private handleError;
+  private notify;
+}
+//#endregion
+export { Point as a, VoiceEvent as c, CursorRenderProps as i, VoiceState as l, CursorBuddyClientOptions as n, PointingTarget as o, CursorBuddySnapshot as r, SpeechBubbleRenderProps as s, CursorBuddyClient as t, WaveformRenderProps as u };
+//# sourceMappingURL=client-CPQnk2_x.d.mts.map

package/dist/client-CPQnk2_x.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client-CPQnk2_x.d.mts","names":[],"sources":["../src/core/utils/elements.ts","../src/core/types.ts","../src/core/client.ts"],"mappings":";;AAgEA;;;;;;UAAiB,aAAA;EAMf;EAJA,EAAA;EAMA;EAJA,OAAA,EAAS,OAAA;EAIE;EAFX,IAAA,EAAM,OAAA;EAQa;EANnB,WAAA;AAAA;;;;KAMU,SAAA,GAAY,GAAA,SAAY,aAAA;;;;AAdpC;;KC7DY,UAAA;;;;KAKA,UAAA;EACN,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;EAAgC,UAAA;AAAA;EAChC,IAAA;EAA8B,QAAA;AAAA;EAC9B,IAAA;AAAA;EACA,IAAA;EAAe,KAAA,EAAO,KAAA;AAAA;;;;AAN5B;;;UAciB,cAAA;EAbX;EAeJ,CAAA;EAbI;EAeJ,CAAA;EAdI;EAgBJ,KAAA;AAAA;;AAaF;;UAAiB,KAAA;EACf,CAAA;EACA,CAAA;AAAA;;;;UAMe,gBAAA;EAIf;EAFA,SAAA;EAMA;EAJA,KAAA;EAMc;EAJd,MAAA;EAae;EAXf,aAAA;;EAEA,cAAA;AAAA;;;;UASe,yBAAA,SAAkC,gBAAA;EAkBlC;EAhBf,SAAA,EAFyC,SAAA;;EAIzC,aAAA;AAAA;;;;UAce,gBAAA;EACf,KAAA,IAAS,OAAA;EACT,IAAA,IAAQ,OAAA,CAAQ,IAAA;EAChB,OAAA,CAAQ,QAAA,GAAW,KAAA;EACnB,OAAA;AAAA;;;;UAMe,iBAAA;EACf,IAAA,CAAK,IAAA,EAAM,IAAA,EAAM,MAAA,GAAS,WAAA,GAAc,OAAA;EACxC,IAAA;AAAA;;;;UAMe,iBAAA;EACf,OAAA,IAAW,OAAA,CAAQ,gBAAA;EACnB,gBAAA,IAAoB,OAAA,CAAQ,yBAAA;AAAA;;AAF9B;;UAQiB,qBAAA;EACf,OAAA,CAAQ,MAAA,EAAQ,cAAA;EAChB,OAAA;EACA,UAAA;EACA,SAAA,CAAU,QAAA;EACV,oBAAA;AAAA;;;;UAMe,mBAAA;EACf,YAAA,GAAe,gBAAA;EACf,aAAA,GAAgB,iBAAA;EAChB,aAAA,GAAgB,iBAAA;EAChB,iBAAA,GAAoB,qBAAA;AAAA;;;;UAML,iBAAA;EApBC;EAsBhB,KAAA,EAAO,UAAA;EArBP;EAuBA,UAAA;EArBA;EAuBA,QAAA;EAtBA;EAwBA,KAAA;AAAA;AAlBF;;;AAAA,UAwBiB,uBAAA;EAtBC;EAwBhB,IAAA;EAtBoB;EAwBpB,SAAA;EAxByC;EA0BzC,OAAA;AAAA;;;;UAMe,mBAAA;EAhCf;EAkCA,UAAA;EAlCyC;EAoCzC,WAAA;AAAA;;;;UAMe,wBAAA;EAlCR;EAoCP,YAAA,IAAgB,IAAA;EAhChB;EAkCA,UAAA,IAAc,IAAA;EAhCT;EAkCL,OAAA,IAAW,MAAA,EAAQ,cAAA;EA5BJ;EA8Bf,aAAA,IAAiB,KAAA,EAAO,UAAA;;EAExB,OAAA,IAAW,KAAA,EAAO,KAAA;AAAA;;;;UAMH,mBAAA;EA1BA;EA4Bf,KAAA,EAAO,UAAA;;EAEP,UAAA;EA1BW;EA4BX,QAAA;EAtBuC;EAwBvC,KAAA,EAAO,KAAA;EAlBY;EAoBnB,UAAA;EAhBkB;EAkBlB,SAAA;AAAA;;;;;;;;;;;;cCpIW,iBAAA;EAAA,QACH,QAAA;EAAA,QACA,OAAA;EAAA,QAGA,YAAA;EAAA,QACA,aAAA;EAAA,QACA,aAAA;EAAA,QACA,iBAAA;EAAA,QACA,YAAA;EAAA,QAGA,UAAA;EAAA,QACA,QAAA;EAAA,QACA,KAAA;EAAA,QACA,eAAA;EAAA,QACA,uBAAA;EAAA,QAGA,cAAA;EAAA,QAGA,SAAA;cAGN,QAAA,UACA,OAAA,GAAS,wBAAA,EACT,QAAA,GAAU,mBAAA;EDpFQ;;;;ECuHpB,cAAA,CAAA;EDpHI;;;EC2IE,aAAA,CAAA,GAAiB,OAAA;EDzInB;;;ECwOJ,UAAA,CAAW,OAAA;EDvOoB;;AAQjC;ECuOE,OAAA,CAAQ,CAAA,UAAW,CAAA,UAAW,KAAA;;;;EAO9B,eAAA,CAAA;EDxOA;;;EC+OA,KAAA,CAAA;EDlOoB;;;;ECiPpB,oBAAA,CAAA;EDzO+B;;;ECgP/B,SAAA,CAAU,QAAA;ED5OV;;;;ECqPA,WAAA,CAAA,GAAe,mBAAA;ED/OD;AAShB;;EATgB,QCsPN,aAAA;EAAA,QAaA,KAAA;ED1PyC;;;;;EAAA,QC0QzC,oBAAA;EAAA,QAcM,UAAA;EAAA,QAkBA,IAAA;EAAA,QAiDA,KAAA;EAAA,QAgBN,WAAA;EAAA,QAOA,MAAA;AAAA"}