cursor-buddy 0.0.8 → 0.0.9

package/README.md

@@ -57,7 +57,7 @@ export const cursorBuddy = createCursorBuddyHandler({
 import { toNextJsHandler } from "cursor-buddy/server/next"
 import { cursorBuddy } from "@/lib/cursor-buddy"
 
-export const { GET, POST } = toNextJsHandler(cursorBuddy)
+export const { POST } = toNextJsHandler(cursorBuddy)
 ```
 
 ### 2. Client Setup
@@ -371,13 +371,13 @@ client.stopListening()
 4. An annotated screenshot of the viewport is captured, with numbered markers on visible interactive elements, based on [agent-browser](https://github.com/vercel-labs/agent-browser) implementation.
 5. The client prefers the browser transcript; if it is unavailable or empty in `auto` mode, the recorded audio is transcribed on the server
 6. Screenshot + marker context are sent to the AI model
-7. AI responds with text, optionally including a pointing tag:
-   - Preferred: `[POINT:5:Submit]` for numbered interactive elements
-   - Fallback: `[POINT:640,360:Error text]` for arbitrary screen coordinates
+7. AI responds with text and can optionally call the `point` tool to indicate a location on screen:
+   - `type: "marker"` with `markerId` for numbered interactive elements (most accurate)
+   - `type: "coordinates"` with `x, y` pixel coordinates for anything without a marker
 8. Response is spoken in the browser or on the server based on `speech.mode`,
    and can either wait for the full response or stream sentence-by-sentence
    based on `speech.allowStreaming`
-9. If a marker tag is present, it is resolved back to the live DOM element; if a coordinate tag is present, it is mapped back to the live viewport; then the cursor animates to the target location
+9. If the AI calls the point tool, the cursor animates to the target location — markers resolve to live DOM elements, coordinates map to viewport positions
 10. **If user presses hotkey again at any point, current response is interrupted**
 
 ## Security Best Practices
@@ -415,7 +415,6 @@ export const GET = POST
 
 ## TODOs
 
-- [ ] High: Make tool calls first class: Pointing becomes tool call (once per turn) + re-use pointing bubble UI for tool calls
 - [ ] Medium: Proper test structure without relying on `as any` for audio and voice capture
 
 ## License

package/dist/{client-Crn8tW7w.d.mts → client-Ba6rv-du.d.mts}

@@ -42,10 +42,7 @@ type VoiceEvent = {
   error: Error;
 };
 /**
- * Point coordinates parsed from AI response.
- * Supports two formats:
- * - Marker-based: [POINT:5:label] - references a numbered marker
- * - Coordinate-based: [POINT:640,360:label] - raw pixel coordinates
+ * Point coordinates for cursor pointing.
  */
 interface PointingTarget {
   /** X coordinate in viewport pixels (top-left origin) */
@@ -460,4 +457,4 @@ declare class CursorBuddyClient {
 }
 //#endregion
 export { CursorBuddySnapshot as a, CursorRenderProps as c, SpeechBubbleRenderProps as d, VoiceEvent as f, CursorBuddyMediaMode as i, Point as l, WaveformRenderProps as m, BrowserSpeechPort as n, CursorBuddySpeechConfig as o, VoiceState as p, CursorBuddyClientOptions as r, CursorBuddyTranscriptionConfig as s, CursorBuddyClient as t, PointingTarget as u };
-//# sourceMappingURL=client-Crn8tW7w.d.mts.map
+//# sourceMappingURL=client-Ba6rv-du.d.mts.map

package/dist/client-Ba6rv-du.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client-Ba6rv-du.d.mts","names":[],"sources":["../src/core/utils/elements.ts","../src/core/types.ts","../src/core/client.ts"],"mappings":";;AAgEA;;;;;;UAAiB,aAAA;EAMf;EAJA,EAAA;EAMA;EAJA,OAAA,EAAS,OAAA;EAIE;EAFX,IAAA,EAAM,OAAA;EAQa;EANnB,WAAA;AAAA;;;;KAMU,SAAA,GAAY,GAAA,SAAY,aAAA;;;;AAdpC;;KC7DY,UAAA;;;;KAKA,UAAA;EACN,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;EAAe,KAAA,EAAO,KAAA;AAAA;;;AAV5B;UAeiB,cAAA;;EAEf,CAAA;EAjBoB;EAmBpB,CAAA;EAdoB;EAgBpB,KAAA;AAAA;;;;UAMe,KAAA;EACf,CAAA;EACA,CAAA;AAAA;;;AAdF;UAoBiB,gBAAA;;EAEf,SAAA;EApBA;EAsBA,KAAA;EAlBA;EAoBA,MAAA;EApBK;EAsBL,aAAA;EAhBoB;EAkBpB,cAAA;AAAA;AAVF;;;AAAA,UAmBiB,yBAAA,SAAkC,gBAAA;EAjBjD;EAmBA,SAAA,EAFyC,SAAA;EAbzC;EAiBA,aAAA;AAAA;;;;KAcU,oBAAA;;AAAZ;;UAKiB,8BAAA;EALe;;AAKhC;;;;;AAoBA;;;;;;EANE,IAAA,GAAO,oBAAA;AAAA;;AAkCT;;UA5BiB,uBAAA;EA6BN;;;;;;;;;;;EAjBT,IAAA,GAAO,oBAAA;EAmBC;;;;AAOV;;;;EAhBE,cAAA;AAAA;;;;UAMe,gBAAA;EACf,KAAA,IAAS,OAAA;EACT,IAAA,IAAQ,OAAA,CAAQ,IAAA;EAChB,OAAA,CAAQ,QAAA,GAAW,KAAA;EACnB,OAAA;AAAA;;;AAcF;UARiB,iBAAA;EACf,IAAA,CAAK,IAAA,EAAM,IAAA,EAAM,MAAA,GAAS,WAAA,GAAc,OAAA;EACxC,IAAA;AAAA;;;;UAMe,qBAAA;EACf,WAAA;EACA,KAAA,IAAS,OAAA;EACT,IAAA,IAAQ,OAAA;EACR,SAAA,CAAU,QAAA,GAAW,IAAA;EACrB,OAAA;AAAA;AAMF;;;AAAA,UAAiB,iBAAA;EACf,WAAA;EACA,KAAA,CAAM,IAAA,UAAc,MAAA,GAAS,WAAA,GAAc,OAAA;EAC3C,IAAA;AAAA;;;;UAMe,iBAAA;EACf,OAAA,IAAW,OAAA,CAAQ,gBAAA;EACnB,gBAAA,IAAoB,OAAA,CAAQ,yBAAA;AAAA;;;;UAMb,qBAAA;EACf,OAAA,CAAQ,MAAA,EAAQ,cAAA;EAChB,OAAA;EACA,UAAA;EACA,SAAA,CAAU,QAAA;EACV,oBAAA;AAAA;;;;UAMe,mBAAA;EACf,YAAA,GAAe,gBAAA;EACf,aAAA,GAAgB,iBAAA;EAChB,iBAAA,GAAoB,qBAAA;EACpB,aAAA,GAAgB,iBAAA;EAChB,aAAA,GAAgB,iBAAA;EAChB,iBAAA,GAAoB,qBAAA;AAAA;;;;UAML,iBAAA;EAnBL;EAqBV,KAAA,EAAO,UAAA;EApBa;EAsBpB,UAAA;EAhBe;EAkBf,QAAA;;EAEA,KAAA;AAAA;;;;UAMe,uBAAA;EApB0B;EAsBzC,IAAA;EA3BA;EA6BA,SAAA;EA5BA;EA8BA,OAAA;AAAA;;;;UAMe,mBAAA;EAjCC;EAmChB,UAAA;EAlCoB;EAoCpB,WAAA;AAAA;AA9BF;;;AAAA,UAoCiB,wBAAA;EAlCf;;;;;EAwCA,aAAA,GAAgB,8BAAA;EAlCX;AAMP;;;;;EAmCE,MAAA,GAAS,uBAAA;EA7BT;EA+BA,YAAA,IAAgB,IAAA;EA/BT;EAiCP,UAAA,IAAc,IAAA;EA3BoB;EA6BlC,OAAA,IAAW,MAAA,EAAQ,cAAA;EA3BnB;EA6BA,aAAA,IAAiB,KAAA,EAAO,UAAA;EArBT;EAuBf,OAAA,IAAW,KAAA,EAAO,KAAA;AAAA;;;;UAMH,mBAAA;EANG;EAQlB,KAAA,EAAO,UAAA;EARgB;;;;EAavB,cAAA;EArBA;EAuBA,UAAA;EArBA;EAuBA,QAAA;EArBA;EAuBA,KAAA,EAAO,KAAA;EAvBI;EAyBX,UAAA;EAvBwB;EAyBxB,SAAA;AAAA;;;;;;;;;;;;cCpLW,iBAAA;EAAA,QACH,QAAA;EAAA,QACA,OAAA;EAAA,QAGA,YAAA;EAAA,QACA,aAAA;EAAA,QACA,aAAA;EAAA,QACA,iBAAA;EAAA,QACA,aAAA;EAAA,QACA,iBAAA;EAAA,QACA,YAAA;EAAA,QAGA,cAAA;EAAA,QACA,UAAA;EAAA,QACA,QAAA;EAAA,QACA,KAAA;EAAA,QACA,eAAA;EAAA,QACA,uBAAA;EAAA,QACA,qBAAA;EAAA,QACA,iBAAA;EAAA,QAGA,cAAA;EAAA,QAGA,SAAA;cAGN,QAAA,UACA,OAAA,GAAS,wBAAA,EACT,QAAA,GAAU,mBAAA;EDxHR;;;;ECmKJ,cAAA,CAAA;EDjK+B;;AAKjC;ECiMQ,aAAA,CAAA,GAAiB,OAAA;;;;EAiIvB,UAAA,CAAW,OAAA;ED5TX;;;ECoUA,OAAA,CAAQ,CAAA,UAAW,CAAA,UAAW,KAAA;ED9TV;;;ECqUpB,eAAA,CAAA;ED7Te;;;ECoUf,KAAA,CAAA;EDlUA;;;;ECkVA,oBAAA,CAAA;ED1Uc;;AAShB;ECwUE,SAAA,CAAU,QAAA;;;;;EASV,WAAA,CAAA,GAAe,mBAAA;ED7Uf;;;EAAA,QCoVQ,aAAA;EAAA,QAcA,KAAA;;;;AD/UV;;UCoWU,oBAAA;EAAA,QAcM,UAAA;EDpWa;AAM7B;;;EAN6B,QC0Xb,YAAA;EDxWd;;;EAAA,QC+cc,gBAAA;EDrcA;AAMhB;;;;;;;;;EANgB,QCieN,iBAAA;EDzdR;;;;;;;EAAA,QCsfc,oBAAA;EDpfP;AAMT;;;EANS,QCsgBO,uBAAA;ED/fY;;;EAAA,QC2gBZ,wBAAA;ED3gBd;;;;;;;EAAA,QCyhBc,qBAAA;EDxhBV;AAMN;;;EANM,QCikBI,qBAAA;EAAA,QAYA,WAAA;EDrkBR;;;EAAA,QCglBQ,oBAAA;ED9kBR;;;EAAA,QCqlBQ,aAAA;EDplBD;;AAMT;EANS,QC2lBC,wBAAA;;;;UAOA,iCAAA;ED1lBF;;;EAAA,QCimBE,8BAAA;EDhmBR;;;AAMF;;;EANE,QC0mBc,qBAAA;EDnmBH;;;;EAAA,QCipBG,qBAAA;EDjpBd;;;;;;;;EAAA,QC8qBc,iBAAA;EAAA,QAmBN,cAAA;EAAA,QAOA,MAAA;AAAA"}

package/dist/{client-D73KQZf8.mjs → client-CSVSY-KV.mjs}

@@ -21,80 +21,6 @@ const $isEnabled = atom(true);
 atom(false);
 const $conversationHistory = atom([]);
 //#endregion
-//#region src/core/pointing.ts
-/**
-* Parses POINT tags from AI responses.
-*
-* Supports two formats:
-* - Marker-based: [POINT:5:label] - 3 parts, references a numbered marker
-* - Coordinate-based: [POINT:640,360:label] - 4 parts, raw pixel coordinates
-*/
-const POINTING_TAG_REGEX = /\[POINT:(\d+)(?:,(\d+))?:([^\]]+)\]\s*$/;
-const PARTIAL_POINTING_PREFIXES = new Set([
-	"[",
-	"[P",
-	"[PO",
-	"[POI",
-	"[POIN",
-	"[POINT",
-	"[POINT:"
-]);
-function stripTrailingPointingTag(response, trimResult) {
-	const stripped = response.replace(POINTING_TAG_REGEX, "");
-	return trimResult ? stripped.trim() : stripped;
-}
-function getPartialPointingTagStart(response) {
-	const lastOpenBracket = response.lastIndexOf("[");
-	if (lastOpenBracket === -1) return -1;
-	const suffix = response.slice(lastOpenBracket).trimEnd();
-	if (suffix.includes("]")) return -1;
-	if (suffix.startsWith("[POINT:")) {
-		let start = lastOpenBracket;
-		while (start > 0 && /\s/.test(response[start - 1] ?? "")) start--;
-		return start;
-	}
-	return PARTIAL_POINTING_PREFIXES.has(suffix) ? lastOpenBracket : -1;
-}
-/**
-* Parse pointing tag into structured result.
-* Returns null if no valid POINT tag is found at the end.
-*/
-function parsePointingTagRaw(response) {
-	const match = response.match(POINTING_TAG_REGEX);
-	if (!match) return null;
-	const first = Number.parseInt(match[1], 10);
-	const second = match[2] ? Number.parseInt(match[2], 10) : null;
-	const label = match[3].trim();
-	if (second !== null) return {
-		type: "coordinates",
-		x: first,
-		y: second,
-		label
-	};
-	return {
-		type: "marker",
-		markerId: first,
-		label
-	};
-}
-/**
-* Remove POINT tag from response text for display/TTS.
-*/
-function stripPointingTag(response) {
-	return stripTrailingPointingTag(response, true);
-}
-/**
-* Strip complete or partial trailing POINT syntax while the response streams.
-* This keeps the visible text and TTS input stable even if the tag arrives
-* incrementally over multiple chunks.
-*/
-function stripTrailingPointingSyntax(response) {
-	const withoutCompleteTag = stripTrailingPointingTag(response, false);
-	const partialTagStart = getPartialPointingTagStart(withoutCompleteTag);
-	if (partialTagStart === -1) return withoutCompleteTag.trimEnd();
-	return withoutCompleteTag.slice(0, partialTagStart).trimEnd();
-}
-//#endregion
 //#region src/core/utils/error.ts
 /**
 * Normalize unknown thrown values into Error instances.
@@ -691,7 +617,7 @@ const DEFAULT_STYLE = {
 	labelBackground: "rgba(255, 0, 0, 0.9)",
 	labelColor: "#ffffff",
 	borderWidth: 2,
-	fontSize: 11,
+	fontSize: 15,
 	labelPadding: 4
 };
 /**
@@ -1009,7 +935,7 @@ async function waitForClonedDocumentStyles(doc) {
 }
 function getHtml2CanvasOptions(captureMetrics) {
 	return {
-		scale: 1,
+		scale: window.devicePixelRatio,
 		useCORS: true,
 		logging: false,
 		width: captureMetrics.viewportWidth,
@@ -1584,6 +1510,47 @@ function createStateMachine(initial = "idle") {
 	};
 }
 //#endregion
+//#region src/core/utils/ui-stream-parser.ts
+/**
+* Parse a single line from the UI message stream.
+* The stream format is SSE with "data: " prefix followed by JSON.
+*/
+function parseUIStreamLine(line) {
+	const trimmed = line.trim();
+	if (!trimmed) return null;
+	let jsonStr = trimmed;
+	if (trimmed.startsWith("data: ")) jsonStr = trimmed.slice(6);
+	if (jsonStr === "[DONE]") return null;
+	try {
+		const chunk = JSON.parse(jsonStr);
+		switch (chunk.type) {
+			case "text-delta": return {
+				type: "text-delta",
+				delta: chunk.delta ?? ""
+			};
+			case "tool-input-available": return {
+				type: "tool-input-available",
+				toolName: chunk.toolName ?? "",
+				input: chunk.input
+			};
+			case "finish": return { type: "finish" };
+			case "error": return {
+				type: "error",
+				errorText: chunk.errorText ?? "Unknown error"
+			};
+			default: return { type: "unknown" };
+		}
+	} catch {
+		return null;
+	}
+}
+/**
+* Check if a tool call is a point tool call with valid input.
+*/
+function isPointToolCall(chunk) {
+	return chunk.type === "tool-input-available" && chunk.toolName === "point" && chunk.input != null && typeof chunk.input === "object" && "type" in chunk.input && "label" in chunk.input;
+}
+//#endregion
 //#region src/core/utils/response-processor.ts
 const COMMON_ABBREVIATIONS = [
 	"mr.",
@@ -1652,32 +1619,58 @@ function extractCompletedSegments(text) {
 	};
 }
 /**
-* Tracks a streaming assistant response, exposes a tag-free visible version for
-* the UI, and emits speakable segments as sentence boundaries become stable.
+* Processes a streaming AI SDK UI message stream response.
+* Extracts text for display/TTS and captures point tool calls.
 */
 var ProgressiveResponseProcessor = class {
-	consumedVisibleTextLength = 0;
+	consumedTextLength = 0;
 	pendingShortSegment = "";
-	rawResponse = "";
+	rawText = "";
+	buffer = "";
+	pointToolCall = null;
+	/**
+	* Push raw stream data and extract text chunks and tool calls.
+	* The UI message stream format is newline-delimited JSON.
+	*/
 	push(chunk) {
-		this.rawResponse += chunk;
-		const visibleText = stripTrailingPointingSyntax(this.rawResponse);
-		const { consumedLength, segments } = extractCompletedSegments(visibleText.slice(this.consumedVisibleTextLength));
-		this.consumedVisibleTextLength += consumedLength;
+		this.buffer += chunk;
+		const lines = this.buffer.split("\n");
+		this.buffer = lines.pop() ?? "";
+		const newTextParts = [];
+		for (const line of lines) {
+			const parsed = parseUIStreamLine(line);
+			if (!parsed) continue;
+			if (parsed.type === "text-delta") newTextParts.push(parsed.delta);
+			else if (isPointToolCall(parsed)) {
+				if (!this.pointToolCall) this.pointToolCall = parsed.input;
+			}
+		}
+		if (newTextParts.length > 0) this.rawText += newTextParts.join("");
+		const { consumedLength, segments } = extractCompletedSegments(this.rawText.slice(this.consumedTextLength));
+		this.consumedTextLength += consumedLength;
 		return {
-			visibleText,
-			speechSegments: this.coalesceSegments(segments)
+			visibleText: this.rawText,
+			speechSegments: this.coalesceSegments(segments),
+			pointToolCall: this.pointToolCall
 		};
 	}
+	/**
+	* Finalize processing and return any remaining text/tool call.
+	*/
 	finish() {
-		const finalResponseText = stripPointingTag(this.rawResponse);
-		const trailingText = finalResponseText.slice(this.consumedVisibleTextLength).trim();
+		if (this.buffer) {
+			const parsed = parseUIStreamLine(this.buffer);
+			if (parsed?.type === "text-delta") this.rawText += parsed.delta;
+			else if (parsed && isPointToolCall(parsed) && !this.pointToolCall) this.pointToolCall = parsed.input;
+			this.buffer = "";
+		}
+		const trailingText = this.rawText.slice(this.consumedTextLength).trim();
 		const finalSegmentParts = [this.pendingShortSegment, trailingText].filter(Boolean);
 		this.pendingShortSegment = "";
 		return {
-			fullResponse: this.rawResponse,
-			finalResponseText,
-			speechSegments: finalSegmentParts.length ? [finalSegmentParts.join(" ").trim()] : []
+			finalResponseText: this.rawText.trim(),
+			speechSegments: finalSegmentParts.length ? [finalSegmentParts.join(" ").trim()] : [],
+			pointToolCall: this.pointToolCall
 		};
 	}
 	coalesceSegments(segments) {
@@ -1839,7 +1832,7 @@ var CursorBuddyClient = class {
 			this.options.onTranscript?.(transcript);
 			this.notify();
 			this.prepareSpeechMode();
-			const { cleanResponse, fullResponse, playbackQueue } = await this.chatAndSpeak(transcript, screenshot, signal, {
+			const { cleanResponse, pointToolCall, playbackQueue } = await this.chatAndSpeak(transcript, screenshot, signal, {
 				onFailure: failTurn,
 				onPlaybackStart: () => {
 					this.stateMachine.transition({ type: "RESPONSE_STARTED" });
@@ -1847,18 +1840,17 @@ var CursorBuddyClient = class {
 			});
 			if (turnFailure) throw turnFailure;
 			if (signal?.aborted) return;
-			const parsed = parsePointingTagRaw(fullResponse);
 			this.options.onResponse?.(cleanResponse);
 			let pointTarget = null;
-			if (parsed) if (parsed.type === "marker") {
-				const coords = resolveMarkerToCoordinates(screenshot.markerMap, parsed.markerId);
+			if (pointToolCall) if (pointToolCall.type === "marker") {
+				const coords = resolveMarkerToCoordinates(screenshot.markerMap, pointToolCall.markerId);
 				if (coords) pointTarget = {
 					...coords,
-					label: parsed.label
+					label: pointToolCall.label
 				};
 			} else pointTarget = {
-				...mapCoordinatesToViewport(parsed.x, parsed.y, screenshot),
-				label: parsed.label
+				...mapCoordinatesToViewport(pointToolCall.x, pointToolCall.y, screenshot),
+				label: pointToolCall.label
 			};
 			if (pointTarget) {
 				this.options.onPoint?.(pointTarget);
@@ -2061,7 +2053,7 @@ var CursorBuddyClient = class {
 		this.updateResponse(finalizedResponse.finalResponseText);
 		return {
 			cleanResponse: finalizedResponse.finalResponseText,
-			fullResponse: finalizedResponse.fullResponse,
+			pointToolCall: finalizedResponse.pointToolCall,
 			playbackQueue
 		};
 	}
@@ -2259,4 +2251,4 @@ var CursorBuddyClient = class {
 //#endregion
 export { $buddyScale as a, $buddyRotation as i, $audioLevel as n, $cursorPosition as o, $buddyPosition as r, $pointingTarget as s, CursorBuddyClient as t };
 
-//# sourceMappingURL=client-D73KQZf8.mjs.map
+//# sourceMappingURL=client-CSVSY-KV.mjs.map