cursor-buddy 0.0.8 → 0.0.9-beta.0

package/dist/point-tool-DtHgq6gQ.mjs

@@ -0,0 +1,54 @@
+import { tool } from "ai";
+import { z } from "zod";
+const pointTool = tool({
+	description: "Visually point at something on the user's screen. Use this tool when the user asks you to locate, indicate, highlight, or show a specific visible target on screen. Prefer type 'marker' for interactive elements that have a marker. Use type 'coordinates' only for visible non-interactive content without a marker. Do not describe a pointing action in plain text instead of calling this tool. Call this tool at most once per response, and only after your spoken reply.",
+	inputSchema: z.object({
+		type: z.enum(["marker", "coordinates"]).describe("How to point. Use 'marker' for interactive elements that have a marker. Use 'coordinates' only for visible non-interactive content without a marker."),
+		markerId: z.number().int().min(0).optional().describe("Required when type is 'marker'. The marker ID of the interactive element to point at."),
+		x: z.number().int().min(0).optional().describe("Required when type is 'coordinates'. The horizontal pixel coordinate of the center of the target area."),
+		y: z.number().int().min(0).optional().describe("Required when type is 'coordinates'. The vertical pixel coordinate of the center of the target area."),
+		label: z.string().min(1).max(24).describe("A very short label for the pointer bubble, ideally 2 to 4 words.")
+	}).superRefine((value, ctx) => {
+		if (value.type === "marker") {
+			if (value.markerId == null) ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				path: ["markerId"],
+				message: "markerId is required when type is 'marker'."
+			});
+			if (value.x != null) ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				path: ["x"],
+				message: "x must not be provided when type is 'marker'."
+			});
+			if (value.y != null) ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				path: ["y"],
+				message: "y must not be provided when type is 'marker'."
+			});
+		}
+		if (value.type === "coordinates") {
+			if (value.x == null) ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				path: ["x"],
+				message: "x is required when type is 'coordinates'."
+			});
+			if (value.y == null) ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				path: ["y"],
+				message: "y is required when type is 'coordinates'."
+			});
+			if (value.markerId != null) ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				path: ["markerId"],
+				message: "markerId must not be provided when type is 'coordinates'."
+			});
+		}
+	}),
+	execute: async (params) => {
+		return `Pointed at "${params.label}" on the user's screen.`;
+	}
+});
+//#endregion
+export { pointTool as t };
+
+//# sourceMappingURL=point-tool-DtHgq6gQ.mjs.map

package/dist/point-tool-DtHgq6gQ.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"point-tool-DtHgq6gQ.mjs","names":[],"sources":["../src/shared/point-tool.ts"],"sourcesContent":["import { tool } from \"ai\"\nimport { z } from \"zod\"\n\nexport const pointToolInputSchema = z\n  .object({\n    type: z\n      .enum([\"marker\", \"coordinates\"])\n      .describe(\n        \"How to point. Use 'marker' for interactive elements that have a marker. Use 'coordinates' only for visible non-interactive content without a marker.\",\n      ),\n\n    markerId: z\n      .number()\n      .int()\n      .min(0)\n      .optional()\n      .describe(\n        \"Required when type is 'marker'. The marker ID of the interactive element to point at.\",\n      ),\n\n    x: z\n      .number()\n      .int()\n      .min(0)\n      .optional()\n      .describe(\n        \"Required when type is 'coordinates'. The horizontal pixel coordinate of the center of the target area.\",\n      ),\n\n    y: z\n      .number()\n      .int()\n      .min(0)\n      .optional()\n      .describe(\n        \"Required when type is 'coordinates'. The vertical pixel coordinate of the center of the target area.\",\n      ),\n\n    label: z\n      .string()\n      .min(1)\n      .max(24)\n      .describe(\n        \"A very short label for the pointer bubble, ideally 2 to 4 words.\",\n      ),\n  })\n  .superRefine((value, ctx) => {\n    if (value.type === \"marker\") {\n      if (value.markerId == null) {\n        ctx.addIssue({\n          code: z.ZodIssueCode.custom,\n          path: [\"markerId\"],\n          message: \"markerId is required when type is 'marker'.\",\n        })\n      }\n\n      if (value.x != null) {\n        ctx.addIssue({\n          code: z.ZodIssueCode.custom,\n          path: [\"x\"],\n          message: \"x must not be provided when type is 'marker'.\",\n        })\n      }\n\n      if (value.y != null) {\n        ctx.addIssue({\n          code: z.ZodIssueCode.custom,\n          path: [\"y\"],\n          message: \"y must not be provided when type is 'marker'.\",\n        })\n      }\n    }\n\n    if (value.type === \"coordinates\") {\n      if (value.x == null) {\n        ctx.addIssue({\n          code: z.ZodIssueCode.custom,\n          path: [\"x\"],\n          message: \"x is required when type is 'coordinates'.\",\n        })\n      }\n\n      if (value.y == null) {\n        ctx.addIssue({\n          code: z.ZodIssueCode.custom,\n          path: [\"y\"],\n          message: \"y is required when type is 'coordinates'.\",\n        })\n      }\n\n      if (value.markerId != null) {\n        ctx.addIssue({\n          code: z.ZodIssueCode.custom,\n          path: [\"markerId\"],\n          message: \"markerId must not be provided when type is 'coordinates'.\",\n        })\n      }\n    }\n  })\n\nexport type PointToolInput = z.infer<typeof pointToolInputSchema>\n\nexport const pointTool = tool({\n  description:\n    \"Visually point at something on the user's screen. \" +\n    \"Use this tool when the user asks you to locate, indicate, highlight, or show a specific visible target on screen. \" +\n    \"Prefer type 'marker' for interactive elements that have a marker. \" +\n    \"Use type 'coordinates' only for visible non-interactive content without a marker. \" +\n    \"Do not describe a pointing action in plain text instead of calling this tool. \" +\n    \"Call this tool at most once per response, and only after your spoken reply.\",\n\n  inputSchema: pointToolInputSchema,\n\n  execute: async (params) => {\n    return `Pointed at \"${params.label}\" on the user's screen.`\n  },\n})\n"],"mappings":";;AAsGA,MAAa,YAAY,KAAK;CAC5B,aACE;CAOF,aA5GkC,EACjC,OAAO;EACN,MAAM,EACH,KAAK,CAAC,UAAU,cAAc,CAAC,CAC/B,SACC,uJACD;EAEH,UAAU,EACP,QAAQ,CACR,KAAK,CACL,IAAI,EAAE,CACN,UAAU,CACV,SACC,wFACD;EAEH,GAAG,EACA,QAAQ,CACR,KAAK,CACL,IAAI,EAAE,CACN,UAAU,CACV,SACC,yGACD;EAEH,GAAG,EACA,QAAQ,CACR,KAAK,CACL,IAAI,EAAE,CACN,UAAU,CACV,SACC,uGACD;EAEH,OAAO,EACJ,QAAQ,CACR,IAAI,EAAE,CACN,IAAI,GAAG,CACP,SACC,mEACD;EACJ,CAAC,CACD,aAAa,OAAO,QAAQ;AAC3B,MAAI,MAAM,SAAS,UAAU;AAC3B,OAAI,MAAM,YAAY,KACpB,KAAI,SAAS;IACX,MAAM,EAAE,aAAa;IACrB,MAAM,CAAC,WAAW;IAClB,SAAS;IACV,CAAC;AAGJ,OAAI,MAAM,KAAK,KACb,KAAI,SAAS;IACX,MAAM,EAAE,aAAa;IACrB,MAAM,CAAC,IAAI;IACX,SAAS;IACV,CAAC;AAGJ,OAAI,MAAM,KAAK,KACb,KAAI,SAAS;IACX,MAAM,EAAE,aAAa;IACrB,MAAM,CAAC,IAAI;IACX,SAAS;IACV,CAAC;;AAIN,MAAI,MAAM,SAAS,eAAe;AAChC,OAAI,MAAM,KAAK,KACb,KAAI,SAAS;IACX,MAAM,EAAE,aAAa;IACrB,MAAM,CAAC,IAAI;IACX,SAAS;IACV,CAAC;AAGJ,OAAI,MAAM,KAAK,KACb,KAAI,SAAS;IACX,MAAM,EAAE,aAAa;IACrB,MAAM,CAAC,IAAI;IACX,SAAS;IACV,CAAC;AAGJ,OAAI,MAAM,YAAY,KACpB,KAAI,SAAS;IACX,MAAM,EAAE,aAAa;IACrB,MAAM,CAAC,WAAW;IAClB,SAAS;IACV,CAAC;;GAGN;CAeF,SAAS,OAAO,WAAW;AACzB,SAAO,eAAe,OAAO,MAAM;;CAEtC,CAAC"}

package/dist/point-tool-kIviMn1q.d.mts

@@ -0,0 +1,46 @@
+import * as _$ai from "ai";
+import { z } from "zod";
+
+//#region src/shared/point-tool.d.ts
+declare const pointToolInputSchema: z.ZodEffects<z.ZodObject<{
+  type: z.ZodEnum<["marker", "coordinates"]>;
+  markerId: z.ZodOptional<z.ZodNumber>;
+  x: z.ZodOptional<z.ZodNumber>;
+  y: z.ZodOptional<z.ZodNumber>;
+  label: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+  type: "marker" | "coordinates";
+  label: string;
+  markerId?: number | undefined;
+  x?: number | undefined;
+  y?: number | undefined;
+}, {
+  type: "marker" | "coordinates";
+  label: string;
+  markerId?: number | undefined;
+  x?: number | undefined;
+  y?: number | undefined;
+}>, {
+  type: "marker" | "coordinates";
+  label: string;
+  markerId?: number | undefined;
+  x?: number | undefined;
+  y?: number | undefined;
+}, {
+  type: "marker" | "coordinates";
+  label: string;
+  markerId?: number | undefined;
+  x?: number | undefined;
+  y?: number | undefined;
+}>;
+type PointToolInput = z.infer<typeof pointToolInputSchema>;
+declare const pointTool: _$ai.Tool<{
+  type: "marker" | "coordinates";
+  label: string;
+  markerId?: number | undefined;
+  x?: number | undefined;
+  y?: number | undefined;
+}, string>;
+//#endregion
+export { pointTool as n, PointToolInput as t };
+//# sourceMappingURL=point-tool-kIviMn1q.d.mts.map

package/dist/point-tool-kIviMn1q.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"point-tool-kIviMn1q.d.mts","names":[],"sources":["../src/shared/point-tool.ts"],"mappings":";;;;cAGa,oBAAA,EAAoB,CAAA,CAAA,UAAA,CAAA,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAiGrB,cAAA,GAAiB,CAAA,CAAE,KAAA,QAAa,oBAAA;AAAA,cAE/B,SAAA,EAcX,IAAA,CAdoB,IAAA"}

package/dist/react/index.d.mts

@@ -1,5 +1,5 @@
 "use client";
-import { c as CursorRenderProps, d as SpeechBubbleRenderProps, i as CursorBuddyMediaMode, m as WaveformRenderProps, o as CursorBuddySpeechConfig, p as VoiceState, r as CursorBuddyClientOptions, s as CursorBuddyTranscriptionConfig, u as PointingTarget } from "../client-Crn8tW7w.mjs";
+import { c as CursorRenderProps, d as SpeechBubbleRenderProps, i as CursorBuddyMediaMode, m as WaveformRenderProps, o as CursorBuddySpeechConfig, p as VoiceState, r as CursorBuddyClientOptions, s as CursorBuddyTranscriptionConfig, u as PointingTarget } from "../client-Ba6rv-du.mjs";
 import * as _$react_jsx_runtime0 from "react/jsx-runtime";
 
 //#region src/react/components/Overlay.d.ts

package/dist/react/index.mjs

@@ -1,5 +1,5 @@
 "use client";
-import { a as $buddyScale, i as $buddyRotation, n as $audioLevel, o as $cursorPosition, r as $buddyPosition, s as $pointingTarget, t as CursorBuddyClient } from "../client-D73KQZf8.mjs";
+import { a as $buddyScale, i as $buddyRotation, n as $audioLevel, o as $cursorPosition, r as $buddyPosition, s as $pointingTarget, t as CursorBuddyClient } from "../client-CevxN9EX.mjs";
 import { useStore } from "@nanostores/react";
 import { createContext, useCallback, useContext, useEffect, useRef, useState, useSyncExternalStore } from "react";
 import { jsx, jsxs } from "react/jsx-runtime";

package/dist/server/adapters/next.d.mts

@@ -1,4 +1,4 @@
-import { t as CursorBuddyHandler } from "../../types-BxBhjZju.mjs";
+import { t as CursorBuddyHandler } from "../../types-COQKMo5C.mjs";
 
 //#region src/server/adapters/next.d.ts
 /**

package/dist/server/index.d.mts

@@ -1,4 +1,5 @@
-import { n as CursorBuddyHandlerConfig, t as CursorBuddyHandler } from "../types-BxBhjZju.mjs";
+import { n as pointTool, t as PointToolInput } from "../point-tool-kIviMn1q.mjs";
+import { n as CursorBuddyHandlerConfig, t as CursorBuddyHandler } from "../types-COQKMo5C.mjs";
 
 //#region src/server/handler.d.ts
 /**
@@ -24,11 +25,7 @@ import { n as CursorBuddyHandlerConfig, t as CursorBuddyHandler } from "../types
 declare function createCursorBuddyHandler(config: CursorBuddyHandlerConfig): CursorBuddyHandler;
 //#endregion
 //#region src/server/system-prompt.d.ts
-/**
- * Default system prompt for the cursor buddy AI.
- * Instructs the model on how to respond conversationally and use POINT tags.
- */
-declare const DEFAULT_SYSTEM_PROMPT = "You are a helpful AI assistant that lives inside a web page as a cursor companion.\n\nYou can see screenshots of the user's viewport and hear their voice. Respond conversationally \u2014 your responses will be spoken aloud via text-to-speech, so keep them concise and natural.\n\n## Pointing at Elements\n\nWhen you want to direct the user's attention to something on screen, add a pointing tag at the END of your response. Only ONE pointing tag is allowed per response.\n\n### Interactive Elements (Preferred)\nInteractive elements (buttons, links, inputs, etc.) have invisible reference markers. Use the marker number to point at these:\n\n[POINT:marker_number:label]\n\nExample: \"Click this button right here. [POINT:5:Submit]\"\n\nThis is the most accurate pointing method \u2014 always prefer it when pointing at interactive elements.\n\n### Anywhere Else (Fallback)\nFor non-interactive content (text, images, areas without markers), use pixel coordinates:\n\n[POINT:x,y:label]\n\nWhere x,y are coordinates in screenshot image pixels (top-left origin).\n\nExample: \"The error message is shown here. [POINT:450,320:Error text]\"\n\n### Guidelines\n- NEVER mention the numbered markers or annotations to the user \u2014 these are invisible helpers for you only\n- Only point when it genuinely helps answer the user's specific question or request\n- Do NOT point at elements just because they have markers \u2014 point only when relevant to the conversation\n- Prefer marker-based pointing when the element has a marker and pointing is appropriate\n- Only use coordinates when pointing at unmarked content\n- Use natural descriptions (\"this button\", \"over here\", \"right there\")\n- Coordinates should be the CENTER of the element you're pointing at\n- Keep labels short (2-4 words)\n\n## Response Style\n\n- Be concise \u2014 aim for 1-3 sentences\n- Sound natural when spoken aloud\n- Avoid technical jargon unless the user is technical\n- If you can't see something clearly, say so\n- Never mention that you're looking at a \"screenshot\" \u2014 say \"I can see...\" or \"Looking at your screen...\"\n- Never mention the numbered markers or annotations you see on elements\n";
+declare const DEFAULT_SYSTEM_PROMPT = "You are a helpful AI assistant that lives inside a web page as a cursor companion.\n\nYou can see the user's current screen and hear what they say. Respond conversationally. Your response will be spoken aloud with text-to-speech, so keep it natural, concise, and easy to follow.\n\n## Core behavior\n\n- Speak like a helpful companion, not a robot\n- Keep most responses to 1-3 short sentences\n- Focus on what is visible right now on the user's screen\n- If something is unclear or not visible, say that plainly\n- Do not mention screenshots, overlays, annotations, or internal helper data\n- Do not mention marker numbers to the user\n\n## The point tool\n\nYou have a `point` tool that can visually indicate something on the user's screen.\n\nUse the `point` tool when the user is asking you to identify, locate, indicate, highlight, or show something visible on screen.\n\nCommon cases where you should use `point`:\n- the user asks where something is\n- the user asks what to click\n- the user says things like \"show me\", \"point to it\", \"where is it\", \"which one\", \"what should I click\", or \"highlight that\"\n\nDo not use the `point` tool when spoken guidance alone is enough and the user is not asking you to identify a specific on-screen target.\n\nExamples where spoken guidance alone may be enough:\n- explaining what a page does\n- answering a general question about what is on screen\n- giving brief next-step advice that does not depend on locating a specific element\n\nIf using the `point` tool:\n- first give the spoken response\n- then call the tool\n- call it at most once per response\n- point only at the most relevant target\n- never replace the tool call with plain text like \"(point here)\" or \"I\u2019m pointing at it now\"\n\n\nIf the user asks where something is on screen, what to click, or asks you to point something out, you should usually use the point tool rather than only describing it in words.\nDo not say things like \"I can point to it if you want\" when the user already asked where it is. In that case, answer briefly and use the point tool.\n\n## How to point\n\nPrefer marker-based pointing for interactive elements when a marker is available.\nInteractive elements may include buttons, links, inputs, tabs, menus, toggles, and other clickable controls.\n\nUse:\n- `type: \"marker\"` for interactive elements that have a marker\n- `type: \"coordinates\"` only for visible non-interactive content without a marker\n\nNever use coordinates for an interactive element if a marker is available.\n\nCoordinates must refer to the center of the target area.\n\nWhen calling the point tool, choose exactly one mode:\n\n- Marker mode:\n  - use type \"marker\"\n  - provide markerId\n  - do not provide x or y\n\n- Coordinates mode:\n  - use type \"coordinates\"\n  - provide x and y\n  - do not provide markerId\n\nNever combine markerId with x or y in the same tool call.\n\n## What to say\n\nWhen the user asks you to point something out:\n- briefly answer in a natural spoken way\n- then use the tool if the request is about locating or indicating something on screen\n\nGood spoken style:\n- \"Click this button right here.\"\n- \"The error message is over here.\"\n- \"This is the field you want.\"\n- \"That setting is in this section.\"\n\nAvoid:\n- mentioning marker IDs\n- mentioning internal tools\n- describing internal reasoning\n- saying you are looking at a screenshot\n\n## If the target is not clear\n\nIf you cannot confidently find the requested thing on screen:\n- say you cannot see it clearly or cannot find it\n- do not point at a random or uncertain target\n\n## Priority\n\nYour first priority is being helpful and correct.\nYour second priority is using the `point` tool whenever the user is asking you to visually identify a specific thing on screen.\n";
 //#endregion
-export { type CursorBuddyHandler, type CursorBuddyHandlerConfig, DEFAULT_SYSTEM_PROMPT, createCursorBuddyHandler };
+export { type CursorBuddyHandler, type CursorBuddyHandlerConfig, DEFAULT_SYSTEM_PROMPT, type PointToolInput, createCursorBuddyHandler, pointTool };
 //# sourceMappingURL=index.d.mts.map

package/dist/server/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../../src/server/handler.ts","../../src/server/system-prompt.ts"],"mappings":";;;;;AAyBA;;;;;;;;;;;;ACrBA;;;;;;iBDqBgB,wBAAA,CACd,MAAA,EAAQ,wBAAA,GACP,kBAAA;;;;;;AAFH;cCrBa,qBAAA"}
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../src/server/handler.ts","../../src/server/system-prompt.ts"],"mappings":";;;;;;;AAyBA;;;;;;;;;;;;ACzBA;;;;;iBDyBgB,wBAAA,CACd,MAAA,EAAQ,wBAAA,GACP,kBAAA;;;cC3BU,qBAAA"}

package/dist/server/index.mjs

@@ -1,53 +1,103 @@
+import { t as pointTool } from "../point-tool-DtHgq6gQ.mjs";
 import { experimental_generateSpeech, experimental_transcribe, streamText } from "ai";
 //#region src/server/system-prompt.ts
-/**
-* Default system prompt for the cursor buddy AI.
-* Instructs the model on how to respond conversationally and use POINT tags.
-*/
 const DEFAULT_SYSTEM_PROMPT = `You are a helpful AI assistant that lives inside a web page as a cursor companion.
 
-You can see screenshots of the user's viewport and hear their voice. Respond conversationally — your responses will be spoken aloud via text-to-speech, so keep them concise and natural.
+You can see the user's current screen and hear what they say. Respond conversationally. Your response will be spoken aloud with text-to-speech, so keep it natural, concise, and easy to follow.
+
+## Core behavior
+
+- Speak like a helpful companion, not a robot
+- Keep most responses to 1-3 short sentences
+- Focus on what is visible right now on the user's screen
+- If something is unclear or not visible, say that plainly
+- Do not mention screenshots, overlays, annotations, or internal helper data
+- Do not mention marker numbers to the user
+
+## The point tool
+
+You have a \`point\` tool that can visually indicate something on the user's screen.
+
+Use the \`point\` tool when the user is asking you to identify, locate, indicate, highlight, or show something visible on screen.
+
+Common cases where you should use \`point\`:
+- the user asks where something is
+- the user asks what to click
+- the user says things like "show me", "point to it", "where is it", "which one", "what should I click", or "highlight that"
+
+Do not use the \`point\` tool when spoken guidance alone is enough and the user is not asking you to identify a specific on-screen target.
+
+Examples where spoken guidance alone may be enough:
+- explaining what a page does
+- answering a general question about what is on screen
+- giving brief next-step advice that does not depend on locating a specific element
+
+If using the \`point\` tool:
+- first give the spoken response
+- then call the tool
+- call it at most once per response
+- point only at the most relevant target
+- never replace the tool call with plain text like "(point here)" or "I’m pointing at it now"
 
-## Pointing at Elements
 
-When you want to direct the user's attention to something on screen, add a pointing tag at the END of your response. Only ONE pointing tag is allowed per response.
+If the user asks where something is on screen, what to click, or asks you to point something out, you should usually use the point tool rather than only describing it in words.
+Do not say things like "I can point to it if you want" when the user already asked where it is. In that case, answer briefly and use the point tool.
 
-### Interactive Elements (Preferred)
-Interactive elements (buttons, links, inputs, etc.) have invisible reference markers. Use the marker number to point at these:
+## How to point
 
-[POINT:marker_number:label]
+Prefer marker-based pointing for interactive elements when a marker is available.
+Interactive elements may include buttons, links, inputs, tabs, menus, toggles, and other clickable controls.
 
-Example: "Click this button right here. [POINT:5:Submit]"
+Use:
+- \`type: "marker"\` for interactive elements that have a marker
+- \`type: "coordinates"\` only for visible non-interactive content without a marker
 
-This is the most accurate pointing method — always prefer it when pointing at interactive elements.
+Never use coordinates for an interactive element if a marker is available.
 
-### Anywhere Else (Fallback)
-For non-interactive content (text, images, areas without markers), use pixel coordinates:
+Coordinates must refer to the center of the target area.
 
-[POINT:x,y:label]
+When calling the point tool, choose exactly one mode:
 
-Where x,y are coordinates in screenshot image pixels (top-left origin).
+- Marker mode:
+  - use type "marker"
+  - provide markerId
+  - do not provide x or y
 
-Example: "The error message is shown here. [POINT:450,320:Error text]"
+- Coordinates mode:
+  - use type "coordinates"
+  - provide x and y
+  - do not provide markerId
 
-### Guidelines
-- NEVER mention the numbered markers or annotations to the user — these are invisible helpers for you only
-- Only point when it genuinely helps answer the user's specific question or request
-- Do NOT point at elements just because they have markers — point only when relevant to the conversation
-- Prefer marker-based pointing when the element has a marker and pointing is appropriate
-- Only use coordinates when pointing at unmarked content
-- Use natural descriptions ("this button", "over here", "right there")
-- Coordinates should be the CENTER of the element you're pointing at
-- Keep labels short (2-4 words)
+Never combine markerId with x or y in the same tool call.
 
-## Response Style
+## What to say
 
-- Be concise — aim for 1-3 sentences
-- Sound natural when spoken aloud
-- Avoid technical jargon unless the user is technical
-- If you can't see something clearly, say so
-- Never mention that you're looking at a "screenshot" — say "I can see..." or "Looking at your screen..."
-- Never mention the numbered markers or annotations you see on elements
+When the user asks you to point something out:
+- briefly answer in a natural spoken way
+- then use the tool if the request is about locating or indicating something on screen
+
+Good spoken style:
+- "Click this button right here."
+- "The error message is over here."
+- "This is the field you want."
+- "That setting is in this section."
+
+Avoid:
+- mentioning marker IDs
+- mentioning internal tools
+- describing internal reasoning
+- saying you are looking at a screenshot
+
+## If the target is not clear
+
+If you cannot confidently find the requested thing on screen:
+- say you cannot see it clearly or cannot find it
+- do not point at a random or uncertain target
+
+## Priority
+
+Your first priority is being helpful and correct.
+Your second priority is using the \`point\` tool whenever the user is asking you to visually identify a specific thing on screen.
 `;
 //#endregion
 //#region src/server/routes/chat.ts
@@ -88,8 +138,46 @@ async function handleChat(request, config) {
 		system: systemPrompt,
 		providerOptions: config?.modelProviderMetadata,
 		messages,
-		tools: config.tools
-	}).toTextStreamResponse();
+		tools: {
+			point: pointTool,
+			...config.tools
+		},
+		experimental_repairToolCall: async ({ toolCall }) => {
+			if (toolCall.toolName !== "point") return null;
+			let parsed;
+			try {
+				parsed = JSON.parse(toolCall.input);
+			} catch {
+				return null;
+			}
+			if (!parsed || typeof parsed !== "object") return null;
+			const input = parsed;
+			if (input.type === "marker") {
+				const repaired = {
+					type: "marker",
+					markerId: input.markerId,
+					label: input.label
+				};
+				return {
+					...toolCall,
+					input: JSON.stringify(repaired)
+				};
+			}
+			if (input.type === "coordinates") {
+				const repaired = {
+					type: "coordinates",
+					x: input.x,
+					y: input.y,
+					label: input.label
+				};
+				return {
+					...toolCall,
+					input: JSON.stringify(repaired)
+				};
+			}
+			return null;
+		}
+	}).toUIMessageStreamResponse();
 }
 //#endregion
 //#region src/server/routes/transcribe.ts
@@ -185,6 +273,6 @@ function createCursorBuddyHandler(config) {
 	};
 }
 //#endregion
-export { DEFAULT_SYSTEM_PROMPT, createCursorBuddyHandler };
+export { DEFAULT_SYSTEM_PROMPT, createCursorBuddyHandler, pointTool };
 
 //# sourceMappingURL=index.mjs.map

package/dist/server/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","names":["transcribe","generateSpeech"],"sources":["../../src/server/system-prompt.ts","../../src/server/routes/chat.ts","../../src/server/routes/transcribe.ts","../../src/server/routes/tts.ts","../../src/server/handler.ts"],"sourcesContent":["/**\n * Default system prompt for the cursor buddy AI.\n * Instructs the model on how to respond conversationally and use POINT tags.\n */\nexport const DEFAULT_SYSTEM_PROMPT = `You are a helpful AI assistant that lives inside a web page as a cursor companion.\n\nYou can see screenshots of the user's viewport and hear their voice. Respond conversationally — your responses will be spoken aloud via text-to-speech, so keep them concise and natural.\n\n## Pointing at Elements\n\nWhen you want to direct the user's attention to something on screen, add a pointing tag at the END of your response. Only ONE pointing tag is allowed per response.\n\n### Interactive Elements (Preferred)\nInteractive elements (buttons, links, inputs, etc.) have invisible reference markers. Use the marker number to point at these:\n\n[POINT:marker_number:label]\n\nExample: \"Click this button right here. [POINT:5:Submit]\"\n\nThis is the most accurate pointing method — always prefer it when pointing at interactive elements.\n\n### Anywhere Else (Fallback)\nFor non-interactive content (text, images, areas without markers), use pixel coordinates:\n\n[POINT:x,y:label]\n\nWhere x,y are coordinates in screenshot image pixels (top-left origin).\n\nExample: \"The error message is shown here. [POINT:450,320:Error text]\"\n\n### Guidelines\n- NEVER mention the numbered markers or annotations to the user — these are invisible helpers for you only\n- Only point when it genuinely helps answer the user's specific question or request\n- Do NOT point at elements just because they have markers — point only when relevant to the conversation\n- Prefer marker-based pointing when the element has a marker and pointing is appropriate\n- Only use coordinates when pointing at unmarked content\n- Use natural descriptions (\"this button\", \"over here\", \"right there\")\n- Coordinates should be the CENTER of the element you're pointing at\n- Keep labels short (2-4 words)\n\n## Response Style\n\n- Be concise — aim for 1-3 sentences\n- Sound natural when spoken aloud\n- Avoid technical jargon unless the user is technical\n- If you can't see something clearly, say so\n- Never mention that you're looking at a \"screenshot\" — say \"I can see...\" or \"Looking at your screen...\"\n- Never mention the numbered markers or annotations you see on elements\n`\n","import { streamText } from \"ai\"\nimport { DEFAULT_SYSTEM_PROMPT } from \"../system-prompt\"\nimport type { ChatRequestBody, CursorBuddyHandlerConfig } from \"../types\"\n\n/**\n * Handle chat requests: screenshot + transcript → AI SSE stream\n */\nexport async function handleChat(\n  request: Request,\n  config: CursorBuddyHandlerConfig,\n): Promise<Response> {\n  const body = (await request.json()) as ChatRequestBody\n  const { screenshot, transcript, history, capture, markerContext } = body\n\n  // Resolve system prompt (string or function)\n  const systemPrompt =\n    typeof config.system === \"function\"\n      ? config.system({ defaultPrompt: DEFAULT_SYSTEM_PROMPT })\n      : (config.system ?? DEFAULT_SYSTEM_PROMPT)\n\n  // Trim history to maxHistory (default 10 exchanges = 20 messages)\n  const maxMessages = (config.maxHistory ?? 10) * 2\n  const trimmedHistory = history.slice(-maxMessages)\n\n  // Build capture context with marker information\n  const captureContextParts: string[] = []\n\n  if (capture) {\n    captureContextParts.push(\n      `Screenshot size: ${capture.width}x${capture.height} pixels.`,\n    )\n  }\n\n  if (markerContext) {\n    captureContextParts.push(\"\", markerContext)\n  }\n\n  const captureContext =\n    captureContextParts.length > 0 ? captureContextParts.join(\"\\n\") : null\n\n  // Build messages array with vision content\n  const messages = [\n    ...trimmedHistory.map((msg) => ({\n      role: msg.role as \"user\" | \"assistant\",\n      content: msg.content,\n    })),\n    {\n      role: \"user\" as const,\n      content: [\n        ...(captureContext\n          ? [\n              {\n                type: \"text\" as const,\n                text: captureContext,\n              },\n            ]\n          : []),\n        {\n          type: \"image\" as const,\n          image: screenshot,\n        },\n        {\n          type: \"text\" as const,\n          text: transcript,\n        },\n      ],\n    },\n  ]\n\n  const result = streamText({\n    model: config.model,\n    system: systemPrompt,\n    providerOptions: config?.modelProviderMetadata,\n    messages,\n    tools: config.tools,\n  })\n\n  return result.toTextStreamResponse()\n}\n","import { experimental_transcribe as transcribe } from \"ai\"\nimport type { CursorBuddyHandlerConfig, TranscribeResponse } from \"../types\"\n\n/**\n * Handle transcription requests: audio file → text\n */\nexport async function handleTranscribe(\n  request: Request,\n  config: CursorBuddyHandlerConfig,\n): Promise<Response> {\n  if (!config.transcriptionModel) {\n    return new Response(\n      JSON.stringify({\n        error:\n          \"Server transcription is not configured. Provide a transcriptionModel or use browser transcription only.\",\n      }),\n      {\n        status: 501,\n        headers: { \"Content-Type\": \"application/json\" },\n      },\n    )\n  }\n\n  const formData = await request.formData()\n  const audioFile = formData.get(\"audio\")\n\n  if (!audioFile || !(audioFile instanceof File)) {\n    return new Response(JSON.stringify({ error: \"No audio file provided\" }), {\n      status: 400,\n      headers: { \"Content-Type\": \"application/json\" },\n    })\n  }\n\n  const audioBuffer = await audioFile.arrayBuffer()\n\n  const result = await transcribe({\n    model: config.transcriptionModel,\n    audio: new Uint8Array(audioBuffer),\n  })\n\n  const response: TranscribeResponse = { text: result.text }\n\n  return new Response(JSON.stringify(response), {\n    headers: { \"Content-Type\": \"application/json\" },\n  })\n}\n","import { experimental_generateSpeech as generateSpeech } from \"ai\"\nimport type { CursorBuddyHandlerConfig, TTSRequestBody } from \"../types\"\n\n/**\n * Handle TTS requests: text → audio\n */\nexport async function handleTTS(\n  request: Request,\n  config: CursorBuddyHandlerConfig,\n): Promise<Response> {\n  if (!config.speechModel) {\n    return new Response(\n      JSON.stringify({\n        error:\n          \"Server speech is not configured. Provide a speechModel or use browser speech only.\",\n      }),\n      {\n        status: 501,\n        headers: { \"Content-Type\": \"application/json\" },\n      },\n    )\n  }\n\n  const outputFormat = \"wav\"\n  const body = (await request.json()) as TTSRequestBody\n  const { text } = body\n\n  if (!text) {\n    return new Response(JSON.stringify({ error: \"No text provided\" }), {\n      status: 400,\n      headers: { \"Content-Type\": \"application/json\" },\n    })\n  }\n\n  const result = await generateSpeech({\n    model: config.speechModel,\n    text,\n    outputFormat,\n  })\n\n  // Create a new ArrayBuffer copy to satisfy TypeScript's strict typing\n  const audioData = new Uint8Array(result.audio.uint8Array)\n\n  return new Response(audioData, {\n    headers: {\n      \"Content-Type\": \"audio/wav\",\n    },\n  })\n}\n","import { handleChat } from \"./routes/chat\"\nimport { handleTranscribe } from \"./routes/transcribe\"\nimport { handleTTS } from \"./routes/tts\"\nimport type { CursorBuddyHandler, CursorBuddyHandlerConfig } from \"./types\"\n\n/**\n * Create a cursor buddy request handler.\n *\n * The handler responds to three routes based on the last path segment:\n * - /chat - Screenshot + transcript → AI SSE stream\n * - /transcribe - Audio → text\n * - /tts - Text → audio\n *\n * @example\n * ```ts\n * import { createCursorBuddyHandler } from \"cursor-buddy/server\"\n * import { openai } from \"@ai-sdk/openai\"\n *\n * const cursorBuddy = createCursorBuddyHandler({\n *   model: openai(\"gpt-4o\"),\n *   speechModel: openai.speech(\"tts-1\"), // optional for browser-only speech\n *   transcriptionModel: openai.transcription(\"whisper-1\"),\n * })\n * ```\n */\nexport function createCursorBuddyHandler(\n  config: CursorBuddyHandlerConfig,\n): CursorBuddyHandler {\n  const handler = async (request: Request): Promise<Response> => {\n    const url = new URL(request.url)\n    const pathSegments = url.pathname.split(\"/\").filter(Boolean)\n    const route = pathSegments[pathSegments.length - 1]\n\n    switch (route) {\n      case \"chat\":\n        return handleChat(request, config)\n\n      case \"transcribe\":\n        return handleTranscribe(request, config)\n\n      case \"tts\":\n        return handleTTS(request, config)\n\n      default:\n        return new Response(\n          JSON.stringify({\n            error: \"Not found\",\n            availableRoutes: [\"/chat\", \"/transcribe\", \"/tts\"],\n          }),\n          {\n            status: 404,\n            headers: { \"Content-Type\": \"application/json\" },\n          },\n        )\n    }\n  }\n\n  return { handler, config }\n}\n"],"mappings":";;;;;;AAIA,MAAa,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACGrC,eAAsB,WACpB,SACA,QACmB;CAEnB,MAAM,EAAE,YAAY,YAAY,SAAS,SAAS,kBADpC,MAAM,QAAQ,MAAM;CAIlC,MAAM,eACJ,OAAO,OAAO,WAAW,aACrB,OAAO,OAAO,EAAE,eAAe,uBAAuB,CAAC,GACtD,OAAO,UAAU;CAGxB,MAAM,eAAe,OAAO,cAAc,MAAM;CAChD,MAAM,iBAAiB,QAAQ,MAAM,CAAC,YAAY;CAGlD,MAAM,sBAAgC,EAAE;AAExC,KAAI,QACF,qBAAoB,KAClB,oBAAoB,QAAQ,MAAM,GAAG,QAAQ,OAAO,UACrD;AAGH,KAAI,cACF,qBAAoB,KAAK,IAAI,cAAc;CAG7C,MAAM,iBACJ,oBAAoB,SAAS,IAAI,oBAAoB,KAAK,KAAK,GAAG;CAGpE,MAAM,WAAW,CACf,GAAG,eAAe,KAAK,SAAS;EAC9B,MAAM,IAAI;EACV,SAAS,IAAI;EACd,EAAE,EACH;EACE,MAAM;EACN,SAAS;GACP,GAAI,iBACA,CACE;IACE,MAAM;IACN,MAAM;IACP,CACF,GACD,EAAE;GACN;IACE,MAAM;IACN,OAAO;IACR;GACD;IACE,MAAM;IACN,MAAM;IACP;GACF;EACF,CACF;AAUD,QARe,WAAW;EACxB,OAAO,OAAO;EACd,QAAQ;EACR,iBAAiB,QAAQ;EACzB;EACA,OAAO,OAAO;EACf,CAAC,CAEY,sBAAsB;;;;;;;ACvEtC,eAAsB,iBACpB,SACA,QACmB;AACnB,KAAI,CAAC,OAAO,mBACV,QAAO,IAAI,SACT,KAAK,UAAU,EACb,OACE,2GACH,CAAC,EACF;EACE,QAAQ;EACR,SAAS,EAAE,gBAAgB,oBAAoB;EAChD,CACF;CAIH,MAAM,aADW,MAAM,QAAQ,UAAU,EACd,IAAI,QAAQ;AAEvC,KAAI,CAAC,aAAa,EAAE,qBAAqB,MACvC,QAAO,IAAI,SAAS,KAAK,UAAU,EAAE,OAAO,0BAA0B,CAAC,EAAE;EACvE,QAAQ;EACR,SAAS,EAAE,gBAAgB,oBAAoB;EAChD,CAAC;CAGJ,MAAM,cAAc,MAAM,UAAU,aAAa;CAOjD,MAAM,WAA+B,EAAE,OALxB,MAAMA,wBAAW;EAC9B,OAAO,OAAO;EACd,OAAO,IAAI,WAAW,YAAY;EACnC,CAAC,EAEkD,MAAM;AAE1D,QAAO,IAAI,SAAS,KAAK,UAAU,SAAS,EAAE,EAC5C,SAAS,EAAE,gBAAgB,oBAAoB,EAChD,CAAC;;;;;;;ACtCJ,eAAsB,UACpB,SACA,QACmB;AACnB,KAAI,CAAC,OAAO,YACV,QAAO,IAAI,SACT,KAAK,UAAU,EACb,OACE,sFACH,CAAC,EACF;EACE,QAAQ;EACR,SAAS,EAAE,gBAAgB,oBAAoB;EAChD,CACF;CAGH,MAAM,eAAe;CAErB,MAAM,EAAE,SADM,MAAM,QAAQ,MAAM;AAGlC,KAAI,CAAC,KACH,QAAO,IAAI,SAAS,KAAK,UAAU,EAAE,OAAO,oBAAoB,CAAC,EAAE;EACjE,QAAQ;EACR,SAAS,EAAE,gBAAgB,oBAAoB;EAChD,CAAC;CAGJ,MAAM,SAAS,MAAMC,4BAAe;EAClC,OAAO,OAAO;EACd;EACA;EACD,CAAC;CAGF,MAAM,YAAY,IAAI,WAAW,OAAO,MAAM,WAAW;AAEzD,QAAO,IAAI,SAAS,WAAW,EAC7B,SAAS,EACP,gBAAgB,aACjB,EACF,CAAC;;;;;;;;;;;;;;;;;;;;;;;;ACtBJ,SAAgB,yBACd,QACoB;CACpB,MAAM,UAAU,OAAO,YAAwC;EAE7D,MAAM,eADM,IAAI,IAAI,QAAQ,IAAI,CACP,SAAS,MAAM,IAAI,CAAC,OAAO,QAAQ;AAG5D,UAFc,aAAa,aAAa,SAAS,IAEjD;GACE,KAAK,OACH,QAAO,WAAW,SAAS,OAAO;GAEpC,KAAK,aACH,QAAO,iBAAiB,SAAS,OAAO;GAE1C,KAAK,MACH,QAAO,UAAU,SAAS,OAAO;GAEnC,QACE,QAAO,IAAI,SACT,KAAK,UAAU;IACb,OAAO;IACP,iBAAiB;KAAC;KAAS;KAAe;KAAO;IAClD,CAAC,EACF;IACE,QAAQ;IACR,SAAS,EAAE,gBAAgB,oBAAoB;IAChD,CACF;;;AAIP,QAAO;EAAE;EAAS;EAAQ"}
+{"version":3,"file":"index.mjs","names":["transcribe","generateSpeech"],"sources":["../../src/server/system-prompt.ts","../../src/server/routes/chat.ts","../../src/server/routes/transcribe.ts","../../src/server/routes/tts.ts","../../src/server/handler.ts"],"sourcesContent":["export const DEFAULT_SYSTEM_PROMPT = `You are a helpful AI assistant that lives inside a web page as a cursor companion.\n\nYou can see the user's current screen and hear what they say. Respond conversationally. Your response will be spoken aloud with text-to-speech, so keep it natural, concise, and easy to follow.\n\n## Core behavior\n\n- Speak like a helpful companion, not a robot\n- Keep most responses to 1-3 short sentences\n- Focus on what is visible right now on the user's screen\n- If something is unclear or not visible, say that plainly\n- Do not mention screenshots, overlays, annotations, or internal helper data\n- Do not mention marker numbers to the user\n\n## The point tool\n\nYou have a \\`point\\` tool that can visually indicate something on the user's screen.\n\nUse the \\`point\\` tool when the user is asking you to identify, locate, indicate, highlight, or show something visible on screen.\n\nCommon cases where you should use \\`point\\`:\n- the user asks where something is\n- the user asks what to click\n- the user says things like \"show me\", \"point to it\", \"where is it\", \"which one\", \"what should I click\", or \"highlight that\"\n\nDo not use the \\`point\\` tool when spoken guidance alone is enough and the user is not asking you to identify a specific on-screen target.\n\nExamples where spoken guidance alone may be enough:\n- explaining what a page does\n- answering a general question about what is on screen\n- giving brief next-step advice that does not depend on locating a specific element\n\nIf using the \\`point\\` tool:\n- first give the spoken response\n- then call the tool\n- call it at most once per response\n- point only at the most relevant target\n- never replace the tool call with plain text like \"(point here)\" or \"I’m pointing at it now\"\n\n\nIf the user asks where something is on screen, what to click, or asks you to point something out, you should usually use the point tool rather than only describing it in words.\nDo not say things like \"I can point to it if you want\" when the user already asked where it is. In that case, answer briefly and use the point tool.\n\n## How to point\n\nPrefer marker-based pointing for interactive elements when a marker is available.\nInteractive elements may include buttons, links, inputs, tabs, menus, toggles, and other clickable controls.\n\nUse:\n- \\`type: \"marker\"\\` for interactive elements that have a marker\n- \\`type: \"coordinates\"\\` only for visible non-interactive content without a marker\n\nNever use coordinates for an interactive element if a marker is available.\n\nCoordinates must refer to the center of the target area.\n\nWhen calling the point tool, choose exactly one mode:\n\n- Marker mode:\n  - use type \"marker\"\n  - provide markerId\n  - do not provide x or y\n\n- Coordinates mode:\n  - use type \"coordinates\"\n  - provide x and y\n  - do not provide markerId\n\nNever combine markerId with x or y in the same tool call.\n\n## What to say\n\nWhen the user asks you to point something out:\n- briefly answer in a natural spoken way\n- then use the tool if the request is about locating or indicating something on screen\n\nGood spoken style:\n- \"Click this button right here.\"\n- \"The error message is over here.\"\n- \"This is the field you want.\"\n- \"That setting is in this section.\"\n\nAvoid:\n- mentioning marker IDs\n- mentioning internal tools\n- describing internal reasoning\n- saying you are looking at a screenshot\n\n## If the target is not clear\n\nIf you cannot confidently find the requested thing on screen:\n- say you cannot see it clearly or cannot find it\n- do not point at a random or uncertain target\n\n## Priority\n\nYour first priority is being helpful and correct.\nYour second priority is using the \\`point\\` tool whenever the user is asking you to visually identify a specific thing on screen.\n`\n","import { streamText } from \"ai\"\nimport { pointTool } from \"../../shared/point-tool\"\nimport { DEFAULT_SYSTEM_PROMPT } from \"../system-prompt\"\nimport type { ChatRequestBody, CursorBuddyHandlerConfig } from \"../types\"\n\n/**\n * Handle chat requests: screenshot + transcript → AI SSE stream\n */\nexport async function handleChat(\n  request: Request,\n  config: CursorBuddyHandlerConfig,\n): Promise<Response> {\n  const body = (await request.json()) as ChatRequestBody\n  const { screenshot, transcript, history, capture, markerContext } = body\n\n  // Resolve system prompt (string or function)\n  const systemPrompt =\n    typeof config.system === \"function\"\n      ? config.system({ defaultPrompt: DEFAULT_SYSTEM_PROMPT })\n      : (config.system ?? DEFAULT_SYSTEM_PROMPT)\n\n  // Trim history to maxHistory (default 10 exchanges = 20 messages)\n  const maxMessages = (config.maxHistory ?? 10) * 2\n  const trimmedHistory = history.slice(-maxMessages)\n\n  // Build capture context with marker information\n  const captureContextParts: string[] = []\n\n  if (capture) {\n    captureContextParts.push(\n      `Screenshot size: ${capture.width}x${capture.height} pixels.`,\n    )\n  }\n\n  if (markerContext) {\n    captureContextParts.push(\"\", markerContext)\n  }\n\n  const captureContext =\n    captureContextParts.length > 0 ? captureContextParts.join(\"\\n\") : null\n\n  // Build messages array with vision content\n  const messages = [\n    ...trimmedHistory.map((msg) => ({\n      role: msg.role as \"user\" | \"assistant\",\n      content: msg.content,\n    })),\n    {\n      role: \"user\" as const,\n      content: [\n        ...(captureContext\n          ? [\n              {\n                type: \"text\" as const,\n                text: captureContext,\n              },\n            ]\n          : []),\n        {\n          type: \"image\" as const,\n          image: screenshot,\n        },\n        {\n          type: \"text\" as const,\n          text: transcript,\n        },\n      ],\n    },\n  ]\n\n  const result = streamText({\n    model: config.model,\n    system: systemPrompt,\n    providerOptions: config?.modelProviderMetadata,\n    messages,\n    tools: {\n      point: pointTool,\n      ...config.tools,\n    },\n    experimental_repairToolCall: async ({ toolCall }) => {\n      if (toolCall.toolName !== \"point\") return null\n\n      let parsed: unknown\n      try {\n        parsed = JSON.parse(toolCall.input)\n      } catch {\n        return null\n      }\n\n      if (!parsed || typeof parsed !== \"object\") return null\n\n      const input = parsed as Record<string, unknown>\n\n      if (input.type === \"marker\") {\n        const repaired = {\n          type: \"marker\",\n          markerId: input.markerId,\n          label: input.label,\n        }\n\n        return {\n          ...toolCall,\n          input: JSON.stringify(repaired),\n        }\n      }\n\n      if (input.type === \"coordinates\") {\n        const repaired = {\n          type: \"coordinates\",\n          x: input.x,\n          y: input.y,\n          label: input.label,\n        }\n\n        return {\n          ...toolCall,\n          input: JSON.stringify(repaired),\n        }\n      }\n\n      return null\n    },\n  })\n\n  return result.toUIMessageStreamResponse()\n}\n","import { experimental_transcribe as transcribe } from \"ai\"\nimport type { CursorBuddyHandlerConfig, TranscribeResponse } from \"../types\"\n\n/**\n * Handle transcription requests: audio file → text\n */\nexport async function handleTranscribe(\n  request: Request,\n  config: CursorBuddyHandlerConfig,\n): Promise<Response> {\n  if (!config.transcriptionModel) {\n    return new Response(\n      JSON.stringify({\n        error:\n          \"Server transcription is not configured. Provide a transcriptionModel or use browser transcription only.\",\n      }),\n      {\n        status: 501,\n        headers: { \"Content-Type\": \"application/json\" },\n      },\n    )\n  }\n\n  const formData = await request.formData()\n  const audioFile = formData.get(\"audio\")\n\n  if (!audioFile || !(audioFile instanceof File)) {\n    return new Response(JSON.stringify({ error: \"No audio file provided\" }), {\n      status: 400,\n      headers: { \"Content-Type\": \"application/json\" },\n    })\n  }\n\n  const audioBuffer = await audioFile.arrayBuffer()\n\n  const result = await transcribe({\n    model: config.transcriptionModel,\n    audio: new Uint8Array(audioBuffer),\n  })\n\n  const response: TranscribeResponse = { text: result.text }\n\n  return new Response(JSON.stringify(response), {\n    headers: { \"Content-Type\": \"application/json\" },\n  })\n}\n","import { experimental_generateSpeech as generateSpeech } from \"ai\"\nimport type { CursorBuddyHandlerConfig, TTSRequestBody } from \"../types\"\n\n/**\n * Handle TTS requests: text → audio\n */\nexport async function handleTTS(\n  request: Request,\n  config: CursorBuddyHandlerConfig,\n): Promise<Response> {\n  if (!config.speechModel) {\n    return new Response(\n      JSON.stringify({\n        error:\n          \"Server speech is not configured. Provide a speechModel or use browser speech only.\",\n      }),\n      {\n        status: 501,\n        headers: { \"Content-Type\": \"application/json\" },\n      },\n    )\n  }\n\n  const outputFormat = \"wav\"\n  const body = (await request.json()) as TTSRequestBody\n  const { text } = body\n\n  if (!text) {\n    return new Response(JSON.stringify({ error: \"No text provided\" }), {\n      status: 400,\n      headers: { \"Content-Type\": \"application/json\" },\n    })\n  }\n\n  const result = await generateSpeech({\n    model: config.speechModel,\n    text,\n    outputFormat,\n  })\n\n  // Create a new ArrayBuffer copy to satisfy TypeScript's strict typing\n  const audioData = new Uint8Array(result.audio.uint8Array)\n\n  return new Response(audioData, {\n    headers: {\n      \"Content-Type\": \"audio/wav\",\n    },\n  })\n}\n","import { handleChat } from \"./routes/chat\"\nimport { handleTranscribe } from \"./routes/transcribe\"\nimport { handleTTS } from \"./routes/tts\"\nimport type { CursorBuddyHandler, CursorBuddyHandlerConfig } from \"./types\"\n\n/**\n * Create a cursor buddy request handler.\n *\n * The handler responds to three routes based on the last path segment:\n * - /chat - Screenshot + transcript → AI SSE stream\n * - /transcribe - Audio → text\n * - /tts - Text → audio\n *\n * @example\n * ```ts\n * import { createCursorBuddyHandler } from \"cursor-buddy/server\"\n * import { openai } from \"@ai-sdk/openai\"\n *\n * const cursorBuddy = createCursorBuddyHandler({\n *   model: openai(\"gpt-4o\"),\n *   speechModel: openai.speech(\"tts-1\"), // optional for browser-only speech\n *   transcriptionModel: openai.transcription(\"whisper-1\"),\n * })\n * ```\n */\nexport function createCursorBuddyHandler(\n  config: CursorBuddyHandlerConfig,\n): CursorBuddyHandler {\n  const handler = async (request: Request): Promise<Response> => {\n    const url = new URL(request.url)\n    const pathSegments = url.pathname.split(\"/\").filter(Boolean)\n    const route = pathSegments[pathSegments.length - 1]\n\n    switch (route) {\n      case \"chat\":\n        return handleChat(request, config)\n\n      case \"transcribe\":\n        return handleTranscribe(request, config)\n\n      case \"tts\":\n        return handleTTS(request, config)\n\n      default:\n        return new Response(\n          JSON.stringify({\n            error: \"Not found\",\n            availableRoutes: [\"/chat\", \"/transcribe\", \"/tts\"],\n          }),\n          {\n            status: 404,\n            headers: { \"Content-Type\": \"application/json\" },\n          },\n        )\n    }\n  }\n\n  return { handler, config }\n}\n"],"mappings":";;;AAAA,MAAa,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACQrC,eAAsB,WACpB,SACA,QACmB;CAEnB,MAAM,EAAE,YAAY,YAAY,SAAS,SAAS,kBADpC,MAAM,QAAQ,MAAM;CAIlC,MAAM,eACJ,OAAO,OAAO,WAAW,aACrB,OAAO,OAAO,EAAE,eAAe,uBAAuB,CAAC,GACtD,OAAO,UAAU;CAGxB,MAAM,eAAe,OAAO,cAAc,MAAM;CAChD,MAAM,iBAAiB,QAAQ,MAAM,CAAC,YAAY;CAGlD,MAAM,sBAAgC,EAAE;AAExC,KAAI,QACF,qBAAoB,KAClB,oBAAoB,QAAQ,MAAM,GAAG,QAAQ,OAAO,UACrD;AAGH,KAAI,cACF,qBAAoB,KAAK,IAAI,cAAc;CAG7C,MAAM,iBACJ,oBAAoB,SAAS,IAAI,oBAAoB,KAAK,KAAK,GAAG;CAGpE,MAAM,WAAW,CACf,GAAG,eAAe,KAAK,SAAS;EAC9B,MAAM,IAAI;EACV,SAAS,IAAI;EACd,EAAE,EACH;EACE,MAAM;EACN,SAAS;GACP,GAAI,iBACA,CACE;IACE,MAAM;IACN,MAAM;IACP,CACF,GACD,EAAE;GACN;IACE,MAAM;IACN,OAAO;IACR;GACD;IACE,MAAM;IACN,MAAM;IACP;GACF;EACF,CACF;AAwDD,QAtDe,WAAW;EACxB,OAAO,OAAO;EACd,QAAQ;EACR,iBAAiB,QAAQ;EACzB;EACA,OAAO;GACL,OAAO;GACP,GAAG,OAAO;GACX;EACD,6BAA6B,OAAO,EAAE,eAAe;AACnD,OAAI,SAAS,aAAa,QAAS,QAAO;GAE1C,IAAI;AACJ,OAAI;AACF,aAAS,KAAK,MAAM,SAAS,MAAM;WAC7B;AACN,WAAO;;AAGT,OAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;GAElD,MAAM,QAAQ;AAEd,OAAI,MAAM,SAAS,UAAU;IAC3B,MAAM,WAAW;KACf,MAAM;KACN,UAAU,MAAM;KAChB,OAAO,MAAM;KACd;AAED,WAAO;KACL,GAAG;KACH,OAAO,KAAK,UAAU,SAAS;KAChC;;AAGH,OAAI,MAAM,SAAS,eAAe;IAChC,MAAM,WAAW;KACf,MAAM;KACN,GAAG,MAAM;KACT,GAAG,MAAM;KACT,OAAO,MAAM;KACd;AAED,WAAO;KACL,GAAG;KACH,OAAO,KAAK,UAAU,SAAS;KAChC;;AAGH,UAAO;;EAEV,CAAC,CAEY,2BAA2B;;;;;;;ACtH3C,eAAsB,iBACpB,SACA,QACmB;AACnB,KAAI,CAAC,OAAO,mBACV,QAAO,IAAI,SACT,KAAK,UAAU,EACb,OACE,2GACH,CAAC,EACF;EACE,QAAQ;EACR,SAAS,EAAE,gBAAgB,oBAAoB;EAChD,CACF;CAIH,MAAM,aADW,MAAM,QAAQ,UAAU,EACd,IAAI,QAAQ;AAEvC,KAAI,CAAC,aAAa,EAAE,qBAAqB,MACvC,QAAO,IAAI,SAAS,KAAK,UAAU,EAAE,OAAO,0BAA0B,CAAC,EAAE;EACvE,QAAQ;EACR,SAAS,EAAE,gBAAgB,oBAAoB;EAChD,CAAC;CAGJ,MAAM,cAAc,MAAM,UAAU,aAAa;CAOjD,MAAM,WAA+B,EAAE,OALxB,MAAMA,wBAAW;EAC9B,OAAO,OAAO;EACd,OAAO,IAAI,WAAW,YAAY;EACnC,CAAC,EAEkD,MAAM;AAE1D,QAAO,IAAI,SAAS,KAAK,UAAU,SAAS,EAAE,EAC5C,SAAS,EAAE,gBAAgB,oBAAoB,EAChD,CAAC;;;;;;;ACtCJ,eAAsB,UACpB,SACA,QACmB;AACnB,KAAI,CAAC,OAAO,YACV,QAAO,IAAI,SACT,KAAK,UAAU,EACb,OACE,sFACH,CAAC,EACF;EACE,QAAQ;EACR,SAAS,EAAE,gBAAgB,oBAAoB;EAChD,CACF;CAGH,MAAM,eAAe;CAErB,MAAM,EAAE,SADM,MAAM,QAAQ,MAAM;AAGlC,KAAI,CAAC,KACH,QAAO,IAAI,SAAS,KAAK,UAAU,EAAE,OAAO,oBAAoB,CAAC,EAAE;EACjE,QAAQ;EACR,SAAS,EAAE,gBAAgB,oBAAoB;EAChD,CAAC;CAGJ,MAAM,SAAS,MAAMC,4BAAe;EAClC,OAAO,OAAO;EACd;EACA;EACD,CAAC;CAGF,MAAM,YAAY,IAAI,WAAW,OAAO,MAAM,WAAW;AAEzD,QAAO,IAAI,SAAS,WAAW,EAC7B,SAAS,EACP,gBAAgB,aACjB,EACF,CAAC;;;;;;;;;;;;;;;;;;;;;;;;ACtBJ,SAAgB,yBACd,QACoB;CACpB,MAAM,UAAU,OAAO,YAAwC;EAE7D,MAAM,eADM,IAAI,IAAI,QAAQ,IAAI,CACP,SAAS,MAAM,IAAI,CAAC,OAAO,QAAQ;AAG5D,UAFc,aAAa,aAAa,SAAS,IAEjD;GACE,KAAK,OACH,QAAO,WAAW,SAAS,OAAO;GAEpC,KAAK,aACH,QAAO,iBAAiB,SAAS,OAAO;GAE1C,KAAK,MACH,QAAO,UAAU,SAAS,OAAO;GAEnC,QACE,QAAO,IAAI,SACT,KAAK,UAAU;IACb,OAAO;IACP,iBAAiB;KAAC;KAAS;KAAe;KAAO;IAClD,CAAC,EACF;IACE,QAAQ;IACR,SAAS,EAAE,gBAAgB,oBAAoB;IAChD,CACF;;;AAIP,QAAO;EAAE;EAAS;EAAQ"}

package/dist/{types-BxBhjZju.d.mts → types-COQKMo5C.d.mts}

@@ -41,4 +41,4 @@ interface CursorBuddyHandler {
 }
 //#endregion
 export { CursorBuddyHandlerConfig as n, CursorBuddyHandler as t };
-//# sourceMappingURL=types-BxBhjZju.d.mts.map
+//# sourceMappingURL=types-COQKMo5C.d.mts.map

package/dist/{types-BxBhjZju.d.mts.map → types-COQKMo5C.d.mts.map}

@@ -1 +1 @@
-{"version":3,"file":"types-BxBhjZju.d.mts","names":[],"sources":["../src/server/types.ts"],"mappings":";;;;;AAKA;UAAiB,wBAAA;;EAEf,KAAA,EAAO,aAAA;EACP,qBAAA,GAAwB,MAAA;EAMV;;;;EAAd,WAAA,GAAc,WAAA;EAeA;;;;EATd,kBAAA,GAAqB,kBAAA;EANrB;;;;EAYA,MAAA,cAAoB,GAAA;IAAO,aAAA;EAAA;EAG3B;EAAA,KAAA,GAAQ,MAAA,SAAe,IAAA;EAAA;EAGvB,UAAA;AAAA;;AAMF;;UAAiB,kBAAA;EAEI;EAAnB,OAAA,GAAU,OAAA,EAAS,OAAA,KAAY,OAAA,CAAQ,QAAA;EAAR;EAG/B,MAAA,EAAQ,wBAAA;AAAA"}
+{"version":3,"file":"types-COQKMo5C.d.mts","names":[],"sources":["../src/server/types.ts"],"mappings":";;;;;AAKA;UAAiB,wBAAA;;EAEf,KAAA,EAAO,aAAA;EACP,qBAAA,GAAwB,MAAA;EAMV;;;;EAAd,WAAA,GAAc,WAAA;EAeA;;;;EATd,kBAAA,GAAqB,kBAAA;EANrB;;;;EAYA,MAAA,cAAoB,GAAA;IAAO,aAAA;EAAA;EAG3B;EAAA,KAAA,GAAQ,MAAA,SAAe,IAAA;EAAA;EAGvB,UAAA;AAAA;;AAMF;;UAAiB,kBAAA;EAEI;EAAnB,OAAA,GAAU,OAAA,EAAS,OAAA,KAAY,OAAA,CAAQ,QAAA;EAAR;EAG/B,MAAA,EAAQ,wBAAA;AAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cursor-buddy",
-  "version": "0.0.8",
+  "version": "0.0.9-beta.0",
   "description": "AI-powered cursor companion for web apps",
   "type": "module",
   "license": "MIT",
@@ -80,7 +80,8 @@
     "@nanostores/react": "^1.1.0",
     "ai": "^6.0.158",
     "html2canvas-pro": "^2.0.2",
-    "nanostores": "^1.2.0"
+    "nanostores": "^1.2.0",
+    "zod": "^3.24.0"
   },
   "devDependencies": {
     "@types/react": "^19.0.8",

package/dist/client-Crn8tW7w.d.mts.map

@@ -1 +0,0 @@
-{"version":3,"file":"client-Crn8tW7w.d.mts","names":[],"sources":["../src/core/utils/elements.ts","../src/core/types.ts","../src/core/client.ts"],"mappings":";;AAgEA;;;;;;UAAiB,aAAA;EAMf;EAJA,EAAA;EAMA;EAJA,OAAA,EAAS,OAAA;EAIE;EAFX,IAAA,EAAM,OAAA;EAQa;EANnB,WAAA;AAAA;;;;KAMU,SAAA,GAAY,GAAA,SAAY,aAAA;;;;AAdpC;;KC7DY,UAAA;;;;KAKA,UAAA;EACN,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;AAAA;EACA,IAAA;EAAe,KAAA,EAAO,KAAA;AAAA;;;AAV5B;;;;UAkBiB,cAAA;EAbL;EAeV,CAAA;;EAEA,CAAA;EAhBI;EAkBJ,KAAA;AAAA;;;AAaF;UAAiB,KAAA;EACf,CAAA;EACA,CAAA;AAAA;AAMF;;;AAAA,UAAiB,gBAAA;EAEf;EAAA,SAAA;EAIA;EAFA,KAAA;EAMA;EAJA,MAAA;EAIc;EAFd,aAAA;EAWyC;EATzC,cAAA;AAAA;;;;UASe,yBAAA,SAAkC,gBAAA;EAIpC;EAFb,SAAA,EAFyC,SAAA;EAkBX;EAd9B,aAAA;AAAA;;;;KAcU,oBAAA;;;;UAKK,8BAAA;EAgDA;;;;;;;;;;;;;EAlCf,IAAA,GAAO,oBAAA;AAAA;;;;UAMQ,uBAAA;EAsCA;;;;;;;;;;;EA1Bf,IAAA,GAAO,oBAAA;EA2BU;;;;;AAOnB;;;EAxBE,cAAA;AAAA;;;;UAMe,gBAAA;EACf,KAAA,IAAS,OAAA;EACT,IAAA,IAAQ,OAAA,CAAQ,IAAA;EAChB,OAAA,CAAQ,QAAA,GAAW,KAAA;EACnB,OAAA;AAAA;;AAyBF;;UAnBiB,iBAAA;EACf,IAAA,CAAK,IAAA,EAAM,IAAA,EAAM,MAAA,GAAS,WAAA,GAAc,OAAA;EACxC,IAAA;AAAA;;;;UAMe,qBAAA;EACf,WAAA;EACA,KAAA,IAAS,OAAA;EACT,IAAA,IAAQ,OAAA;EACR,SAAA,CAAU,QAAA,GAAW,IAAA;EACrB,OAAA;AAAA;;;;UAMe,iBAAA;EACf,WAAA;EACA,KAAA,CAAM,IAAA,UAAc,MAAA,GAAS,WAAA,GAAc,OAAA;EAC3C,IAAA;AAAA;;;;UAMe,iBAAA;EACf,OAAA,IAAW,OAAA,CAAQ,gBAAA;EACnB,gBAAA,IAAoB,OAAA,CAAQ,yBAAA;AAAA;;;;UAMb,qBAAA;EACf,OAAA,CAAQ,MAAA,EAAQ,cAAA;EAChB,OAAA;EACA,UAAA;EACA,SAAA,CAAU,QAAA;EACV,oBAAA;AAAA;;;;UAMe,mBAAA;EACf,YAAA,GAAe,gBAAA;EACf,aAAA,GAAgB,iBAAA;EAChB,iBAAA,GAAoB,qBAAA;EACpB,aAAA,GAAgB,iBAAA;EAChB,aAAA,GAAgB,iBAAA;EAChB,iBAAA,GAAoB,qBAAA;AAAA;;;;UAML,iBAAA;EAXA;EAaf,KAAA,EAAO,UAAA;EAZS;EAchB,UAAA;EAboB;EAepB,QAAA;EAdgB;EAgBhB,KAAA;AAAA;;;;UAMe,uBAAA;EAdA;EAgBf,IAAA;;EAEA,SAAA;EAhBA;EAkBA,OAAA;AAAA;;;;UAMe,mBAAA;EAZA;EAcf,UAAA;;EAEA,WAAA;AAAA;;;;UAMe,wBAAA;EAVA;;;;;EAgBf,aAAA,GAAgB,8BAAA;EANuB;;;;;;EAavC,MAAA,GAAS,uBAAA;EAUc;EARvB,YAAA,IAAgB,IAAA;EAThB;EAWA,UAAA,IAAc,IAAA;EAJd;EAMA,OAAA,IAAW,MAAA,EAAQ,cAAA;EAJnB;EAMA,aAAA,IAAiB,KAAA,EAAO,UAAA;EAJxB;EAMA,OAAA,IAAW,KAAA,EAAO,KAAA;AAAA;;;;UAMH,mBAAA;EARE;EAUjB,KAAA,EAAO,UAAA;EARW;;;;EAalB,cAAA;EAPkC;EASlC,UAAA;EAIY;EAFZ,QAAA;EATO;EAWP,KAAA,EAAO,KAAA;EAJP;EAMA,UAAA;EAFA;EAIA,SAAA;AAAA;;;;;;;;;;;;cC9LW,iBAAA;EAAA,QACH,QAAA;EAAA,QACA,OAAA;EAAA,QAGA,YAAA;EAAA,QACA,aAAA;EAAA,QACA,aAAA;EAAA,QACA,iBAAA;EAAA,QACA,aAAA;EAAA,QACA,iBAAA;EAAA,QACA,YAAA;EAAA,QAGA,cAAA;EAAA,QACA,UAAA;EAAA,QACA,QAAA;EAAA,QACA,KAAA;EAAA,QACA,eAAA;EAAA,QACA,uBAAA;EAAA,QACA,qBAAA;EAAA,QACA,iBAAA;EAAA,QAGA,cAAA;EAAA,QAGA,SAAA;cAGN,QAAA,UACA,OAAA,GAAS,wBAAA,EACT,QAAA,GAAU,mBAAA;EDxHR;;;;ECmKJ,cAAA,CAAA;EDjK+B;;AAQjC;EC8LQ,aAAA,CAAA,GAAiB,OAAA;;;;EAoIvB,UAAA,CAAW,OAAA;ED5TX;;;ECoUA,OAAA,CAAQ,CAAA,UAAW,CAAA,UAAW,KAAA;EDvTV;;;EC8TpB,eAAA,CAAA;EDtTe;;;EC6Tf,KAAA,CAAA;ED3TA;;;;EC2UA,oBAAA,CAAA;EDnUc;;AAShB;ECiUE,SAAA,CAAU,QAAA;;;;;EASV,WAAA,CAAA,GAAe,mBAAA;EDtUf;;;EAAA,QC6UQ,aAAA;EAAA,QAcA,KAAA;;;;ADxUV;;UC6VU,oBAAA;EAAA,QAcM,UAAA;ED7Va;AAM7B;;;EAN6B,QCmXb,YAAA;EDjWd;;;EAAA,QCwcc,gBAAA;ED9bA;AAMhB;;;;;;;;;EANgB,QC0dN,iBAAA;EDldR;;;;;;;EAAA,QC+ec,oBAAA;ED7eP;AAMT;;;EANS,QC+fO,uBAAA;EDxfY;;;EAAA,QCogBZ,wBAAA;EDpgBd;;;;;;;EAAA,QCkhBc,qBAAA;EDjhBV;AAMN;;;EANM,QC0jBI,qBAAA;EAAA,QAYA,WAAA;ED9jBR;;;EAAA,QCykBQ,oBAAA;EDvkBR;;;EAAA,QC8kBQ,aAAA;ED7kBD;;AAMT;EANS,QColBC,wBAAA;;;;UAOA,iCAAA;EDnlBF;;;EAAA,QC0lBE,8BAAA;EDzlBR;;;AAMF;;;EANE,QCmmBc,qBAAA;ED5lBH;;;;EAAA,QC0oBG,qBAAA;ED1oBd;;;;;;;;EAAA,QCuqBc,iBAAA;EAAA,QAmBN,cAAA;EAAA,QAOA,MAAA;AAAA"}