@ai11y/core 0.0.1

package/dist/util/attributes.d.mts

@@ -0,0 +1,57 @@
+//#region src/util/attributes.d.ts
+/**
+ * Attribute name constants for data-ai-* attributes
+ */
+declare const ATTRIBUTE_ID = "data-ai-id";
+declare const ATTRIBUTE_LABEL = "data-ai-label";
+declare const ATTRIBUTE_INTENT = "data-ai-intent";
+declare const ATTRIBUTE_SENSITIVE = "data-ai-sensitive";
+/**
+ * Gets the value of the data-ai-id attribute from an element
+ *
+ * @param element - The element to get the attribute from
+ * @returns The marker ID or null if not found
+ */
+declare function getMarkerId(element: Element): string | null;
+/**
+ * Gets the value of the data-ai-label attribute from an element
+ *
+ * @param element - The element to get the attribute from
+ * @returns The marker label or null if not found
+ */
+declare function getMarkerLabel(element: Element): string | null;
+/**
+ * Gets the value of the data-ai-intent attribute from an element
+ *
+ * @param element - The element to get the attribute from
+ * @returns The marker intent or null if not found
+ */
+declare function getMarkerIntent(element: Element): string | null;
+/**
+ * Creates a query selector for finding an element by marker ID
+ *
+ * @param markerId - The marker ID to search for
+ * @returns A query selector string
+ *
+ * @example
+ * ```ts
+ * const selector = getMarkerSelector('connect_stripe');
+ * // Returns: '[data-ai-id="connect_stripe"]'
+ * ```
+ */
+declare function getMarkerSelector(markerId: string): string;
+/**
+ * Creates a query selector for finding all elements with marker IDs
+ *
+ * @returns A query selector string
+ *
+ * @example
+ * ```ts
+ * const selector = getAllMarkersSelector();
+ * // Returns: '[data-ai-id]'
+ * ```
+ */
+declare function getAllMarkersSelector(): string;
+//#endregion
+export { ATTRIBUTE_ID, ATTRIBUTE_INTENT, ATTRIBUTE_LABEL, ATTRIBUTE_SENSITIVE, getAllMarkersSelector, getMarkerId, getMarkerIntent, getMarkerLabel, getMarkerSelector };
+//# sourceMappingURL=attributes.d.mts.map

package/dist/util/attributes.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"attributes.d.mts","names":[],"sources":["../../src/util/attributes.ts"],"sourcesContent":[],"mappings":";;AAGA;AACA;AACa,cAFA,YAAA,GAEgB,YAAA;AAChB,cAFA,eAAA,GAEmB,eAAA;AAQhB,cATH,gBAAA,GAS+B,gBAAA;AAU5B,cAlBH,mBAAA,GAkBkC,mBAAA;AAU/C;AAgBA;AAeA;;;;iBAnDgB,WAAA,UAAqB;;;;;;;iBAUrB,cAAA,UAAwB;;;;;;;iBAUxB,eAAA,UAAyB;;;;;;;;;;;;;iBAgBzB,iBAAA;;;;;;;;;;;;iBAeA,qBAAA,CAAA"}

package/dist/util/attributes.mjs

@@ -0,0 +1,68 @@
+//#region src/util/attributes.ts
+/**
+* Attribute name constants for data-ai-* attributes
+*/
+const ATTRIBUTE_ID = "data-ai-id";
+const ATTRIBUTE_LABEL = "data-ai-label";
+const ATTRIBUTE_INTENT = "data-ai-intent";
+const ATTRIBUTE_SENSITIVE = "data-ai-sensitive";
+/**
+* Gets the value of the data-ai-id attribute from an element
+*
+* @param element - The element to get the attribute from
+* @returns The marker ID or null if not found
+*/
+function getMarkerId(element) {
+	return element.getAttribute(ATTRIBUTE_ID);
+}
+/**
+* Gets the value of the data-ai-label attribute from an element
+*
+* @param element - The element to get the attribute from
+* @returns The marker label or null if not found
+*/
+function getMarkerLabel(element) {
+	return element.getAttribute(ATTRIBUTE_LABEL);
+}
+/**
+* Gets the value of the data-ai-intent attribute from an element
+*
+* @param element - The element to get the attribute from
+* @returns The marker intent or null if not found
+*/
+function getMarkerIntent(element) {
+	return element.getAttribute(ATTRIBUTE_INTENT);
+}
+/**
+* Creates a query selector for finding an element by marker ID
+*
+* @param markerId - The marker ID to search for
+* @returns A query selector string
+*
+* @example
+* ```ts
+* const selector = getMarkerSelector('connect_stripe');
+* // Returns: '[data-ai-id="connect_stripe"]'
+* ```
+*/
+function getMarkerSelector(markerId) {
+	return `[${ATTRIBUTE_ID}="${markerId}"]`;
+}
+/**
+* Creates a query selector for finding all elements with marker IDs
+*
+* @returns A query selector string
+*
+* @example
+* ```ts
+* const selector = getAllMarkersSelector();
+* // Returns: '[data-ai-id]'
+* ```
+*/
+function getAllMarkersSelector() {
+	return `[${ATTRIBUTE_ID}]`;
+}
+
+//#endregion
+export { ATTRIBUTE_ID, ATTRIBUTE_INTENT, ATTRIBUTE_LABEL, ATTRIBUTE_SENSITIVE, getAllMarkersSelector, getMarkerId, getMarkerIntent, getMarkerLabel, getMarkerSelector };
+//# sourceMappingURL=attributes.mjs.map

package/dist/util/attributes.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"attributes.mjs","names":[],"sources":["../../src/util/attributes.ts"],"sourcesContent":["/**\n * Attribute name constants for data-ai-* attributes\n */\nexport const ATTRIBUTE_ID = \"data-ai-id\";\nexport const ATTRIBUTE_LABEL = \"data-ai-label\";\nexport const ATTRIBUTE_INTENT = \"data-ai-intent\";\nexport const ATTRIBUTE_SENSITIVE = \"data-ai-sensitive\";\n\n/**\n * Gets the value of the data-ai-id attribute from an element\n *\n * @param element - The element to get the attribute from\n * @returns The marker ID or null if not found\n */\nexport function getMarkerId(element: Element): string | null {\n\treturn element.getAttribute(ATTRIBUTE_ID);\n}\n\n/**\n * Gets the value of the data-ai-label attribute from an element\n *\n * @param element - The element to get the attribute from\n * @returns The marker label or null if not found\n */\nexport function getMarkerLabel(element: Element): string | null {\n\treturn element.getAttribute(ATTRIBUTE_LABEL);\n}\n\n/**\n * Gets the value of the data-ai-intent attribute from an element\n *\n * @param element - The element to get the attribute from\n * @returns The marker intent or null if not found\n */\nexport function getMarkerIntent(element: Element): string | null {\n\treturn element.getAttribute(ATTRIBUTE_INTENT);\n}\n\n/**\n * Creates a query selector for finding an element by marker ID\n *\n * @param markerId - The marker ID to search for\n * @returns A query selector string\n *\n * @example\n * ```ts\n * const selector = getMarkerSelector('connect_stripe');\n * // Returns: '[data-ai-id=\"connect_stripe\"]'\n * ```\n */\nexport function getMarkerSelector(markerId: string): string {\n\treturn `[${ATTRIBUTE_ID}=\"${markerId}\"]`;\n}\n\n/**\n * Creates a query selector for finding all elements with marker IDs\n *\n * @returns A query selector string\n *\n * @example\n * ```ts\n * const selector = getAllMarkersSelector();\n * // Returns: '[data-ai-id]'\n * ```\n */\nexport function getAllMarkersSelector(): string {\n\treturn `[${ATTRIBUTE_ID}]`;\n}\n"],"mappings":";;;;AAGA,MAAa,eAAe;AAC5B,MAAa,kBAAkB;AAC/B,MAAa,mBAAmB;AAChC,MAAa,sBAAsB;;;;;;;AAQnC,SAAgB,YAAY,SAAiC;AAC5D,QAAO,QAAQ,aAAa,aAAa;;;;;;;;AAS1C,SAAgB,eAAe,SAAiC;AAC/D,QAAO,QAAQ,aAAa,gBAAgB;;;;;;;;AAS7C,SAAgB,gBAAgB,SAAiC;AAChE,QAAO,QAAQ,aAAa,iBAAiB;;;;;;;;;;;;;;AAe9C,SAAgB,kBAAkB,UAA0B;AAC3D,QAAO,IAAI,aAAa,IAAI,SAAS;;;;;;;;;;;;;AActC,SAAgB,wBAAgC;AAC/C,QAAO,IAAI,aAAa"}

package/dist/util/format.d.mts

@@ -0,0 +1,18 @@
+//#region src/util/format.d.ts
+/**
+ * Formats a marker ID into a readable label
+ * Converts snake_case to Title Case
+ *
+ * @param id - The marker ID to format
+ * @returns A formatted label
+ *
+ * @example
+ * ```ts
+ * formatMarkerId('connect_stripe');
+ * // Returns: 'Connect Stripe'
+ * ```
+ */
+declare function formatMarkerId(id: string): string;
+//#endregion
+export { formatMarkerId };
+//# sourceMappingURL=format.d.mts.map

package/dist/util/format.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.d.mts","names":[],"sources":["../../src/util/format.ts"],"sourcesContent":[],"mappings":";;AAaA;;;;;;;;;;;;iBAAgB,cAAA"}

package/dist/util/format.mjs

@@ -0,0 +1,21 @@
+//#region src/util/format.ts
+/**
+* Formats a marker ID into a readable label
+* Converts snake_case to Title Case
+*
+* @param id - The marker ID to format
+* @returns A formatted label
+*
+* @example
+* ```ts
+* formatMarkerId('connect_stripe');
+* // Returns: 'Connect Stripe'
+* ```
+*/
+function formatMarkerId(id) {
+	return id.replace(/_/g, " ").replace(/\b\w/g, (char) => char.toUpperCase());
+}
+
+//#endregion
+export { formatMarkerId };
+//# sourceMappingURL=format.mjs.map

package/dist/util/format.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.mjs","names":[],"sources":["../../src/util/format.ts"],"sourcesContent":["/**\n * Formats a marker ID into a readable label\n * Converts snake_case to Title Case\n *\n * @param id - The marker ID to format\n * @returns A formatted label\n *\n * @example\n * ```ts\n * formatMarkerId('connect_stripe');\n * // Returns: 'Connect Stripe'\n * ```\n */\nexport function formatMarkerId(id: string): string {\n\treturn id.replace(/_/g, \" \").replace(/\\b\\w/g, (char) => char.toUpperCase());\n}\n"],"mappings":";;;;;;;;;;;;;;AAaA,SAAgB,eAAe,IAAoB;AAClD,QAAO,GAAG,QAAQ,MAAM,IAAI,CAAC,QAAQ,UAAU,SAAS,KAAK,aAAa,CAAC"}

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@ai11y/core",
+  "version": "0.0.1",
+  "type": "module",
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "devDependencies": {
+    "tsdown": "0.18.1",
+    "typescript": "5.9.3"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "watch": "tsdown --watch"
+  }
+}

package/src/agent/agent-adapter.ts

@@ -0,0 +1,75 @@
+import type { Ai11yContext } from "../context.js";
+import { runLLMAgent } from "./llm-agent.js";
+import { runRuleBasedAgent } from "./rule-based-agent.js";
+import type {
+	AgentAdapterConfig,
+	AgentResponse,
+	ConversationMessage,
+} from "./types.js";
+
+/**
+ * Detect if we're likely offline (browser only)
+ */
+function isLikelyOffline(): boolean {
+	if (typeof navigator !== "undefined" && "onLine" in navigator) {
+		return !navigator.onLine;
+	}
+	return false;
+}
+
+/**
+ * Select and run the appropriate agent based on configuration.
+ *
+ * This adapter handles agent selection logic:
+ * - "llm": Always use LLM agent (requires llmConfig)
+ * - "rule-based": Always use rule-based agent
+ * - "auto": Use LLM if configured, fallback to rule-based on error or when offline
+ */
+export async function runAgentAdapter(
+	input: string,
+	context: Ai11yContext,
+	config: AgentAdapterConfig = {},
+	messages?: ConversationMessage[],
+): Promise<AgentResponse> {
+	const { mode = "auto", llmConfig = null, forceRuleBased = false } = config;
+
+	// Force rule-based agent if explicitly requested
+	if (forceRuleBased || mode === "rule-based") {
+		return runRuleBasedAgent(input, context);
+	}
+
+	if (mode === "llm") {
+		if (!llmConfig) {
+			console.warn(
+				"LLM mode requested but no llmConfig provided. Falling back to rule-based agent.",
+			);
+			return runRuleBasedAgent(input, context);
+		}
+		return runLLMAgent(input, context, llmConfig, messages);
+	}
+
+	// Auto mode: try LLM if available, fallback to rule-based
+	if (mode === "auto" || !mode) {
+		// Check if we're offline - use rule-based
+		if (isLikelyOffline()) {
+			return runRuleBasedAgent(input, context);
+		}
+
+		// Try LLM if configured
+		if (llmConfig) {
+			try {
+				return await runLLMAgent(input, context, llmConfig, messages);
+			} catch (error) {
+				// runLLMAgent already falls back to rule-based, but we catch here for safety
+				console.warn("LLM agent failed, using rule-based fallback:", error);
+				return runRuleBasedAgent(input, context);
+			}
+		}
+
+		// Default to rule-based
+		return runRuleBasedAgent(input, context);
+	}
+
+	// Fallback (shouldn't reach here)
+	return runRuleBasedAgent(input, context);
+}

package/src/agent/index.ts

@@ -0,0 +1,21 @@
+/**
+ * Agent module - provides different agent implementations
+ *
+ * Agents:
+ * - runRuleBasedAgent: Pattern-matching agent for common commands (works offline)
+ * - runLLMAgent: LLM-based agent using server-side API
+ * - runAgentAdapter: Unified adapter that selects the appropriate agent
+ */
+
+export { runAgentAdapter } from "./agent-adapter.js";
+export { runLLMAgent } from "./llm-agent.js";
+export { runRuleBasedAgent } from "./rule-based-agent.js";
+export type {
+	AgentAdapterConfig,
+	AgentConfig,
+	AgentMode,
+	AgentRequest,
+	AgentResponse,
+	ConversationMessage,
+	LLMAgentConfig,
+} from "./types.js";

package/src/agent/llm-agent.ts

@@ -0,0 +1,64 @@
+import type { Ai11yContext } from "../context.js";
+import type {
+	AgentResponse,
+	ConversationMessage,
+	LLMAgentConfig,
+} from "./types.js";
+
+/**
+ * LLM-based agent using server-side API endpoint.
+ * Falls back to rule-based agent if the API is not available.
+ */
+export async function runLLMAgent(
+	input: string,
+	context: Ai11yContext,
+	config: LLMAgentConfig,
+	messages?: ConversationMessage[],
+): Promise<AgentResponse> {
+	try {
+		const response = await fetch(config.apiEndpoint, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+			},
+			body: JSON.stringify({
+				input,
+				context: {
+					markers: context.markers,
+					...(context.inViewMarkerIds !== undefined && {
+						inViewMarkerIds: context.inViewMarkerIds,
+					}),
+					...(context.route !== undefined && { route: context.route }),
+					...(context.state !== undefined && { state: context.state }),
+					...(context.error !== undefined && {
+						error: context.error
+							? {
+									error: {
+										message: context.error.error.message,
+									},
+									meta: context.error.meta,
+									timestamp: context.error.timestamp,
+								}
+							: null,
+					}),
+				},
+				messages: messages?.slice(-10),
+			}),
+		});
+
+		if (!response.ok) {
+			const errorText = await response.text();
+			throw new Error(
+				`API request failed: ${response.status} ${response.statusText}. ${errorText}`,
+			);
+		}
+
+		const data: AgentResponse = await response.json();
+		return data;
+	} catch (error) {
+		// If API is not available or there's an error, fall back to rule-based
+		console.warn("LLM agent error, falling back to rule-based agent:", error);
+		const { runRuleBasedAgent } = await import("./rule-based-agent.js");
+		return runRuleBasedAgent(input, context);
+	}
+}

package/src/agent/plan.ts

@@ -0,0 +1,41 @@
+import type { Ai11yContext } from "../context.js";
+import { runAgentAdapter } from "./agent-adapter.js";
+import type {
+	AgentAdapterConfig,
+	AgentResponse,
+	ConversationMessage,
+} from "./types.js";
+
+/**
+ * Plans instructions based on UI context and user input
+ *
+ * @param ui - The current UI context
+ * @param input - User input/command
+ * @param config - Optional agent configuration
+ * @param messages - Optional conversation history
+ * @returns The full agent response containing both reply and instructions
+ *
+ * @example
+ * ```ts
+ * const ui = ctx.describe()
+ * const response = await plan(ui, "click save button", config)
+ * for (const instruction of response.instructions || []) {
+ *   ctx.act(instruction)
+ * }
+ * ```
+ */
+export async function plan(
+	ui: Ai11yContext,
+	input: string,
+	config?: AgentAdapterConfig,
+	messages?: ConversationMessage[],
+): Promise<AgentResponse> {
+	const response: AgentResponse = await runAgentAdapter(
+		input,
+		ui,
+		config,
+		messages,
+	);
+
+	return response;
+}

package/src/agent/rule-based-agent.ts

@@ -0,0 +1,269 @@
+import type { Ai11yContext } from "../context.js";
+import type { Instruction } from "../instruction.js";
+import type { AgentResponse } from "./types.js";
+
+/**
+ * Rule-based agent that uses pattern matching for common commands.
+ * Provides immediate responses without requiring an LLM.
+ */
+export function runRuleBasedAgent(
+	input: string,
+	context: Ai11yContext,
+): AgentResponse {
+	const lowerInput = input.toLowerCase().trim();
+
+	// Handle navigation commands
+	if (
+		lowerInput.includes("go to") ||
+		lowerInput.includes("navigate to") ||
+		lowerInput.includes("open") ||
+		lowerInput.includes("take me to")
+	) {
+		// First, check if input matches any marker
+		const searchText = lowerInput
+			.replace(/go to|navigate to|open|take me to/g, "")
+			.trim();
+		const matchingMarker = context.markers.find((m) => {
+			const markerText = `${m.label} ${m.intent}`.toLowerCase();
+			return (
+				markerText.includes(searchText) ||
+				lowerInput.includes(m.label.toLowerCase()) ||
+				lowerInput.includes(m.intent.toLowerCase())
+			);
+		});
+
+		// If a marker matches, decide based on element type and visibility
+		if (matchingMarker) {
+			const isInView =
+				context.inViewMarkerIds?.includes(matchingMarker.id) ?? false;
+			const isLink = matchingMarker.elementType === "a";
+
+			// If it's a link marker and in view, click it (navigate)
+			if (isLink && isInView) {
+				return {
+					reply: `Navigating to ${matchingMarker.label}...`,
+					instructions: [{ action: "click", id: matchingMarker.id }],
+				};
+			}
+
+			// If it's a link marker but not in view, scroll to it first
+			if (isLink && !isInView) {
+				return {
+					reply: `Scrolling to ${matchingMarker.label}...`,
+					instructions: [{ action: "scroll", id: matchingMarker.id }],
+				};
+			}
+
+			// For non-link markers (sections, divs), scroll to them
+			return {
+				reply: `Scrolling to ${matchingMarker.label}...`,
+				instructions: [{ action: "scroll", id: matchingMarker.id }],
+			};
+		}
+
+		// No marker found, proceed with route navigation
+		const instructions: Instruction[] = [];
+
+		// Extract route from input
+		let route = "/";
+		if (lowerInput.includes("billing")) {
+			route = "/billing";
+		} else if (lowerInput.includes("integrations")) {
+			route = "/integrations";
+		} else if (lowerInput.includes("home")) {
+			route = "/";
+		}
+
+		if (route !== context.route) {
+			instructions.push({ action: "navigate", route });
+			return {
+				reply: `Navigating to ${route === "/" ? "home" : route}...`,
+				instructions,
+			};
+		} else {
+			return {
+				reply: `You're already on ${route === "/" ? "home" : route}.`,
+			};
+		}
+	}
+
+	// Handle click commands
+	if (lowerInput.includes("click") || lowerInput.includes("press")) {
+		// Find matching marker
+		const matchingMarker = context.markers.find((m) => {
+			const markerText = `${m.label} ${m.intent}`.toLowerCase();
+			return (
+				markerText.includes(lowerInput.replace(/click|press/g, "").trim()) ||
+				lowerInput.includes(m.label.toLowerCase()) ||
+				lowerInput.includes(m.intent.toLowerCase())
+			);
+		});
+
+		if (matchingMarker) {
+			return {
+				reply: `Clicking ${matchingMarker.label}...`,
+				instructions: [{ action: "click", id: matchingMarker.id }],
+			};
+		} else {
+			return {
+				reply: `I couldn't find a button or element matching "${input}". Available options: ${context.markers.map((m) => m.label).join(", ") || "none"}`,
+			};
+		}
+	}
+
+	// Handle scroll commands (scroll to element)
+	if (
+		lowerInput.includes("scroll to") ||
+		lowerInput.includes("scroll") ||
+		lowerInput.includes("show me")
+	) {
+		const matchingMarker = context.markers.find((m) => {
+			const markerText = `${m.label} ${m.intent}`.toLowerCase();
+			const searchText = lowerInput
+				.replace(/scroll to|scroll|show me/g, "")
+				.trim();
+			return (
+				markerText.includes(searchText) ||
+				lowerInput.includes(m.label.toLowerCase())
+			);
+		});
+
+		if (matchingMarker) {
+			// If user explicitly wants to scroll, use scroll tool
+			// Otherwise, if they say "show me" or "highlight", use highlight (which now scrolls)
+			if (lowerInput.includes("scroll")) {
+				return {
+					reply: `Scrolling to ${matchingMarker.label}...`,
+					instructions: [{ action: "scroll", id: matchingMarker.id }],
+				};
+			}
+		}
+	}
+
+	// Handle highlight commands
+	if (lowerInput.includes("highlight") || lowerInput.includes("show")) {
+		const matchingMarker = context.markers.find((m) => {
+			const markerText = `${m.label} ${m.intent}`.toLowerCase();
+			return (
+				markerText.includes(lowerInput.replace(/highlight|show/g, "").trim()) ||
+				lowerInput.includes(m.label.toLowerCase())
+			);
+		});
+
+		if (matchingMarker) {
+			return {
+				reply: `Highlighting ${matchingMarker.label}...`,
+				instructions: [{ action: "highlight", id: matchingMarker.id }],
+			};
+		}
+	}
+
+	// Handle fill input commands
+	if (
+		lowerInput.includes("fill") ||
+		lowerInput.includes("enter") ||
+		lowerInput.includes("type") ||
+		lowerInput.includes("put")
+	) {
+		// Try to extract value from commands like:
+		// "fill email with test@example.com"
+		// "enter test@example.com in email"
+		// "type test@example.com in email input"
+		// "put test@example.com in email"
+		let value: string | undefined;
+		let markerSearchText = lowerInput;
+
+		// Pattern: "fill [marker] with [value]" or "fill [marker] [value]"
+		const fillWithMatch = lowerInput.match(
+			/(?:fill|enter|type|put)\s+(.+?)\s+(?:with|in|into)\s+(.+)/i,
+		);
+		if (fillWithMatch) {
+			markerSearchText = fillWithMatch[1].trim();
+			value = fillWithMatch[2].trim();
+		} else {
+			// Pattern: "fill [value] in [marker]" or "enter [value] in [marker]"
+			const fillInMatch = lowerInput.match(
+				/(?:fill|enter|type|put)\s+(.+?)\s+in\s+(.+)/i,
+			);
+			if (fillInMatch) {
+				value = fillInMatch[1].trim();
+				markerSearchText = fillInMatch[2].trim();
+			} else {
+				// Pattern: "[value] in [marker]"
+				const inMatch = lowerInput.match(/(.+?)\s+in\s+(.+)/i);
+				if (inMatch) {
+					value = inMatch[1].trim();
+					markerSearchText = inMatch[2].trim();
+				}
+			}
+		}
+
+		// Find matching marker (look for input-like elements)
+		const matchingMarker = context.markers.find((m) => {
+			const markerText = `${m.label} ${m.intent}`.toLowerCase();
+			const cleanedMarkerSearch = markerSearchText
+				.replace(/fill|enter|type|put|input|field/g, "")
+				.trim();
+			return (
+				markerText.includes(cleanedMarkerSearch) ||
+				cleanedMarkerSearch.includes(m.label.toLowerCase()) ||
+				cleanedMarkerSearch.includes(m.id.toLowerCase())
+			);
+		});
+
+		if (matchingMarker && value) {
+			return {
+				reply: `Filling ${matchingMarker.label} with "${value}"...`,
+				instructions: [
+					{
+						action: "fillInput",
+						id: matchingMarker.id,
+						value: value,
+					},
+				],
+			};
+		} else if (matchingMarker && !value) {
+			return {
+				reply: `I found ${matchingMarker.label}, but I need a value to fill. Please specify what to enter, for example: "fill ${matchingMarker.label.toLowerCase()} with [value]"`,
+			};
+		}
+	}
+
+	// Handle error context
+	if (context.error) {
+		const error = context.error.error;
+		const errorMessage = error.message || "An error occurred";
+
+		if (
+			lowerInput.includes("retry") ||
+			lowerInput.includes("try again") ||
+			lowerInput.includes("fix")
+		) {
+			// Try to find a retry action
+			if (context.error.meta?.markerId) {
+				const marker = context.markers.find(
+					(m) => m.id === context.error?.meta?.markerId,
+				);
+				if (marker) {
+					return {
+						reply: `Retrying ${marker.label}...`,
+						instructions: [{ action: "click", id: marker.id }],
+					};
+				}
+			}
+
+			return {
+				reply: `I can help you retry. The error was: ${errorMessage}. Would you like me to try clicking the button again?`,
+			};
+		}
+
+		return {
+			reply: `I noticed an error: ${errorMessage}. You can ask me to "retry" or "try again" to attempt the action once more.`,
+		};
+	}
+
+	// Default response
+	return {
+		reply: `I can help you navigate, click buttons, highlight elements, or fill inputs. Try saying "go to billing", "click connect stripe", "show me the enable billing button", or "fill email with test@example.com".`,
+	};
+}