@predicatelabs/sdk 0.99.9

package/dist/visual-agent.d.ts

@@ -0,0 +1,120 @@
+/**
+ * Visual Agent - Uses labeled screenshots with vision-capable LLMs
+ *
+ * This agent extends SentienceAgent to use visual prompts:
+ * 1. Takes snapshot with screenshot enabled
+ * 2. Draws bounding boxes and labels element IDs on the screenshot
+ * 3. Uses anti-collision algorithm to position labels (4 sides + 4 corners)
+ * 4. Sends labeled screenshot to vision-capable LLM
+ * 5. Extracts element ID from LLM response
+ * 6. Clicks the element using click()
+ *
+ * Dependencies:
+ *    - sharp: Required for image processing
+ *      Install with: npm install sharp
+ *    - canvas: Required for drawing on images
+ *      Install with: npm install canvas
+ *    - Vision-capable LLM: Requires an LLM provider that supports vision (e.g., GPT-4o, Claude 3)
+ */
+import { SentienceBrowser } from './browser';
+import { SnapshotOptions } from './snapshot';
+import { LLMProvider } from './llm-provider';
+import { Tracer } from './tracing/tracer';
+import { SentienceAgent, AgentActResult } from './agent';
+/**
+ * Visual agent that uses labeled screenshots with vision-capable LLMs.
+ *
+ * Extends SentienceAgent to override act() method with visual prompting.
+ *
+ * Requirements:
+ *    - sharp: Required for image processing
+ *      Install with: npm install sharp
+ *    - canvas: Required for drawing on images
+ *      Install with: npm install canvas
+ *    - Vision-capable LLM: Requires an LLM provider that supports vision (e.g., GPT-4o, Claude 3)
+ */
+export declare class SentienceVisualAgent extends SentienceAgent {
+    /**
+     * Initialize Visual Agent
+     *
+     * @param browser - SentienceBrowser instance
+     * @param llm - LLM provider (must support vision, e.g., GPT-4o, Claude 3)
+     * @param snapshotLimit - Default maximum elements to include
+     * @param verbose - Print execution logs
+     * @param tracer - Optional Tracer instance
+     * @param showOverlay - Show green bbox overlay in browser
+     */
+    constructor(browser: SentienceBrowser, llm: LLMProvider, snapshotLimit?: number, verbose?: boolean, tracer?: Tracer, showOverlay?: boolean);
+    /**
+     * Decode base64 screenshot data URL to image buffer
+     *
+     * @param screenshotDataUrl - Base64-encoded data URL (e.g., "data:image/png;base64,...")
+     * @returns Image buffer
+     */
+    private decodeScreenshot;
+    /**
+     * Find best position for label using anti-collision algorithm.
+     *
+     * Tries 8 positions: 4 sides (top, bottom, left, right) + 4 corners.
+     * Returns the first position that doesn't collide with existing labels.
+     *
+     * @param elementBbox - Element bounding box {x, y, width, height}
+     * @param existingLabels - List of existing label bounding boxes
+     * @param imageWidth - Image width in pixels
+     * @param imageHeight - Image height in pixels
+     * @param labelWidth - Label width in pixels
+     * @param labelHeight - Label height in pixels
+     * @returns (x, y) position for label
+     */
+    private findLabelPosition;
+    /**
+     * Draw labeled screenshot with bounding boxes and element IDs.
+     *
+     * @param snapshot - Snapshot with screenshot data
+     * @param elements - List of elements to label
+     * @returns Image buffer with labels drawn
+     */
+    private drawLabeledScreenshot;
+    /**
+     * Encode image buffer to base64 data URL with size optimization.
+     *
+     * Vision LLM APIs typically have size limits (e.g., 20MB for OpenAI).
+     * This function automatically compresses images if they're too large.
+     *
+     * @param imageBuffer - Image buffer
+     * @param format - Image format ('PNG' or 'JPEG')
+     * @param maxSizeMb - Maximum size in MB before compression (default: 20MB)
+     * @returns Base64-encoded data URL
+     */
+    private encodeImageToBase64;
+    /**
+     * Query LLM with vision (labeled screenshot).
+     *
+     * @param imageDataUrl - Base64-encoded image data URL
+     * @param goal - User's goal/task
+     * @returns LLMResponse with element ID
+     */
+    private queryLLMWithVision;
+    /**
+     * Extract element ID integer from LLM response.
+     *
+     * @param llmResponse - LLM response text
+     * @returns Element ID as integer, or undefined if not found
+     */
+    private extractElementId;
+    /**
+     * Override act() method to use visual prompting with full tracing support.
+     *
+     * @param goal - User's goal/task
+     * @param maxRetries - Maximum retry attempts
+     * @param snapshotOptions - Optional snapshot options (screenshot will be enabled)
+     * @returns AgentActResult
+     */
+    act(goal: string, _maxRetries?: number, snapshotOptions?: SnapshotOptions): Promise<AgentActResult>;
+}
+/**
+ * Predicate rebrand alias for SentienceVisualAgent.
+ * Kept as a runtime alias to avoid breaking existing integrations.
+ */
+export declare const PredicateVisualAgent: typeof SentienceVisualAgent;
+//# sourceMappingURL=visual-agent.d.ts.map

package/dist/visual-agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"visual-agent.d.ts","sourceRoot":"","sources":["../src/visual-agent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAY,eAAe,EAAE,MAAM,YAAY,CAAC;AAEvD,OAAO,EAAE,WAAW,EAAe,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAM1C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AA6BzD;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,cAAc;IACtD;;;;;;;;;OASG;gBAED,OAAO,EAAE,gBAAgB,EACzB,GAAG,EAAE,WAAW,EAChB,aAAa,GAAE,MAAW,EAC1B,OAAO,GAAE,OAAc,EACvB,MAAM,CAAC,EAAE,MAAM,EACf,WAAW,GAAE,OAAe;IAc9B;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAYxB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,iBAAiB;IAyEzB;;;;;;OAMG;YACW,qBAAqB;IA4InC;;;;;;;;;;OAUG;YACW,mBAAmB;IA6CjC;;;;;;OAMG;YACW,kBAAkB;IAyDhC;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAqFxB;;;;;;;OAOG;IACG,GAAG,CACP,IAAI,EAAE,MAAM,EACZ,WAAW,GAAE,MAAU,EACvB,eAAe,CAAC,EAAE,eAAe,GAChC,OAAO,CAAC,cAAc,CAAC;CA2U3B;AAED;;;GAGG;AACH,eAAO,MAAM,oBAAoB,6BAAuB,CAAC"}