illuma-agents 1.0.41 → 1.0.43

package/dist/types/tools/DesktopTools.d.ts

@@ -0,0 +1,104 @@
+import { DynamicStructuredTool } from '@langchain/core/tools';
+/**
+ * Desktop tool names - keep in sync with Ranger Desktop Electron app
+ * These tools execute locally in the Electron app, NOT on the server
+ */
+export declare const EDesktopTools: {
+    readonly SCREENSHOT: "computer_screenshot";
+    readonly CLICK: "computer_click";
+    readonly DOUBLE_CLICK: "computer_double_click";
+    readonly RIGHT_CLICK: "computer_right_click";
+    readonly TYPE: "computer_type";
+    readonly KEY: "computer_key";
+    readonly KEY_COMBO: "computer_key_combo";
+    readonly SCROLL: "computer_scroll";
+    readonly DRAG: "computer_drag";
+    readonly GET_ACTIVE_WINDOW: "computer_get_active_window";
+    readonly GET_MOUSE_POSITION: "computer_get_mouse_position";
+    readonly CLIPBOARD_READ: "clipboard_read";
+    readonly CLIPBOARD_WRITE: "clipboard_write";
+    readonly CLIPBOARD_PASTE: "clipboard_paste";
+    readonly WAIT: "computer_wait";
+};
+export type DesktopToolName = (typeof EDesktopTools)[keyof typeof EDesktopTools];
+/**
+ * Callback function type for waiting on desktop action results
+ * This allows the server (Ranger) to provide a callback that waits for the Electron app
+ * to POST results back to the server before returning to the LLM.
+ *
+ * @param action - The desktop action (click, type, screenshot, etc.)
+ * @param args - Arguments for the action
+ * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)
+ * @returns Promise that resolves with the actual desktop result
+ */
+export type DesktopToolCallback = (action: string, args: Record<string, unknown>, toolCallId: string) => Promise<DesktopActionResult>;
+/**
+ * Result returned from desktop action execution
+ */
+export interface DesktopActionResult {
+    success: boolean;
+    error?: string;
+    screenshot?: {
+        base64: string;
+        width: number;
+        height: number;
+    };
+    activeWindow?: {
+        title: string;
+        app: string;
+        bounds?: {
+            x: number;
+            y: number;
+            width: number;
+            height: number;
+        };
+    };
+    mousePosition?: {
+        x: number;
+        y: number;
+    };
+    clipboard?: string;
+}
+/**
+ * Check if desktop capability is available based on request headers or context
+ * The Ranger Desktop Electron app sets these headers when connected:
+ * - X-Ranger-Desktop: true
+ * - X-Ranger-Desktop-Capable: true
+ */
+export declare function hasDesktopCapability(req?: {
+    headers?: Record<string, string | string[] | undefined>;
+}): boolean;
+/**
+ * Desktop tool response interface
+ * This is what the Electron app returns after executing the action
+ */
+export interface DesktopToolResponse {
+    requiresDesktopExecution: true;
+    action: string;
+    args: Record<string, unknown>;
+    toolCallId?: string;
+}
+/**
+ * Options for creating desktop tools
+ */
+export interface CreateDesktopToolsOptions {
+    /**
+     * Optional callback that waits for desktop action results.
+     * When provided, tools will await this callback to get actual results from the Electron app.
+     * When not provided, tools return markers immediately (for non-server contexts).
+     */
+    waitForResult?: DesktopToolCallback;
+}
+/**
+ * Create desktop automation tools for the agent
+ * These tools allow AI to control the user's desktop when Ranger Desktop is running
+ */
+export declare function createDesktopTools(options?: CreateDesktopToolsOptions): DynamicStructuredTool[];
+/**
+ * Get all desktop tool names
+ */
+export declare function getDesktopToolNames(): DesktopToolName[];
+/**
+ * Check if a tool name is a desktop tool
+ */
+export declare function isDesktopTool(name: string): name is DesktopToolName;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "illuma-agents",
-  "version": "1.0.41",
+  "version": "1.0.43",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/graphs/Graph.ts

@@ -1071,9 +1071,31 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
         try {
           // Use structured output invocation (non-streaming)
           // Get a fresh model WITHOUT tools bound - bindTools() returns RunnableBinding which lacks withStructuredOutput
+          // Also disable thinking mode - Anthropic/Bedrock doesn't allow tool_choice with thinking enabled
+          const structuredClientOptions = { ...agentContext.clientOptions } as t.ClientOptions;
+          
+          // Remove thinking configuration for Bedrock
+          if (agentContext.provider === Providers.BEDROCK) {
+            const bedrockOpts = structuredClientOptions as t.BedrockAnthropicClientOptions;
+            if (bedrockOpts.additionalModelRequestFields?.['thinking']) {
+              // eslint-disable-next-line @typescript-eslint/no-explicit-any
+              const additionalFields = Object.assign({}, bedrockOpts.additionalModelRequestFields) as any;
+              delete additionalFields.thinking;
+              bedrockOpts.additionalModelRequestFields = additionalFields;
+            }
+          }
+          
+          // Remove thinking configuration for Anthropic
+          if (agentContext.provider === Providers.ANTHROPIC) {
+            const anthropicOpts = structuredClientOptions as t.AnthropicClientOptions;
+            if (anthropicOpts.thinking) {
+              delete anthropicOpts.thinking;
+            }
+          }
+          
           const structuredModel = this.getNewModel({
             provider: agentContext.provider,
-            clientOptions: agentContext.clientOptions,
+            clientOptions: structuredClientOptions,
           });
           
           const { structuredResponse, rawMessage } =

package/src/index.ts

@@ -12,6 +12,7 @@ export * from './graphs';
 export * from './tools/Calculator';
 export * from './tools/CodeExecutor';
 export * from './tools/BrowserTools';
+export * from './tools/DesktopTools';
 export * from './tools/ProgrammaticToolCalling';
 export * from './tools/ToolSearch';
 export * from './tools/handlers';

package/src/stream.ts

@@ -737,6 +737,23 @@ export function createContentAggregator(): t.ContentAggregatorResult {
       };
 
       updateContent(runStep.index, contentPart, true);
+    } else if (event === GraphEvents.ON_STRUCTURED_OUTPUT) {
+      // Handle structured output as text content with formatted JSON
+      const structuredData = data as unknown as {
+        structuredResponse: Record<string, unknown>;
+        schema: Record<string, unknown>;
+      };
+      
+      if (structuredData.structuredResponse) {
+        const jsonText = JSON.stringify(structuredData.structuredResponse, null, 2);
+        const contentPart: t.MessageContentComplex = {
+          type: ContentTypes.TEXT,
+          text: jsonText,
+        };
+        // Add at index 0 or next available
+        const nextIndex = contentParts.length;
+        updateContent(nextIndex, contentPart);
+      }
     }
   };