@causyne/agent 1.10.20 → 1.10.21

package/src/tools/computerUse.js

@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Causyne Vision Desktop & Coordinate Grounding Tool Wrapper.
+ */
+import { parseVisionGroundingPrediction, visionGroundingToDesktopAct, groundBoxCenter } from '../computerUse/visionGrounding.js';
+import { desktopAct } from './desktopAct.js';
+import { auditLog } from '../auditLog.js';
+export const computerUse = {
+    schema: {
+        name: 'computer_use',
+        description: 'Ground and execute graphical computer-use actions (click coordinates, type text, send key presses, drag, wait) derived from vision-language models and coordinate grounding predictions. Automatically translates normalized bounding boxes and coordinates against current screen dimensions with safety boundary checks. Requires explicit approval.',
+        parameters: {
+            type: 'object',
+            properties: {
+                action: {
+                    type: 'string',
+                    enum: ['execute_prediction', 'click', 'type', 'key', 'drag', 'wait'],
+                    description: 'Type of computer-use action to perform.'
+                },
+                prediction: {
+                    type: 'string',
+                    description: 'Raw prediction text from a vision-language model or coordinate grounding engine, e.g. "Thought: Click search bar. Action: click(start_box=\'(500, 200)\')".'
+                },
+                coordinates: {
+                    type: 'array',
+                    items: { type: 'number' },
+                    description: 'Coordinates as [x, y] or bounding box [x1, y1, x2, y2] normalized (0-1000) or in screen pixels.'
+                },
+                text: {
+                    type: 'string',
+                    description: 'Text to type into the active or targeted element.'
+                },
+                key: {
+                    type: 'string',
+                    description: 'Key or hotkey to press, e.g. "return", "tab", "escape".'
+                },
+                seconds: {
+                    type: 'number',
+                    description: 'Seconds to wait (0.1 to 10).'
+                },
+                app: {
+                    type: 'string',
+                    description: 'Application to focus before executing actions, e.g. "Safari".'
+                },
+                screenContext: {
+                    type: 'object',
+                    properties: {
+                        width: { type: 'number' },
+                        height: { type: 'number' },
+                        scaleFactor: { type: 'number' }
+                    },
+                    description: 'Current display dimensions and scale factor for coordinate grounding.'
+                }
+            }
+        }
+    },
+    async execute(args = {}, context = {}) {
+        const screen = args.screenContext || { width: 1920, height: 1080, scaleFactor: 1 };
+        const app = typeof args.app === 'string' ? args.app.trim() : '';
+        let desktopActions = [];
+        // Mode 1: Parse full model prediction from vision grounding engine
+        if (args.prediction || args.action === 'execute_prediction') {
+            const pred = args.prediction || '';
+            if (!pred.trim()) {
+                return { success: false, error: 'No prediction string provided for execute_prediction.' };
+            }
+            const parsed = parseVisionGroundingPrediction(pred, screen);
+            if (parsed.actions.length === 0) {
+                return { success: false, error: 'Could not parse any executable GUI actions from prediction string.' };
+            }
+            desktopActions = visionGroundingToDesktopAct(parsed.actions);
+        }
+        else {
+            // Mode 2: Direct action request
+            const actionType = args.action || 'click';
+            if (actionType === 'click') {
+                const coords = args.coordinates || [];
+                const point = groundBoxCenter(coords, screen);
+                if (!point)
+                    return { success: false, error: 'Click requires valid coordinates [x, y].' };
+                desktopActions.push({ verb: 'clickxy', x: point.x, y: point.y });
+            }
+            else if (actionType === 'type') {
+                if (args.coordinates && args.coordinates.length >= 2) {
+                    const point = groundBoxCenter(args.coordinates, screen);
+                    if (point)
+                        desktopActions.push({ verb: 'clickxy', x: point.x, y: point.y });
+                }
+                if (!args.text)
+                    return { success: false, error: 'Type action requires "text" parameter.' };
+                desktopActions.push({ verb: 'type', text: args.text });
+            }
+            else if (actionType === 'key') {
+                if (!args.key)
+                    return { success: false, error: 'Key action requires "key" parameter.' };
+                const keyMap = {
+                    return: 36, enter: 36, tab: 48, space: 49, backspace: 51, escape: 53, esc: 53
+                };
+                const keyCode = keyMap[args.key.toLowerCase()];
+                if (keyCode !== undefined) {
+                    desktopActions.push({ verb: 'key', keyCode });
+                }
+                else {
+                    desktopActions.push({ verb: 'type', text: args.key });
+                }
+            }
+            else if (actionType === 'wait') {
+                desktopActions.push({ verb: 'wait', seconds: Math.min(10, Math.max(0.1, args.seconds || 1)) });
+            }
+            else {
+                return { success: false, error: `Unsupported computer_use action: ${actionType}` };
+            }
+        }
+        if (desktopActions.length === 0) {
+            return { success: false, error: 'No actions to execute.' };
+        }
+        if (desktopActions.length > 12) {
+            return { success: false, error: `Maximum 12 actions allowed per call; got ${desktopActions.length}.` };
+        }
+        auditLog.log({
+            event: 'computer_use_actuation',
+            app,
+            actionCount: desktopActions.length,
+            verbs: desktopActions.map((a) => a.verb)
+        });
+        const execResult = await desktopAct.execute({ app, actions: desktopActions }, context);
+        const finalRes = {
+            success: Boolean(execResult.success),
+            actions: desktopActions
+        };
+        if (execResult.output)
+            finalRes.output = execResult.output;
+        if (execResult.error)
+            finalRes.error = execResult.error;
+        return finalRes;
+    }
+};

package/src/tools/computerUse.ts

@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Causyne Vision Desktop & Coordinate Grounding Tool Wrapper.
+ */
+
+import { parseVisionGroundingPrediction, visionGroundingToDesktopAct, groundBoxCenter, type ScreenContext } from '../computerUse/visionGrounding.js';
+import { desktopAct } from './desktopAct.js';
+import { auditLog } from '../auditLog.js';
+
+export interface ComputerUseArgs {
+  action?: 'execute_prediction' | 'click' | 'type' | 'key' | 'drag' | 'wait';
+  prediction?: string;
+  coordinates?: number[];
+  text?: string;
+  key?: string;
+  seconds?: number;
+  app?: string;
+  screenContext?: ScreenContext;
+}
+
+export const computerUse = {
+  schema: {
+    name: 'computer_use',
+    description: 'Ground and execute graphical computer-use actions (click coordinates, type text, send key presses, drag, wait) derived from vision-language models and coordinate grounding predictions. Automatically translates normalized bounding boxes and coordinates against current screen dimensions with safety boundary checks. Requires explicit approval.',
+    parameters: {
+      type: 'object',
+      properties: {
+        action: {
+          type: 'string',
+          enum: ['execute_prediction', 'click', 'type', 'key', 'drag', 'wait'],
+          description: 'Type of computer-use action to perform.'
+        },
+        prediction: {
+          type: 'string',
+          description: 'Raw prediction text from a vision-language model or coordinate grounding engine, e.g. "Thought: Click search bar. Action: click(start_box=\'(500, 200)\')".'
+        },
+        coordinates: {
+          type: 'array',
+          items: { type: 'number' },
+          description: 'Coordinates as [x, y] or bounding box [x1, y1, x2, y2] normalized (0-1000) or in screen pixels.'
+        },
+        text: {
+          type: 'string',
+          description: 'Text to type into the active or targeted element.'
+        },
+        key: {
+          type: 'string',
+          description: 'Key or hotkey to press, e.g. "return", "tab", "escape".'
+        },
+        seconds: {
+          type: 'number',
+          description: 'Seconds to wait (0.1 to 10).'
+        },
+        app: {
+          type: 'string',
+          description: 'Application to focus before executing actions, e.g. "Safari".'
+        },
+        screenContext: {
+          type: 'object',
+          properties: {
+            width: { type: 'number' },
+            height: { type: 'number' },
+            scaleFactor: { type: 'number' }
+          },
+          description: 'Current display dimensions and scale factor for coordinate grounding.'
+        }
+      }
+    }
+  },
+
+  async execute(args: ComputerUseArgs = {}, context: any = {}): Promise<{ success: boolean; output?: string; error?: string; actions?: any[] }> {
+    const screen: ScreenContext = args.screenContext || { width: 1920, height: 1080, scaleFactor: 1 };
+    const app = typeof args.app === 'string' ? args.app.trim() : '';
+
+    let desktopActions: any[] = [];
+
+    // Mode 1: Parse full model prediction from vision grounding engine
+    if (args.prediction || args.action === 'execute_prediction') {
+      const pred = args.prediction || '';
+      if (!pred.trim()) {
+        return { success: false, error: 'No prediction string provided for execute_prediction.' };
+      }
+      const parsed = parseVisionGroundingPrediction(pred, screen);
+      if (parsed.actions.length === 0) {
+        return { success: false, error: 'Could not parse any executable GUI actions from prediction string.' };
+      }
+      desktopActions = visionGroundingToDesktopAct(parsed.actions);
+    } else {
+      // Mode 2: Direct action request
+      const actionType = args.action || 'click';
+      if (actionType === 'click') {
+        const coords = args.coordinates || [];
+        const point = groundBoxCenter(coords, screen);
+        if (!point) return { success: false, error: 'Click requires valid coordinates [x, y].' };
+        desktopActions.push({ verb: 'clickxy', x: point.x, y: point.y });
+      } else if (actionType === 'type') {
+        if (args.coordinates && args.coordinates.length >= 2) {
+          const point = groundBoxCenter(args.coordinates, screen);
+          if (point) desktopActions.push({ verb: 'clickxy', x: point.x, y: point.y });
+        }
+        if (!args.text) return { success: false, error: 'Type action requires "text" parameter.' };
+        desktopActions.push({ verb: 'type', text: args.text });
+      } else if (actionType === 'key') {
+        if (!args.key) return { success: false, error: 'Key action requires "key" parameter.' };
+        const keyMap: Record<string, number> = {
+          return: 36, enter: 36, tab: 48, space: 49, backspace: 51, escape: 53, esc: 53
+        };
+        const keyCode = keyMap[args.key.toLowerCase()];
+        if (keyCode !== undefined) {
+          desktopActions.push({ verb: 'key', keyCode });
+        } else {
+          desktopActions.push({ verb: 'type', text: args.key });
+        }
+      } else if (actionType === 'wait') {
+        desktopActions.push({ verb: 'wait', seconds: Math.min(10, Math.max(0.1, args.seconds || 1)) });
+      } else {
+        return { success: false, error: `Unsupported computer_use action: ${actionType}` };
+      }
+    }
+
+    if (desktopActions.length === 0) {
+      return { success: false, error: 'No actions to execute.' };
+    }
+
+    if (desktopActions.length > 12) {
+      return { success: false, error: `Maximum 12 actions allowed per call; got ${desktopActions.length}.` };
+    }
+
+    auditLog.log({
+      event: 'computer_use_actuation',
+      app,
+      actionCount: desktopActions.length,
+      verbs: desktopActions.map((a) => a.verb)
+    });
+
+    const execResult = await desktopAct.execute({ app, actions: desktopActions }, context);
+    const finalRes: { success: boolean; output?: string; error?: string; actions?: any[] } = {
+      success: Boolean(execResult.success),
+      actions: desktopActions
+    };
+    if (execResult.output) finalRes.output = execResult.output;
+    if (execResult.error) finalRes.error = execResult.error;
+    return finalRes;
+  }
+};

package/src/tools/desktopAct.d.ts

@@ -0,0 +1,10 @@
+export const desktopAct: {
+  schema: Record<string, unknown>;
+  execute(args?: Record<string, unknown>, context?: Record<string, unknown>): Promise<{
+    success: boolean;
+    output?: string;
+    error?: string;
+    requiresConsent?: boolean;
+    useToolInstead?: string;
+  }>;
+};

package/src/tools/registry.js

@@ -1,4 +1,6 @@
 import { creativeApps } from './creativeApps.js';
+import { computerUse } from './computerUse.js';
+import { uiForge } from './uiForge.js';
 import { sysRun, sysRunBg } from './sysRun.js';
 import { fsRead } from './fsRead.js';
 import { fsWrite } from './fsWrite.js';
@@ -120,7 +122,8 @@ const TOOL_MODULES = [
   gameAssetHarvester, gameAudioSynth, gameVfxEmitterGen,
   gameShaderForge, gameSceneBuilder, gameEntitySpawner,
   gameTilemapGenerator, gameUiHudBuilder, gameEconomyBalancer,
-  gameDataForge, gameWebRuntimeInspector, gameCanvasAudit, readNativeSkill, createSkill, creativeApps
+  gameDataForge, gameWebRuntimeInspector, gameCanvasAudit, readNativeSkill, createSkill, creativeApps,
+  computerUse, uiForge
 ];
 
 // Base static definitions (backwards compatible). Order is preserved from the

package/src/tools/uiForge.js

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2011-2024 Thesys Inc.
+ * SPDX-License-Identifier: MIT
+ *
+ * Causyne UI Forge Tool Wrapper.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { generateHtmlComponent, generatePreviewHtml, generateReactComponent } from '../uiEngine/uiForge.js';
+import { resolveUserPath } from '../utils/paths.js';
+export const uiForge = {
+    schema: {
+        name: 'ui_forge',
+        description: 'Causyne UI Forge: Generate production-grade accessible, responsive UI components (cards, dashboards, forms, tables) and design systems. Emits clean semantic HTML/CSS, React+Tailwind, or Svelte/Vue with a self-contained live preview.',
+        parameters: {
+            type: 'object',
+            properties: {
+                framework: {
+                    type: 'string',
+                    enum: ['html-css', 'react', 'vue', 'svelte'],
+                    description: 'Target frontend framework. Defaults to "html-css" (zero-dependency semantic HTML + Vanilla CSS).'
+                },
+                spec: {
+                    type: 'object',
+                    properties: {
+                        name: { type: 'string', description: 'Component PascalCase name, e.g. AnalyticsCard.' },
+                        type: {
+                            type: 'string',
+                            enum: ['card', 'navbar', 'form', 'dashboard-stats', 'table', 'modal', 'custom'],
+                            description: 'Pattern type.'
+                        },
+                        title: { type: 'string', description: 'Header or component title.' },
+                        description: { type: 'string', description: 'Subtitle or descriptive text.' },
+                        theme: { type: 'string', enum: ['dark', 'light'], description: 'Theme palette (dark or light).' },
+                        items: {
+                            type: 'array',
+                            items: {
+                                type: 'object',
+                                properties: {
+                                    id: { type: 'string' },
+                                    label: { type: 'string' },
+                                    value: { type: ['string', 'number'] },
+                                    description: { type: 'string' }
+                                },
+                                required: ['label']
+                            },
+                            description: 'List of data items, metrics, fields or elements.'
+                        },
+                        actions: {
+                            type: 'array',
+                            items: {
+                                type: 'object',
+                                properties: {
+                                    label: { type: 'string' },
+                                    action: { type: 'string' },
+                                    primary: { type: 'boolean' }
+                                },
+                                required: ['label', 'action']
+                            },
+                            description: 'Interactive button actions.'
+                        }
+                    },
+                    required: ['type']
+                },
+                outputPath: {
+                    type: 'string',
+                    description: 'Optional file path to write the generated component and preview directly to workspace, e.g. "src/components/AnalyticsCard.html".'
+                }
+            },
+            required: ['spec']
+        }
+    },
+    async execute(args = {}) {
+        if (!args.spec || !args.spec.type) {
+            return { success: false, error: 'UI Forge specification with a valid component "type" is required.' };
+        }
+        const framework = args.framework || 'html-css';
+        const spec = args.spec;
+        let code = '';
+        let css = '';
+        let previewHtml = '';
+        if (framework === 'react') {
+            code = generateReactComponent(spec);
+            const htmlComp = generateHtmlComponent(spec);
+            css = htmlComp.css;
+            previewHtml = generatePreviewHtml(htmlComp.html, htmlComp.css, spec.theme || 'dark');
+        }
+        else {
+            const htmlComp = generateHtmlComponent(spec);
+            code = htmlComp.html;
+            css = htmlComp.css;
+            previewHtml = generatePreviewHtml(htmlComp.html, htmlComp.css, spec.theme || 'dark');
+        }
+        let writtenFiles = [];
+        if (args.outputPath) {
+            try {
+                const resolved = resolveUserPath(args.outputPath);
+                const dir = path.dirname(resolved);
+                if (!fs.existsSync(dir))
+                    fs.mkdirSync(dir, { recursive: true });
+                fs.writeFileSync(resolved, code, 'utf8');
+                writtenFiles.push(resolved);
+                const ext = path.extname(resolved);
+                const previewFile = path.join(dir, `${path.basename(resolved, ext)}.preview.html`);
+                fs.writeFileSync(previewFile, previewHtml, 'utf8');
+                writtenFiles.push(previewFile);
+            }
+            catch (err) {
+                return { success: false, error: `Failed to write component files: ${err.message}` };
+            }
+        }
+        const reportLines = [
+            `Generated ${spec.type} component using Causyne UI Forge (${framework}).`,
+            `Theme: ${spec.theme || 'dark'} | Items: ${(spec.items || []).length} | Actions: ${(spec.actions || []).length}`
+        ];
+        if (writtenFiles.length > 0) {
+            reportLines.push(`Saved component file: ${writtenFiles[0]}`);
+            reportLines.push(`Saved preview file:   ${writtenFiles[1]}`);
+        }
+        else {
+            reportLines.push('', '--- Component Code ---', code);
+        }
+        const res = {
+            success: true,
+            output: reportLines.join('\n'),
+            code
+        };
+        if (writtenFiles[1])
+            res.previewPath = writtenFiles[1];
+        return res;
+    }
+};

package/src/tools/uiForge.ts

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2011-2024 Thesys Inc.
+ * SPDX-License-Identifier: MIT
+ *
+ * Causyne UI Forge Tool Wrapper.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { generateHtmlComponent, generatePreviewHtml, generateReactComponent, type UIComponentSpec, type UIFramework } from '../uiEngine/uiForge.js';
+import { resolveUserPath } from '../utils/paths.js';
+
+export interface UiForgeArgs {
+  framework?: UIFramework;
+  spec?: UIComponentSpec;
+  outputPath?: string;
+}
+
+export const uiForge = {
+  schema: {
+    name: 'ui_forge',
+    description: 'Causyne UI Forge: Generate production-grade accessible, responsive UI components (cards, dashboards, forms, tables) and design systems. Emits clean semantic HTML/CSS, React+Tailwind, or Svelte/Vue with a self-contained live preview.',
+    parameters: {
+      type: 'object',
+      properties: {
+        framework: {
+          type: 'string',
+          enum: ['html-css', 'react', 'vue', 'svelte'],
+          description: 'Target frontend framework. Defaults to "html-css" (zero-dependency semantic HTML + Vanilla CSS).'
+        },
+        spec: {
+          type: 'object',
+          properties: {
+            name: { type: 'string', description: 'Component PascalCase name, e.g. AnalyticsCard.' },
+            type: {
+              type: 'string',
+              enum: ['card', 'navbar', 'form', 'dashboard-stats', 'table', 'modal', 'custom'],
+              description: 'Pattern type.'
+            },
+            title: { type: 'string', description: 'Header or component title.' },
+            description: { type: 'string', description: 'Subtitle or descriptive text.' },
+            theme: { type: 'string', enum: ['dark', 'light'], description: 'Theme palette (dark or light).' },
+            items: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  id: { type: 'string' },
+                  label: { type: 'string' },
+                  value: { type: ['string', 'number'] },
+                  description: { type: 'string' }
+                },
+                required: ['label']
+              },
+              description: 'List of data items, metrics, fields or elements.'
+            },
+            actions: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  label: { type: 'string' },
+                  action: { type: 'string' },
+                  primary: { type: 'boolean' }
+                },
+                required: ['label', 'action']
+              },
+              description: 'Interactive button actions.'
+            }
+          },
+          required: ['type']
+        },
+        outputPath: {
+          type: 'string',
+          description: 'Optional file path to write the generated component and preview directly to workspace, e.g. "src/components/AnalyticsCard.html".'
+        }
+      },
+      required: ['spec']
+    }
+  },
+
+  async execute(args: UiForgeArgs = {}): Promise<{ success: boolean; output?: string; code?: string; previewPath?: string; error?: string }> {
+    if (!args.spec || !args.spec.type) {
+      return { success: false, error: 'UI Forge specification with a valid component "type" is required.' };
+    }
+
+    const framework: UIFramework = args.framework || 'html-css';
+    const spec = args.spec;
+
+    let code = '';
+    let css = '';
+    let previewHtml = '';
+
+    if (framework === 'react') {
+      code = generateReactComponent(spec);
+      const htmlComp = generateHtmlComponent(spec);
+      css = htmlComp.css;
+      previewHtml = generatePreviewHtml(htmlComp.html, htmlComp.css, spec.theme || 'dark');
+    } else {
+      const htmlComp = generateHtmlComponent(spec);
+      code = htmlComp.html;
+      css = htmlComp.css;
+      previewHtml = generatePreviewHtml(htmlComp.html, htmlComp.css, spec.theme || 'dark');
+    }
+
+    let writtenFiles: string[] = [];
+    if (args.outputPath) {
+      try {
+        const resolved = resolveUserPath(args.outputPath);
+        const dir = path.dirname(resolved);
+        if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+
+        fs.writeFileSync(resolved, code, 'utf8');
+        writtenFiles.push(resolved);
+
+        const ext = path.extname(resolved);
+        const previewFile = path.join(dir, `${path.basename(resolved, ext)}.preview.html`);
+        fs.writeFileSync(previewFile, previewHtml, 'utf8');
+        writtenFiles.push(previewFile);
+      } catch (err: any) {
+        return { success: false, error: `Failed to write component files: ${err.message}` };
+      }
+    }
+
+    const reportLines = [
+      `Generated ${spec.type} component using Causyne UI Forge (${framework}).`,
+      `Theme: ${spec.theme || 'dark'} | Items: ${(spec.items || []).length} | Actions: ${(spec.actions || []).length}`
+    ];
+
+    if (writtenFiles.length > 0) {
+      reportLines.push(`Saved component file: ${writtenFiles[0]}`);
+      reportLines.push(`Saved preview file:   ${writtenFiles[1]}`);
+    } else {
+      reportLines.push('', '--- Component Code ---', code);
+    }
+
+    const res: { success: boolean; output?: string; code?: string; previewPath?: string; error?: string } = {
+      success: true,
+      output: reportLines.join('\n'),
+      code
+    };
+    if (writtenFiles[1]) res.previewPath = writtenFiles[1];
+    return res;
+  }
+};