@causyne/agent 1.10.19 → 1.10.21

package/src/computerUse/visionGrounding.js

@@ -0,0 +1,249 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Adapted for Causyne Vision Desktop & Coordinate Grounding Engine.
+ */
+const DEFAULT_FACTOR = [1000, 1000];
+/**
+ * Extract numbers from a coordinate string like "(250, 400)" or "[100, 200, 300, 400]"
+ */
+export function extractCoordinateNumbers(raw) {
+    const clean = String(raw || '').replace(/[()[\]]/g, ' ').trim();
+    if (!clean)
+        return [];
+    return clean
+        .split(/[\s,]+/)
+        .map((s) => Number.parseFloat(s))
+        .filter((n) => Number.isFinite(n));
+}
+/**
+ * Ground normalized coordinates (0-1000) to actual screen pixels.
+ */
+export function groundPoint(nums, screen = { width: 1920, height: 1080, scaleFactor: 1 }, factor = DEFAULT_FACTOR) {
+    if (nums.length < 2)
+        return null;
+    const rawX = nums[0] ?? 0;
+    const rawY = nums[1] ?? 0;
+    const scale = screen.scaleFactor || 1;
+    const normX = Math.max(0, Math.min(1, rawX / factor[0]));
+    const normY = Math.max(0, Math.min(1, rawY / factor[1]));
+    return {
+        x: Math.round(normX * screen.width * scale),
+        y: Math.round(normY * screen.height * scale)
+    };
+}
+/**
+ * Ground a bounding box (x1, y1, x2, y2) into its midpoint (center click).
+ */
+export function groundBoxCenter(nums, screen = { width: 1920, height: 1080, scaleFactor: 1 }, factor = DEFAULT_FACTOR) {
+    if (nums.length < 2)
+        return null;
+    if (nums.length === 2) {
+        return groundPoint(nums, screen, factor);
+    }
+    const x1 = nums[0] ?? 0;
+    const y1 = nums[1] ?? 0;
+    const x2 = nums[2] ?? x1;
+    const y2 = nums[3] ?? y1;
+    const midX = (x1 + x2) / 2;
+    const midY = (y1 + y2) / 2;
+    return groundPoint([midX, midY], screen, factor);
+}
+/**
+ * Parse vision grounding action format:
+ * E.g. Action: click(start_box='(240, 500)')
+ */
+export function parseSingleAction(rawAction, screenContext = { width: 1920, height: 1080, scaleFactor: 1 }) {
+    const line = rawAction.trim();
+    if (!line)
+        return null;
+    const callMatch = line.match(/^([a-zA-Z_0-9]+)\s*\((.*)\)$/);
+    if (!callMatch) {
+        if (/finished/i.test(line)) {
+            return { action: 'finished', rawStr: line };
+        }
+        return null;
+    }
+    const funcName = callMatch[1]?.toLowerCase();
+    const argsRaw = callMatch[2] || '';
+    const argsMap = {};
+    const kvRegex = /([a-zA-Z_0-9]+)\s*=\s*['"]([^'"]+)['"]/g;
+    let match;
+    while ((match = kvRegex.exec(argsRaw)) !== null) {
+        const key = match[1];
+        const val = match[2];
+        if (key && val) {
+            argsMap[key] = val;
+        }
+    }
+    if (Object.keys(argsMap).length === 0) {
+        const boxMatch = argsRaw.match(/['"]?(\([0-9\s,.]+\)|\[[0-9\s,.]+\])['"]?/);
+        if (boxMatch && boxMatch[1]) {
+            argsMap['start_box'] = boxMatch[1];
+        }
+    }
+    const actionObj = {
+        action: funcName,
+        rawStr: line
+    };
+    const startBox = argsMap['start_box'] || argsMap['point'] || argsMap['box'];
+    if (startBox) {
+        const nums = extractCoordinateNumbers(startBox);
+        const center = groundBoxCenter(nums, screenContext);
+        if (center) {
+            actionObj.coordinates = center;
+            actionObj.startCoordinates = center;
+        }
+    }
+    const endBox = argsMap['end_box'];
+    if (endBox) {
+        const nums = extractCoordinateNumbers(endBox);
+        const center = groundBoxCenter(nums, screenContext);
+        if (center) {
+            actionObj.endCoordinates = center;
+        }
+    }
+    if (argsMap['content']) {
+        actionObj.text = argsMap['content'];
+    }
+    if (argsMap['text']) {
+        actionObj.text = argsMap['text'];
+    }
+    if (argsMap['key']) {
+        actionObj.key = argsMap['key'];
+    }
+    if (argsMap['direction']) {
+        const dir = argsMap['direction'].toLowerCase();
+        if (dir === 'up' || dir === 'down' || dir === 'left' || dir === 'right') {
+            actionObj.direction = dir;
+        }
+    }
+    if (argsMap['time'] || argsMap['seconds']) {
+        const sec = Number.parseFloat(argsMap['time'] || argsMap['seconds'] || '1');
+        actionObj.seconds = Number.isFinite(sec) ? sec : 1;
+    }
+    return actionObj;
+}
+/**
+ * Parse complete vision grounding prediction text with Thought and Action blocks.
+ */
+export function parseVisionGroundingPrediction(predictionText, screenContext = { width: 1920, height: 1080, scaleFactor: 1 }) {
+    let thought;
+    let reflection;
+    const text = (predictionText || '').trim();
+    if (text.includes('Thought:')) {
+        const tMatch = text.match(/Thought:\s*([\s\S]+?)(?=\s*Action[:：]|$)/i);
+        if (tMatch && tMatch[1]) {
+            thought = tMatch[1].trim();
+        }
+    }
+    if (text.includes('Reflection:')) {
+        const rMatch = text.match(/Reflection:\s*([\s\S]+?)(?=\s*(?:Action_Summary|Action)[:：]|$)/i);
+        if (rMatch && rMatch[1]) {
+            reflection = rMatch[1].trim();
+        }
+    }
+    const actionLines = [];
+    if (/Action[:：]/i.test(text)) {
+        const parts = text.split(/Action[:：]/i);
+        for (let i = 1; i < parts.length; i++) {
+            const seg = parts[i] || '';
+            const lines = seg.split(/\n+/).map((l) => l.trim()).filter(Boolean);
+            actionLines.push(...lines);
+        }
+    }
+    else {
+        actionLines.push(...text.split(/\n+/).map((l) => l.trim()).filter(Boolean));
+    }
+    const actions = [];
+    for (const line of actionLines) {
+        const parsed = parseSingleAction(line, screenContext);
+        if (parsed) {
+            if (thought && !parsed.thought)
+                parsed.thought = thought;
+            if (reflection && !parsed.reflection)
+                parsed.reflection = reflection;
+            actions.push(parsed);
+        }
+    }
+    return { thought, reflection, actions };
+}
+const MAC_KEY_CODES = {
+    return: 36,
+    enter: 36,
+    tab: 48,
+    space: 49,
+    backspace: 51,
+    escape: 53,
+    esc: 53,
+    up: 126,
+    down: 125,
+    left: 123,
+    right: 124
+};
+/**
+ * Convert parsed vision grounding actions into Causyne desktop_act equivalent instructions.
+ */
+export function visionGroundingToDesktopAct(actions) {
+    const result = [];
+    for (const act of actions) {
+        switch (act.action) {
+            case 'click':
+            case 'left_click':
+            case 'double_click':
+            case 'triple_click': {
+                const point = act.coordinates;
+                if (point) {
+                    result.push({
+                        verb: 'clickxy',
+                        x: point.x,
+                        y: point.y
+                    });
+                    if (act.action === 'double_click') {
+                        result.push({ verb: 'clickxy', x: point.x, y: point.y });
+                    }
+                }
+                break;
+            }
+            case 'type': {
+                if (act.coordinates) {
+                    result.push({ verb: 'clickxy', x: act.coordinates.x, y: act.coordinates.y });
+                }
+                if (act.text) {
+                    result.push({ verb: 'type', text: act.text });
+                }
+                break;
+            }
+            case 'press_key':
+            case 'hotkey': {
+                const keyName = (act.key || '').toLowerCase().trim();
+                const code = MAC_KEY_CODES[keyName];
+                if (code !== undefined) {
+                    result.push({ verb: 'key', keyCode: code });
+                }
+                else if (keyName) {
+                    result.push({ verb: 'type', text: keyName });
+                }
+                break;
+            }
+            case 'wait': {
+                result.push({ verb: 'wait', seconds: act.seconds || 1 });
+                break;
+            }
+            case 'drag': {
+                if (act.startCoordinates) {
+                    result.push({ verb: 'clickxy', x: act.startCoordinates.x, y: act.startCoordinates.y });
+                }
+                if (act.endCoordinates) {
+                    result.push({ verb: 'clickxy', x: act.endCoordinates.x, y: act.endCoordinates.y });
+                }
+                break;
+            }
+            case 'finished':
+            default:
+                break;
+        }
+    }
+    return result;
+}

package/src/computerUse/visionGrounding.ts

@@ -0,0 +1,331 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Adapted for Causyne Vision Desktop & Coordinate Grounding Engine.
+ */
+
+export interface ScreenContext {
+  width: number;
+  height: number;
+  scaleFactor?: number;
+}
+
+export type ActionVerb =
+  | 'click'
+  | 'left_click'
+  | 'right_click'
+  | 'double_click'
+  | 'triple_click'
+  | 'middle_click'
+  | 'mouse_down'
+  | 'mouse_up'
+  | 'drag'
+  | 'move'
+  | 'type'
+  | 'press_key'
+  | 'hotkey'
+  | 'scroll'
+  | 'wait'
+  | 'finished';
+
+export interface VisionAction {
+  action: ActionVerb;
+  thought?: string;
+  reflection?: string;
+  coordinates?: { x: number; y: number };
+  startCoordinates?: { x: number; y: number };
+  endCoordinates?: { x: number; y: number };
+  text?: string;
+  key?: string;
+  direction?: 'up' | 'down' | 'left' | 'right';
+  seconds?: number;
+  rawStr?: string;
+}
+
+export interface DesktopActEquivalent {
+  verb: 'press' | 'clickxy' | 'type' | 'key' | 'wait';
+  x?: number;
+  y?: number;
+  text?: string;
+  keyCode?: number;
+  target?: string;
+  seconds?: number;
+}
+
+const DEFAULT_FACTOR: [number, number] = [1000, 1000];
+
+/**
+ * Extract numbers from a coordinate string like "(250, 400)" or "[100, 200, 300, 400]"
+ */
+export function extractCoordinateNumbers(raw: string): number[] {
+  const clean = String(raw || '').replace(/[()[\]]/g, ' ').trim();
+  if (!clean) return [];
+  return clean
+    .split(/[\s,]+/)
+    .map((s) => Number.parseFloat(s))
+    .filter((n) => Number.isFinite(n));
+}
+
+/**
+ * Ground normalized coordinates (0-1000) to actual screen pixels.
+ */
+export function groundPoint(
+  nums: number[],
+  screen: ScreenContext = { width: 1920, height: 1080, scaleFactor: 1 },
+  factor: [number, number] = DEFAULT_FACTOR
+): { x: number; y: number } | null {
+  if (nums.length < 2) return null;
+  const rawX = nums[0] ?? 0;
+  const rawY = nums[1] ?? 0;
+  const scale = screen.scaleFactor || 1;
+
+  const normX = Math.max(0, Math.min(1, rawX / factor[0]));
+  const normY = Math.max(0, Math.min(1, rawY / factor[1]));
+
+  return {
+    x: Math.round(normX * screen.width * scale),
+    y: Math.round(normY * screen.height * scale)
+  };
+}
+
+/**
+ * Ground a bounding box (x1, y1, x2, y2) into its midpoint (center click).
+ */
+export function groundBoxCenter(
+  nums: number[],
+  screen: ScreenContext = { width: 1920, height: 1080, scaleFactor: 1 },
+  factor: [number, number] = DEFAULT_FACTOR
+): { x: number; y: number } | null {
+  if (nums.length < 2) return null;
+  if (nums.length === 2) {
+    return groundPoint(nums, screen, factor);
+  }
+  const x1 = nums[0] ?? 0;
+  const y1 = nums[1] ?? 0;
+  const x2 = nums[2] ?? x1;
+  const y2 = nums[3] ?? y1;
+
+  const midX = (x1 + x2) / 2;
+  const midY = (y1 + y2) / 2;
+  return groundPoint([midX, midY], screen, factor);
+}
+
+/**
+ * Parse vision grounding action format:
+ * E.g. Action: click(start_box='(240, 500)')
+ */
+export function parseSingleAction(
+  rawAction: string,
+  screenContext: ScreenContext = { width: 1920, height: 1080, scaleFactor: 1 }
+): VisionAction | null {
+  const line = rawAction.trim();
+  if (!line) return null;
+
+  const callMatch = line.match(/^([a-zA-Z_0-9]+)\s*\((.*)\)$/);
+  if (!callMatch) {
+    if (/finished/i.test(line)) {
+      return { action: 'finished', rawStr: line };
+    }
+    return null;
+  }
+
+  const funcName = callMatch[1]?.toLowerCase() as ActionVerb;
+  const argsRaw = callMatch[2] || '';
+
+  const argsMap: Record<string, string> = {};
+  const kvRegex = /([a-zA-Z_0-9]+)\s*=\s*['"]([^'"]+)['"]/g;
+  let match: RegExpExecArray | null;
+  while ((match = kvRegex.exec(argsRaw)) !== null) {
+    const key = match[1];
+    const val = match[2];
+    if (key && val) {
+      argsMap[key] = val;
+    }
+  }
+
+  if (Object.keys(argsMap).length === 0) {
+    const boxMatch = argsRaw.match(/['"]?(\([0-9\s,.]+\)|\[[0-9\s,.]+\])['"]?/);
+    if (boxMatch && boxMatch[1]) {
+      argsMap['start_box'] = boxMatch[1];
+    }
+  }
+
+  const actionObj: VisionAction = {
+    action: funcName,
+    rawStr: line
+  };
+
+  const startBox = argsMap['start_box'] || argsMap['point'] || argsMap['box'];
+  if (startBox) {
+    const nums = extractCoordinateNumbers(startBox);
+    const center = groundBoxCenter(nums, screenContext);
+    if (center) {
+      actionObj.coordinates = center;
+      actionObj.startCoordinates = center;
+    }
+  }
+
+  const endBox = argsMap['end_box'];
+  if (endBox) {
+    const nums = extractCoordinateNumbers(endBox);
+    const center = groundBoxCenter(nums, screenContext);
+    if (center) {
+      actionObj.endCoordinates = center;
+    }
+  }
+
+  if (argsMap['content']) {
+    actionObj.text = argsMap['content'];
+  }
+  if (argsMap['text']) {
+    actionObj.text = argsMap['text'];
+  }
+  if (argsMap['key']) {
+    actionObj.key = argsMap['key'];
+  }
+  if (argsMap['direction']) {
+    const dir = argsMap['direction'].toLowerCase();
+    if (dir === 'up' || dir === 'down' || dir === 'left' || dir === 'right') {
+      actionObj.direction = dir;
+    }
+  }
+  if (argsMap['time'] || argsMap['seconds']) {
+    const sec = Number.parseFloat(argsMap['time'] || argsMap['seconds'] || '1');
+    actionObj.seconds = Number.isFinite(sec) ? sec : 1;
+  }
+
+  return actionObj;
+}
+
+/**
+ * Parse complete vision grounding prediction text with Thought and Action blocks.
+ */
+export function parseVisionGroundingPrediction(
+  predictionText: string,
+  screenContext: ScreenContext = { width: 1920, height: 1080, scaleFactor: 1 }
+): { thought?: string | undefined; reflection?: string | undefined; actions: VisionAction[] } {
+  let thought: string | undefined;
+  let reflection: string | undefined;
+
+  const text = (predictionText || '').trim();
+
+  if (text.includes('Thought:')) {
+    const tMatch = text.match(/Thought:\s*([\s\S]+?)(?=\s*Action[:：]|$)/i);
+    if (tMatch && tMatch[1]) {
+      thought = tMatch[1].trim();
+    }
+  }
+  if (text.includes('Reflection:')) {
+    const rMatch = text.match(/Reflection:\s*([\s\S]+?)(?=\s*(?:Action_Summary|Action)[:：]|$)/i);
+    if (rMatch && rMatch[1]) {
+      reflection = rMatch[1].trim();
+    }
+  }
+
+  const actionLines: string[] = [];
+  if (/Action[:：]/i.test(text)) {
+    const parts = text.split(/Action[:：]/i);
+    for (let i = 1; i < parts.length; i++) {
+      const seg = parts[i] || '';
+      const lines = seg.split(/\n+/).map((l) => l.trim()).filter(Boolean);
+      actionLines.push(...lines);
+    }
+  } else {
+    actionLines.push(...text.split(/\n+/).map((l) => l.trim()).filter(Boolean));
+  }
+
+  const actions: VisionAction[] = [];
+
+  for (const line of actionLines) {
+    const parsed = parseSingleAction(line, screenContext);
+    if (parsed) {
+      if (thought && !parsed.thought) parsed.thought = thought;
+      if (reflection && !parsed.reflection) parsed.reflection = reflection;
+      actions.push(parsed);
+    }
+  }
+
+  return { thought, reflection, actions };
+}
+
+const MAC_KEY_CODES: Record<string, number> = {
+  return: 36,
+  enter: 36,
+  tab: 48,
+  space: 49,
+  backspace: 51,
+  escape: 53,
+  esc: 53,
+  up: 126,
+  down: 125,
+  left: 123,
+  right: 124
+};
+
+/**
+ * Convert parsed vision grounding actions into Causyne desktop_act equivalent instructions.
+ */
+export function visionGroundingToDesktopAct(actions: VisionAction[]): DesktopActEquivalent[] {
+  const result: DesktopActEquivalent[] = [];
+
+  for (const act of actions) {
+    switch (act.action) {
+      case 'click':
+      case 'left_click':
+      case 'double_click':
+      case 'triple_click': {
+        const point = act.coordinates;
+        if (point) {
+          result.push({
+            verb: 'clickxy',
+            x: point.x,
+            y: point.y
+          });
+          if (act.action === 'double_click') {
+            result.push({ verb: 'clickxy', x: point.x, y: point.y });
+          }
+        }
+        break;
+      }
+      case 'type': {
+        if (act.coordinates) {
+          result.push({ verb: 'clickxy', x: act.coordinates.x, y: act.coordinates.y });
+        }
+        if (act.text) {
+          result.push({ verb: 'type', text: act.text });
+        }
+        break;
+      }
+      case 'press_key':
+      case 'hotkey': {
+        const keyName = (act.key || '').toLowerCase().trim();
+        const code = MAC_KEY_CODES[keyName];
+        if (code !== undefined) {
+          result.push({ verb: 'key', keyCode: code });
+        } else if (keyName) {
+          result.push({ verb: 'type', text: keyName });
+        }
+        break;
+      }
+      case 'wait': {
+        result.push({ verb: 'wait', seconds: act.seconds || 1 });
+        break;
+      }
+      case 'drag': {
+        if (act.startCoordinates) {
+          result.push({ verb: 'clickxy', x: act.startCoordinates.x, y: act.startCoordinates.y });
+        }
+        if (act.endCoordinates) {
+          result.push({ verb: 'clickxy', x: act.endCoordinates.x, y: act.endCoordinates.y });
+        }
+        break;
+      }
+      case 'finished':
+      default:
+        break;
+    }
+  }
+
+  return result;
+}

package/src/config.js

@@ -78,30 +78,41 @@ export function loadConfig() {
         }
       }
 
-      // Self-heal a model id written before src/modelIds.js existed. The
-      // provider normalises too, but tokenTracker.getPricing() and the /model
-      // display both read config.model directly —
-      // they must agree with what is actually sent, not with what a stale
-      // Android build once wrote here.
+      // Self-heal a model id written before src/modelIds.js existed.
       const canonicalModel = canonicalModelId(parsed.model);
       if (canonicalModel !== parsed.model) {
         parsed.model = canonicalModel;
         saveConfig(parsed);
       }
 
+      // Self-heal legacy unconfigured DashScope bug:
+      // If config was written with provider 'dashscope' or 'dashscope-byok' but without an apiKey,
+      // it was an unintended default from older builds. Heal to an environment key or unconfigured.
+      if ((parsed.provider === 'dashscope' || parsed.provider === 'dashscope-byok') && !parsed.apiKey) {
+        const envDetected = detectEnvironmentProvider();
+        if (envDetected) {
+          parsed.provider = envDetected.provider;
+          parsed.model = envDetected.model;
+          parsed.apiKey = envDetected.apiKey;
+        } else {
+          parsed.provider = '';
+          parsed.model = '';
+        }
+        saveConfig(parsed);
+      }
+
       return parsed;
     } catch (e) {
       console.error('Error reading config.json:', e);
     }
   }
   
-    // Default config
+  // Default config: auto-detect from environment if available, otherwise unconfigured
+  const envDetected = detectEnvironmentProvider();
   const defaultConfig = {
-    provider: 'dashscope',
-    // From the catalogue, not repeated here — this had been left at the
-    // long-retired `qwen-plus` while the picker had moved to qwen3.7.
-    model: DEFAULT_MODEL_BY_PROVIDER.dashscope,
-    apiKey: '',
+    provider: envDetected ? envDetected.provider : '',
+    model: envDetected ? envDetected.model : '',
+    apiKey: envDetected ? envDetected.apiKey : '',
     autonomyMode: 'B',
     tokenBudget: {
       enabled: false,
@@ -116,6 +127,64 @@ export function loadConfig() {
   return defaultConfig;
 }
 
+/**
+ * Auto-detect available API keys from environment variables.
+ * Precedence matches frontier capability: Gemini, Anthropic, OpenAI, DeepSeek, xAI, Moonshot, Qwen/DashScope.
+ */
+export function detectEnvironmentProvider() {
+  const env = process.env;
+  if (env.GEMINI_API_KEY || env.GOOGLE_API_KEY) {
+    return {
+      provider: 'gemini',
+      apiKey: env.GEMINI_API_KEY || env.GOOGLE_API_KEY || '',
+      model: DEFAULT_MODEL_BY_PROVIDER.gemini
+    };
+  }
+  if (env.ANTHROPIC_API_KEY) {
+    return {
+      provider: 'anthropic',
+      apiKey: env.ANTHROPIC_API_KEY,
+      model: DEFAULT_MODEL_BY_PROVIDER.anthropic
+    };
+  }
+  if (env.OPENAI_API_KEY) {
+    return {
+      provider: 'openai',
+      apiKey: env.OPENAI_API_KEY,
+      model: DEFAULT_MODEL_BY_PROVIDER.openai
+    };
+  }
+  if (env.DEEPSEEK_API_KEY) {
+    return {
+      provider: 'deepseek',
+      apiKey: env.DEEPSEEK_API_KEY,
+      model: DEFAULT_MODEL_BY_PROVIDER.deepseek
+    };
+  }
+  if (env.XAI_API_KEY || env.GROK_API_KEY) {
+    return {
+      provider: 'xai',
+      apiKey: env.XAI_API_KEY || env.GROK_API_KEY || '',
+      model: DEFAULT_MODEL_BY_PROVIDER.xai
+    };
+  }
+  if (env.MOONSHOT_API_KEY) {
+    return {
+      provider: 'moonshot',
+      apiKey: env.MOONSHOT_API_KEY,
+      model: DEFAULT_MODEL_BY_PROVIDER.moonshot
+    };
+  }
+  if (env.DASHSCOPE_API_KEY || env.QWEN_API_KEY) {
+    return {
+      provider: 'dashscope-byok',
+      apiKey: env.DASHSCOPE_API_KEY || env.QWEN_API_KEY || '',
+      model: DEFAULT_MODEL_BY_PROVIDER['dashscope-byok']
+    };
+  }
+  return null;
+}
+
 export function saveConfig(config) {
   ensureConfigDir();
   // SECURITY: Deep copy to avoid mutating the in-memory config