otoro-cli 1.3.0 → 1.4.0

package/bin/otoro.js

@@ -68,6 +68,35 @@ program
     await generateImage(prompt.join(' '))
   })
 
+program
+  .command('computer <task...>')
+  .alias('use')
+  .description('Computer Use — Otoro sees your screen and controls mouse/keyboard')
+  .action(async (task) => {
+    requireAuth()
+    const { runComputerTask } = require('../lib/computer')
+    await runComputerTask(task.join(' '))
+  })
+
+program
+  .command('screen')
+  .description('Take a screenshot and describe what Otoro sees')
+  .action(async () => {
+    requireAuth()
+    const { analyzeScreen } = require('../lib/screen')
+    const chalk = require('chalk')
+    const ora = require('ora')
+    const spinner = ora({ text: chalk.gray('Looking at screen...'), color: 'cyan' }).start()
+    const result = await analyzeScreen()
+    if (result.success) {
+      spinner.succeed(chalk.green('Screen analyzed'))
+      console.log(chalk.cyan('\n  What Otoro sees:\n'))
+      console.log('  ' + result.description.split('\n').join('\n  ') + '\n')
+    } else {
+      spinner.fail(chalk.red(result.error))
+    }
+  })
+
 program
   .command('start')
   .description('Start Otoro agent daemon — connects to server for remote tasks')

package/lib/agent.js

@@ -4,6 +4,8 @@ const path = require('path')
 const chalk = require('chalk')
 const { getConfig } = require('./config')
 const { readFile, writeFile, editFile, listFiles, runCommand, searchCode, openApp, openUrl, getSystemInfo, takeScreenshot } = require('./tools')
+const { mouseClick, mouseMove, typeText, pressKey, analyzeScreen } = require('./screen')
+const { runComputerTask } = require('./computer')
 const { chatCompletion } = require('./api')
 
 class OtoroAgent {
@@ -132,6 +134,25 @@ class OtoroAgent {
         case 'system_info':
           result = getSystemInfo()
           break
+        case 'mouse_click':
+          result = mouseClick(payload.x, payload.y, payload.button)
+          break
+        case 'mouse_move':
+          result = mouseMove(payload.x, payload.y)
+          break
+        case 'type_text':
+          result = typeText(payload.text)
+          break
+        case 'press_key':
+          result = pressKey(payload.key)
+          break
+        case 'analyze_screen':
+          result = await analyzeScreen(payload.question)
+          break
+        case 'computer_use':
+          console.log(chalk.cyan(`  🖥️  Computer Use: ${payload.task}`))
+          result = await runComputerTask(payload.task, payload.max_steps || 10)
+          break
         case 'list_files':
           result = listFiles(payload.dir || '.', payload.pattern || '')
           break

package/lib/computer.js

@@ -0,0 +1,74 @@
+const chalk = require('chalk')
+const ora = require('ora')
+const { analyzeScreen, computerUseStep, executeComputerActions, takeScreenshot } = require('./screen')
+
+async function runComputerTask(task, maxSteps = 15) {
+  console.log(chalk.cyan.bold('\n  🐙 Otoro Computer Use\n'))
+  console.log(chalk.gray(`  Task: ${task}`))
+  console.log(chalk.gray(`  Platform: ${process.platform}\n`))
+
+  // Step 1: See the screen
+  let spinner = ora({ text: chalk.gray('Looking at screen...'), color: 'cyan' }).start()
+  let screen = await analyzeScreen()
+  if (!screen.success) {
+    spinner.fail(chalk.red(`Can't see screen: ${screen.error}`))
+    console.log(chalk.yellow('\n  Tips:'))
+    if (process.platform === 'linux') console.log(chalk.gray('  • Install scrot or gnome-screenshot: sudo apt install scrot'))
+    if (process.platform === 'darwin') console.log(chalk.gray('  • Grant Screen Recording permission: System Settings → Privacy → Screen Recording'))
+    return
+  }
+  spinner.succeed(chalk.green('Screen captured'))
+  console.log(chalk.gray(`  I see: ${screen.description.slice(0, 150)}...\n`))
+
+  // Step 2: Loop — AI sees screen, decides action, executes, repeat
+  for (let step = 1; step <= maxSteps; step++) {
+    console.log(chalk.cyan(`  Step ${step}/${maxSteps}`))
+
+    spinner = ora({ text: chalk.gray('Deciding action...'), color: 'cyan' }).start()
+    const aiResponse = await computerUseStep(task, screen.description)
+    spinner.stop()
+
+    if (!aiResponse) {
+      console.log(chalk.red('  No response from AI'))
+      break
+    }
+
+    // Show what Otoro is thinking (strip action tags for display)
+    const thinking = aiResponse.replace(/<action:[^>]*\/>/g, '').trim()
+    if (thinking) console.log(chalk.gray(`  ${thinking.split('\n')[0].slice(0, 100)}`))
+
+    // Execute actions
+    const { results, isDone } = await executeComputerActions(aiResponse)
+
+    for (const r of results) {
+      if (r.success) {
+        if (r.x !== undefined) console.log(chalk.green(`  ✓ Click (${r.x}, ${r.y})`))
+        else if (r.typed) console.log(chalk.green(`  ✓ Typed ${r.typed}`))
+        else if (r.key) console.log(chalk.green(`  ✓ Pressed ${r.key}`))
+        else if (r.waited) console.log(chalk.green(`  ✓ Waited ${r.waited}`))
+      } else {
+        console.log(chalk.red(`  ✗ ${r.error}`))
+      }
+    }
+
+    if (isDone) {
+      console.log(chalk.green.bold('\n  ✓ Task complete!\n'))
+      return
+    }
+
+    // Wait a moment then re-capture screen
+    await new Promise(r => setTimeout(r, 1000))
+    spinner = ora({ text: chalk.gray('Looking at updated screen...'), color: 'cyan' }).start()
+    screen = await analyzeScreen()
+    spinner.stop()
+
+    if (screen.success) {
+      console.log(chalk.gray(`  Screen: ${screen.description.slice(0, 100)}...`))
+    }
+    console.log()
+  }
+
+  console.log(chalk.yellow(`\n  Reached max steps (${maxSteps}). Task may be incomplete.\n`))
+}
+
+module.exports = { runComputerTask }

package/lib/screen.js

@@ -0,0 +1,281 @@
+const { execSync, exec } = require('child_process')
+const fs = require('fs')
+const path = require('path')
+const os = require('os')
+const chalk = require('chalk')
+const http = require('http')
+const { getConfig } = require('./config')
+
+const SCREENSHOT_DIR = path.join(os.tmpdir(), 'otoro-screenshots')
+fs.mkdirSync(SCREENSHOT_DIR, { recursive: true })
+
+// ─── Screenshots ──────────────────────────────────────────────────────────────
+
+function takeScreenshot() {
+  const platform = process.platform
+  const file = path.join(SCREENSHOT_DIR, `screen-${Date.now()}.png`)
+  try {
+    if (platform === 'darwin') {
+      execSync(`screencapture -x "${file}"`, { timeout: 5000 })
+    } else if (platform === 'win32') {
+      // PowerShell screenshot
+      execSync(`powershell -command "Add-Type -AssemblyName System.Windows.Forms; $screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap($screen.Width, $screen.Height); $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen(0, 0, 0, 0, $screen.Size); $bitmap.Save('${file.replace(/\\/g, '\\\\')}'); $graphics.Dispose(); $bitmap.Dispose()"`, { timeout: 10000 })
+    } else {
+      // Linux — try multiple tools
+      try { execSync(`gnome-screenshot -f "${file}" 2>/dev/null`, { timeout: 5000 }) }
+      catch { 
+        try { execSync(`scrot "${file}" 2>/dev/null`, { timeout: 5000 }) }
+        catch { execSync(`import -window root "${file}" 2>/dev/null`, { timeout: 5000 }) }
+      }
+    }
+    if (fs.existsSync(file)) return { success: true, path: file, size: fs.statSync(file).size }
+    return { success: false, error: 'Screenshot not created' }
+  } catch (e) {
+    return { success: false, error: e.message }
+  }
+}
+
+// ─── Mouse Control ────────────────────────────────────────────────────────────
+
+function mouseClick(x, y, button = 'left') {
+  const platform = process.platform
+  try {
+    if (platform === 'darwin') {
+      const btn = button === 'right' ? 'rc' : 'c'
+      execSync(`osascript -e 'tell application "System Events" to click at {${x}, ${y}}'`, { timeout: 3000 })
+    } else if (platform === 'win32') {
+      const btnCode = button === 'right' ? '$right = $true' : ''
+      execSync(`powershell -command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x}, ${y}); Add-Type -MemberDefinition '[DllImport(\\\"user32.dll\\\")] public static extern void mouse_event(int f,int x,int y,int d,int i);' -Name U -Namespace W; [W.U]::mouse_event(${button === 'right' ? '0x0008' : '0x0002'},0,0,0,0); [W.U]::mouse_event(${button === 'right' ? '0x0010' : '0x0004'},0,0,0,0)"`, { timeout: 5000 })
+    } else {
+      execSync(`xdotool mousemove ${x} ${y} click ${button === 'right' ? '3' : '1'}`, { timeout: 3000 })
+    }
+    return { success: true, x, y, button }
+  } catch (e) {
+    return { success: false, error: e.message }
+  }
+}
+
+function mouseMove(x, y) {
+  const platform = process.platform
+  try {
+    if (platform === 'darwin') {
+      execSync(`osascript -e 'tell application "System Events" to set position of cursor to {${x}, ${y}}'`, { timeout: 3000 })
+    } else if (platform === 'win32') {
+      execSync(`powershell -command "[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x}, ${y})"`, { timeout: 3000 })
+    } else {
+      execSync(`xdotool mousemove ${x} ${y}`, { timeout: 3000 })
+    }
+    return { success: true, x, y }
+  } catch (e) {
+    return { success: false, error: e.message }
+  }
+}
+
+// ─── Keyboard Control ─────────────────────────────────────────────────────────
+
+function typeText(text) {
+  const platform = process.platform
+  try {
+    if (platform === 'darwin') {
+      // Escape special chars for AppleScript
+      const escaped = text.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
+      execSync(`osascript -e 'tell application "System Events" to keystroke "${escaped}"'`, { timeout: 5000 })
+    } else if (platform === 'win32') {
+      const escaped = text.replace(/'/g, "''")
+      execSync(`powershell -command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${escaped}')"`, { timeout: 5000 })
+    } else {
+      execSync(`xdotool type --clearmodifiers "${text.replace(/"/g, '\\"')}"`, { timeout: 5000 })
+    }
+    return { success: true, typed: text.length + ' chars' }
+  } catch (e) {
+    return { success: false, error: e.message }
+  }
+}
+
+function pressKey(key) {
+  // key: "enter", "tab", "escape", "backspace", "ctrl+c", "cmd+s", etc.
+  const platform = process.platform
+  try {
+    if (platform === 'darwin') {
+      const keyMap = { enter: 'return', tab: 'tab', escape: 'escape', backspace: 'delete', space: 'space' }
+      const mapped = keyMap[key.toLowerCase()] || key.toLowerCase()
+      if (key.includes('+')) {
+        const [mod, k] = key.split('+')
+        const modMap = { ctrl: 'control', cmd: 'command', alt: 'option', shift: 'shift' }
+        execSync(`osascript -e 'tell application "System Events" to key code 0 using {${modMap[mod] || mod} down}'`, { timeout: 3000 })
+      } else {
+        execSync(`osascript -e 'tell application "System Events" to keystroke "${mapped}"'`, { timeout: 3000 })
+      }
+    } else if (platform === 'win32') {
+      const keyMap = { enter: '{ENTER}', tab: '{TAB}', escape: '{ESC}', backspace: '{BS}', space: ' ' }
+      const mapped = keyMap[key.toLowerCase()] || `{${key.toUpperCase()}}`
+      execSync(`powershell -command "[System.Windows.Forms.SendKeys]::SendWait('${mapped}')"`, { timeout: 3000 })
+    } else {
+      execSync(`xdotool key ${key.replace('cmd', 'super').replace('ctrl', 'ctrl')}`, { timeout: 3000 })
+    }
+    return { success: true, key }
+  } catch (e) {
+    return { success: false, error: e.message }
+  }
+}
+
+// ─── Vision — Send screenshot to Qwen-VL for understanding ───────────────────
+
+async function analyzeScreen(question = 'What is on the screen? Describe the UI elements, buttons, and text visible.') {
+  const screenshot = takeScreenshot()
+  if (!screenshot.success) return { success: false, error: screenshot.error }
+
+  const config = getConfig()
+  const imageData = fs.readFileSync(screenshot.path)
+  const b64 = imageData.toString('base64')
+
+  // Send to Qwen-VL vision model
+  const body = JSON.stringify({
+    model: 'qwen-vl',
+    messages: [
+      { role: 'user', content: [
+        { type: 'text', text: question },
+        { type: 'image_url', image_url: { url: `data:image/png;base64,${b64}` } }
+      ]}
+    ],
+    max_tokens: 1024,
+  })
+
+  return new Promise((resolve) => {
+    const url = new URL(`${config.gpu_url}/v1/chat/completions`)
+    const req = http.request(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${config.api_key}` },
+      timeout: 30000,
+    }, (res) => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try {
+          const result = JSON.parse(data)
+          const description = result.choices?.[0]?.message?.content || ''
+          resolve({ success: true, description, screenshot: screenshot.path })
+        } catch { resolve({ success: false, error: 'Bad response from vision model' }) }
+      })
+    })
+    req.on('error', (e) => resolve({ success: false, error: e.message }))
+    req.write(body)
+    req.end()
+  })
+}
+
+// ─── Live Screen Monitor ──────────────────────────────────────────────────────
+
+class ScreenMonitor {
+  constructor(intervalMs = 3000) {
+    this.interval = intervalMs
+    this.running = false
+    this.timer = null
+    this.lastDescription = ''
+    this.onUpdate = null
+  }
+
+  start(callback) {
+    this.running = true
+    this.onUpdate = callback
+    console.log(chalk.cyan(`  👁  Screen monitor started (every ${this.interval / 1000}s)`))
+    this.tick()
+  }
+
+  stop() {
+    this.running = false
+    if (this.timer) clearTimeout(this.timer)
+    console.log(chalk.gray('  Screen monitor stopped'))
+  }
+
+  async tick() {
+    if (!this.running) return
+    const result = await analyzeScreen('Briefly describe what is currently visible on screen. Note any dialogs, windows, error messages, or UI changes.')
+    if (result.success && result.description !== this.lastDescription) {
+      this.lastDescription = result.description
+      if (this.onUpdate) this.onUpdate(result.description, result.screenshot)
+    }
+    this.timer = setTimeout(() => this.tick(), this.interval)
+  }
+}
+
+// ─── Computer Use Agent — AI sees screen and controls computer ────────────────
+
+async function computerUseStep(task, screenDescription) {
+  const config = getConfig()
+  const body = JSON.stringify({
+    model: 'qwen-coder',
+    messages: [
+      { role: 'system', content: `You are Otoro controlling a ${process.platform} computer. You can see the screen and control mouse/keyboard.
+
+Available actions (use XML tags):
+- <action:click x="123" y="456"/> — click at coordinates
+- <action:rightclick x="123" y="456"/> — right-click
+- <action:type text="hello world"/> — type text
+- <action:key press="enter"/> — press a key (enter, tab, escape, ctrl+c, cmd+s, etc.)
+- <action:move x="123" y="456"/> — move mouse
+- <action:screenshot/> — take a new screenshot
+- <action:wait ms="1000"/> — wait before next action
+- <action:done/> — task is complete
+
+Current screen: ${screenDescription}
+
+Execute the task step by step. After each action, I'll show you the updated screen.` },
+      { role: 'user', content: task }
+    ],
+    max_tokens: 2048,
+  })
+
+  return new Promise((resolve) => {
+    const url = new URL(`${config.gpu_url}/v1/chat/completions`)
+    const req = http.request(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${config.api_key}` },
+      timeout: 60000,
+    }, (res) => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try {
+          const result = JSON.parse(data)
+          resolve(result.choices?.[0]?.message?.content || '')
+        } catch { resolve('') }
+      })
+    })
+    req.on('error', () => resolve(''))
+    req.write(body)
+    req.end()
+  })
+}
+
+async function executeComputerActions(response) {
+  const results = []
+
+  for (const match of response.matchAll(/<action:click\s+x="(\d+)"\s+y="(\d+)"\s*\/>/g)) {
+    results.push(mouseClick(parseInt(match[1]), parseInt(match[2])))
+  }
+  for (const match of response.matchAll(/<action:rightclick\s+x="(\d+)"\s+y="(\d+)"\s*\/>/g)) {
+    results.push(mouseClick(parseInt(match[1]), parseInt(match[2]), 'right'))
+  }
+  for (const match of response.matchAll(/<action:type\s+text="([^"]+)"\s*\/>/g)) {
+    results.push(typeText(match[1]))
+  }
+  for (const match of response.matchAll(/<action:key\s+press="([^"]+)"\s*\/>/g)) {
+    results.push(pressKey(match[1]))
+  }
+  for (const match of response.matchAll(/<action:move\s+x="(\d+)"\s+y="(\d+)"\s*\/>/g)) {
+    results.push(mouseMove(parseInt(match[1]), parseInt(match[2])))
+  }
+  for (const match of response.matchAll(/<action:wait\s+ms="(\d+)"\s*\/>/g)) {
+    await new Promise(r => setTimeout(r, parseInt(match[1])))
+    results.push({ success: true, waited: match[1] + 'ms' })
+  }
+
+  const isDone = response.includes('<action:done/>')
+  return { results, isDone }
+}
+
+module.exports = {
+  takeScreenshot, mouseClick, mouseMove, typeText, pressKey,
+  analyzeScreen, ScreenMonitor, computerUseStep, executeComputerActions
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "otoro-cli",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Otoro AGI — AI coding assistant for your terminal. Code, generate images, execute tasks, and control your projects remotely.",
   "main": "index.js",
   "bin": {