ask-computer 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +15 -0
- data/LICENSE +21 -0
- data/README.md +141 -0
- data/exe/ask-computer +7 -0
- data/lib/ask/computer/app.rb +266 -0
- data/lib/ask/computer/cli.rb +138 -0
- data/lib/ask/computer/config.rb +22 -0
- data/lib/ask/computer/context.rb +32 -0
- data/lib/ask/computer/driver.rb +101 -0
- data/lib/ask/computer/errors.rb +45 -0
- data/lib/ask/computer/history.rb +204 -0
- data/lib/ask/computer/installer.rb +150 -0
- data/lib/ask/computer/sandbox.rb +84 -0
- data/lib/ask/computer/version.rb +7 -0
- data/lib/ask/computer.rb +70 -0
- data/lib/ask/skills/computer.use_computer/SKILL.md +121 -0
- data/lib/ask/skills/computer.use_computer/references/history.md +80 -0
- data/lib/ask/tools/computer/click.rb +27 -0
- data/lib/ask/tools/computer/history_query.rb +56 -0
- data/lib/ask/tools/computer/history_status.rb +36 -0
- data/lib/ask/tools/computer/installer_setup.rb +32 -0
- data/lib/ask/tools/computer/installer_status.rb +26 -0
- data/lib/ask/tools/computer/key.rb +23 -0
- data/lib/ask/tools/computer/screenshot.rb +22 -0
- data/lib/ask/tools/computer/type.rb +23 -0
- data/lib/ask-computer.rb +23 -0
- metadata +156 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: f378e2e365393b2cd3dce570227e5e9cf0c95f6c193a154b2752d3d8249eb8c0
|
|
4
|
+
data.tar.gz: 22f6341c0f2e846c79feb01ecc350dc880032073eac0b70a2045103aa91f69a5
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: dd7433804fd2c620263caed3aba79fac7afbcb77f9916ec8a5b710a26868e6afe115cc04ee1a4d8d66e1a1abfbfdbf060c12e08aacd70b2746ba6095c5520182
|
|
7
|
+
data.tar.gz: 954591cc19cfc242a2b28bbdd35c0e6dbaf544a554339b64322485ac1742e200543b3926ceb1f546bd9e4d4f473f37b0029983a9745f55c40c8f70e01a1661a4
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.1.0] - 2026-08-20
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- Initial release of `ask-computer` — computer use for the ask-rb ecosystem.
|
|
8
|
+
- `Ask::Computer` — driver/history/sandbox facades with `configure` / `reset!`.
|
|
9
|
+
- `Ask::Computer::Driver` — MCP stdio/HTTP client for Cua Driver (`call_tool`, `tools`, `history_available?`, `start`/`stop`).
|
|
10
|
+
- `Ask::Computer::History` — encrypted history client (`status`, `query`, `recent`, `each_page`) with RFC validation (1..200 limit, inclusive sequence bounds, session_id 1..128).
|
|
11
|
+
- `Ask::Computer::Sandbox` — VM sandbox facade (`screenshot`, `ephemeral` with `Handle`).
|
|
12
|
+
- `Ask::Computer::Error` hierarchy mapped from RFC codes (`HistoryKeyLocked`, `StorageCorrupt`, `QuotaReached`, etc.) via `Error.from_code`.
|
|
13
|
+
- `Ask::Tools::Computer::*` — `HistoryStatus`, `HistoryQuery`, `Screenshot`, `Click`, `Type`, `Key` as `Ask::Tool` subclasses.
|
|
14
|
+
- Bundled skill `computer.use_computer` (`SKILL.md` + `references/history.md`) discovered by `ask-skills` `Source::Gems` — no `ask-skills` runtime dep needed.
|
|
15
|
+
- Dependencies: `ask-core >= 0.11.3`, `ask-tools >= 0.6.2`.
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kaka Ruto
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# ask-computer
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/rb/ask-computer)
|
|
4
|
+
|
|
5
|
+
Computer use for the ask-rb ecosystem. Drive desktop apps in the background via Cua Driver — screenshots, clicks, typing, sandboxed VMs, and encrypted Computer History. Wraps Cua's MCP tools as `Ask::Tool` subclasses and ships a bundled `computer.use_computer` skill discovered by `ask-skills`.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
gem "ask-computer"
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
bundle install
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
### Cua Driver — Easy Setup
|
|
18
|
+
|
|
19
|
+
`ask-computer` talks to a running Cua Driver daemon via MCP (stdio `cua-driver mcp`).
|
|
20
|
+
|
|
21
|
+
**One command (recommended):**
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
bundle exec ask-computer setup --with-history # stable + history
|
|
25
|
+
bundle exec ask-computer setup --channel nightly --with-history # nightly preview for Computer History
|
|
26
|
+
bundle exec ask-computer setup --dry-run # preview without running
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Or step by step:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
bundle exec ask-computer install --channel nightly # or stable (default)
|
|
33
|
+
bundle exec ask-computer history enable
|
|
34
|
+
bundle exec ask-computer status # driver version + history health
|
|
35
|
+
bundle exec ask-computer history status # raw history JSON
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Ruby API for the same flow:
|
|
39
|
+
|
|
40
|
+
```ruby
|
|
41
|
+
require "ask-computer"
|
|
42
|
+
|
|
43
|
+
Ask::Computer.setup(channel: "nightly", enable_history: true)
|
|
44
|
+
Ask::Computer.status # => { installed: true, version: "cua-driver 0.21.0", history: { "health" => "ready" } }
|
|
45
|
+
Ask::Computer.install(channel: "stable", dry_run: true) # preview the shell command
|
|
46
|
+
Ask::Computer::Installer.version # => "cua-driver 0.21.0"
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Manual install still works: `/bin/bash -c "$(curl -fsSL https://cua.ai/driver/install.sh)"`
|
|
50
|
+
See https://cua.ai/docs and https://github.com/trycua/cua.
|
|
51
|
+
|
|
52
|
+
## Quick Start
|
|
53
|
+
|
|
54
|
+
```ruby
|
|
55
|
+
require "ask-computer"
|
|
56
|
+
|
|
57
|
+
# Connect to the local daemon
|
|
58
|
+
driver = Ask::Computer.driver
|
|
59
|
+
driver.start
|
|
60
|
+
|
|
61
|
+
# Drive the desktop (background — does not steal cursor)
|
|
62
|
+
driver.call_tool("computer_screenshot", {})
|
|
63
|
+
driver.call_tool("computer_click", { x: 120, y: 80 })
|
|
64
|
+
driver.call_tool("computer_type", { text: "Hello" })
|
|
65
|
+
driver.call_tool("computer_key", { key: "Enter" })
|
|
66
|
+
|
|
67
|
+
# Encrypted Computer History (permission-gated, metadata-only)
|
|
68
|
+
history = Ask::Computer.history
|
|
69
|
+
status = history.status
|
|
70
|
+
# => #<data Status supported=true admitted=true enabled=true ... health="ready">
|
|
71
|
+
|
|
72
|
+
if status.enabled? && status.healthy?
|
|
73
|
+
page = history.recent(limit: 20)
|
|
74
|
+
page.events.each { |e| puts "#{e.type} seq=#{e.data[:sequence]}" }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Bounded, paginated reads
|
|
78
|
+
history.each_page(limit: 50) do |page|
|
|
79
|
+
page.events.each { |e| puts e.type }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Sandbox VMs (requires driver with sandbox tools)
|
|
83
|
+
sandbox = Ask::Computer.sandbox
|
|
84
|
+
sandbox.ephemeral(:linux) do |vm|
|
|
85
|
+
vm.shell("echo hello")
|
|
86
|
+
vm.screenshot
|
|
87
|
+
vm.click(100, 200)
|
|
88
|
+
end
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### As Ask::Tool subclasses (for agent sessions)
|
|
92
|
+
|
|
93
|
+
```ruby
|
|
94
|
+
require "ask-computer"
|
|
95
|
+
|
|
96
|
+
# HistoryStatus / HistoryQuery map to history.status / history.query
|
|
97
|
+
# Screenshot / Click / Type / Key map to the corresponding Cua tools
|
|
98
|
+
Ask::Tools::Computer::HistoryStatus.new.execute
|
|
99
|
+
Ask::Tools::Computer::HistoryQuery.new.execute(limit: 20, session_id: "abc")
|
|
100
|
+
Ask::Tools::Computer::Screenshot.new.execute
|
|
101
|
+
Ask::Tools::Computer::Click.new.execute(x: 100, y: 200)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Configuration
|
|
105
|
+
|
|
106
|
+
```ruby
|
|
107
|
+
Ask::Computer.configure do |c|
|
|
108
|
+
c.driver_command = "cua-driver" # binary for stdio transport
|
|
109
|
+
c.driver_args = ["mcp"] # extra args (default: ["mcp"])
|
|
110
|
+
c.driver_url = nil # if set, uses HTTP transport instead of stdio
|
|
111
|
+
c.transport = :stdio # :stdio or :http
|
|
112
|
+
c.history_default_limit = 50
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Env vars
|
|
116
|
+
# ASK_COMPUTER_DRIVER_COMMAND — override driver binary (default: cua-driver)
|
|
117
|
+
# ASK_COMPUTER_DRIVER_ARGS — override driver args (default: mcp)
|
|
118
|
+
# ASK_COMPUTER_DRIVER_URL — if set, use HTTP transport
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
## Bundled Skill
|
|
122
|
+
|
|
123
|
+
The gem ships `lib/ask/skills/computer.use_computer/SKILL.md` (with `references/history.md`). It is auto-discovered by `ask-skills` via `Source::Gems` — no extra dependency needed. Load it in an agent:
|
|
124
|
+
|
|
125
|
+
```ruby
|
|
126
|
+
Ask::Skills.discover # finds computer.use_computer
|
|
127
|
+
registry["computer.use_computer"]
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## History Contract
|
|
131
|
+
|
|
132
|
+
- `History#status` → `Ask::Computer::Status` (supported/admitted/enabled/paused/health/retention/quota/bytes_used/dropped_events)
|
|
133
|
+
- `History#query(limit:, session_id:, since_sequence:, until_sequence:)` → `QueryResult` (events/metadata_only/model_context_disclosure)
|
|
134
|
+
- `History#recent` — alias for `query`
|
|
135
|
+
- `History#each_page` — paginates with `until_sequence` (inclusive bounds, handles gaps)
|
|
136
|
+
- Typed errors: `HistoryNotAdmitted`, `HistoryKeyLocked`, `HistoryStorageCorrupt`, `InvalidHistoryQuery`, `PermissionDenied`, etc.
|
|
137
|
+
- Events are CloudEvents 1.0, `dataschema: urn:cua-driver:schema:history-event:v0`, ordered by `data.sequence` ascending. Metadata-only — no screenshots or typed text.
|
|
138
|
+
|
|
139
|
+
## Full Documentation
|
|
140
|
+
|
|
141
|
+
The full ask-rb documentation lives at https://ask-rb.github.io/ask-docs.
|
data/exe/ask-computer
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ask
|
|
4
|
+
module Computer
|
|
5
|
+
# High-level facade for driving one desktop app.
|
|
6
|
+
#
|
|
7
|
+
# Wraps the four workarounds found live on macOS (Cua Driver 0.21.0):
|
|
8
|
+
#
|
|
9
|
+
# 1. `list_windows` only returns a count — probe `get_window_state`
|
|
10
|
+
# with small IDs to find the live window.
|
|
11
|
+
# 2. `press_key`/`type_text` without a pid targets the frontmost app,
|
|
12
|
+
# so `bring_to_front` first (falls back to pid-scoped keys when the
|
|
13
|
+
# app has multiple windows).
|
|
14
|
+
# 3. Apps with custom rendering (Calculator) expose `elements=0` — read
|
|
15
|
+
# state back through the clipboard (`Cmd+C`) instead of the AX tree.
|
|
16
|
+
# 4. Every unexpected payload surfaces a typed error that says what to
|
|
17
|
+
# try next, instead of a raw `Symbol into Integer` crash.
|
|
18
|
+
#
|
|
19
|
+
# app = Ask::Computer::App.launch("com.apple.calculator")
|
|
20
|
+
# app.type("7*8") # frontmost keystrokes
|
|
21
|
+
# app.press("return")
|
|
22
|
+
# app.display # clipboard readback for Calculator-like apps
|
|
23
|
+
# app.state # AX text when the app exposes it
|
|
24
|
+
#
|
|
25
|
+
class App
|
|
26
|
+
attr_reader :driver, :pid, :window_id, :bundle_id, :name
|
|
27
|
+
|
|
28
|
+
PROBE_IDS = (1..24).to_a.freeze
|
|
29
|
+
DISPLAY_CLEAR_KEYS = %w[escape].freeze
|
|
30
|
+
|
|
31
|
+
class << self
|
|
32
|
+
# Launch (or find) an app and resolve its live window.
|
|
33
|
+
#
|
|
34
|
+
# @param bundle_id_or_name [String] `com.apple.calculator` or `Calculator`
|
|
35
|
+
# @param driver [Driver] defaults to `Ask::Computer.driver` (must be started)
|
|
36
|
+
# @return [App]
|
|
37
|
+
# @raise [DriverNotAvailable] when no window can be resolved
|
|
38
|
+
def launch(bundle_id_or_name, driver: nil)
|
|
39
|
+
drv = driver || Ask::Computer.driver
|
|
40
|
+
new(pid: resolve_pid(drv, bundle_id_or_name), driver: drv)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def resolve_pid(driver, bundle_id_or_name)
|
|
46
|
+
id = bundle_id_or_name.to_s
|
|
47
|
+
result = driver.call_tool("launch_app", id.start_with?("com.") || id.include?(".") ? { bundle_id: id } : { name: id })
|
|
48
|
+
text = content_text(result)
|
|
49
|
+
pid = text.match(/pid (\d+)/)&.then { _1[1].to_i }
|
|
50
|
+
return pid if pid
|
|
51
|
+
|
|
52
|
+
apps = content_text(driver.call_tool("list_apps", {}))
|
|
53
|
+
short = id.split(".").last.downcase
|
|
54
|
+
line = apps.lines.find { |l| l.downcase.include?(short) && l.match(/pid (\d+)/) }
|
|
55
|
+
found = line&.match(/pid (\d+)/)&.then { _1[1].to_i }
|
|
56
|
+
return found if found
|
|
57
|
+
|
|
58
|
+
raise DriverNotAvailable, "Could not launch or find app #{id.inspect}. Tried bundle_id/name and list_apps."
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def content_text(result)
|
|
62
|
+
return result.to_s unless result.is_a?(Array)
|
|
63
|
+
|
|
64
|
+
result.map { |c| c[:text] || c["text"] || "" }.join("\n")
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def initialize(pid:, driver: nil, window_id: nil, bundle_id: nil, name: nil)
|
|
69
|
+
@driver = driver || Ask::Computer.driver
|
|
70
|
+
@pid = Integer(pid)
|
|
71
|
+
@window_id = window_id && Integer(window_id)
|
|
72
|
+
@resolved_window = !@window_id.nil?
|
|
73
|
+
@bundle_id = bundle_id
|
|
74
|
+
@name = name
|
|
75
|
+
@mutex = Mutex.new
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Resolve (and memoize) the live window for this pid by probing
|
|
79
|
+
# `get_window_state` — works when `list_windows` only returns a count.
|
|
80
|
+
#
|
|
81
|
+
# @return [Integer] the live window id
|
|
82
|
+
# @raise [DriverNotAvailable] when no live window is found
|
|
83
|
+
def resolve_window!(ids: PROBE_IDS)
|
|
84
|
+
@mutex.synchronize do
|
|
85
|
+
return @window_id if @window_id && @resolved_window
|
|
86
|
+
|
|
87
|
+
Array(ids).each do |wid|
|
|
88
|
+
if live_window?(wid)
|
|
89
|
+
@window_id = wid
|
|
90
|
+
@resolved_window = true
|
|
91
|
+
return @window_id
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
raise DriverNotAvailable,
|
|
96
|
+
"No live window for pid #{pid}. The app may not have opened a window yet, " \
|
|
97
|
+
"or its windows are on another Space. Try `bring_to_front` first."
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Bring the app to the foreground so pid-less keystrokes land.
|
|
102
|
+
# Falls back gracefully when the app owns several windows.
|
|
103
|
+
#
|
|
104
|
+
# @return [String] the driver message
|
|
105
|
+
def bring_to_front!
|
|
106
|
+
args = { pid: pid }
|
|
107
|
+
args[:window_id] = window_id if window_id
|
|
108
|
+
text = text_of(driver.call_tool("bring_to_front", args))
|
|
109
|
+
if text.match?(/more than one eligible/i)
|
|
110
|
+
resolve_window!
|
|
111
|
+
text = text_of(driver.call_tool("bring_to_front", { pid: pid, window_id: @window_id }))
|
|
112
|
+
end
|
|
113
|
+
text
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Type text into the app. Prefers the live window's text area when one
|
|
117
|
+
# is exposed, otherwise sends frontmost keystrokes after `bring_to_front`.
|
|
118
|
+
def type(text, element_index: nil)
|
|
119
|
+
bring_to_front!
|
|
120
|
+
args = { text: text.to_s }
|
|
121
|
+
args[:pid] = pid
|
|
122
|
+
args[:window_id] = window_id if window_id
|
|
123
|
+
args[:element_index] = element_index if element_index
|
|
124
|
+
text_of(driver.call_tool("type_text", args))
|
|
125
|
+
rescue Ask::Computer::Error => e
|
|
126
|
+
if e.message.match?(/more than one eligible/i)
|
|
127
|
+
resolve_window!
|
|
128
|
+
retry
|
|
129
|
+
end
|
|
130
|
+
raise friendly_error("type_text", e)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Press a single key (`return`, `escape`, `7`, `*`, ...).
|
|
134
|
+
# Sends pid-scoped first, falls back to frontmost (no pid) when the
|
|
135
|
+
# driver refuses (e.g. off-Space window).
|
|
136
|
+
def press(key)
|
|
137
|
+
wid = window_id || resolve_window!
|
|
138
|
+
begin
|
|
139
|
+
return text_of(driver.call_tool("press_key", { pid: pid, window_id: wid, key: key.to_s }))
|
|
140
|
+
rescue Ask::Computer::Error => e
|
|
141
|
+
raise friendly_error("press_key", e) unless fallbackable?(e)
|
|
142
|
+
end
|
|
143
|
+
bring_to_front!
|
|
144
|
+
text_of(driver.call_tool("press_key", { key: key.to_s }))
|
|
145
|
+
rescue Ask::Computer::Error => e
|
|
146
|
+
raise friendly_error("press_key", e)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Raw AX markdown for the live window. Raises {EmptyAccessibilityTree}
|
|
150
|
+
# with guidance when the app renders custom content (Calculator).
|
|
151
|
+
def state(include_screenshot: false)
|
|
152
|
+
wid = window_id || resolve_window!
|
|
153
|
+
payload = driver.call_tool(
|
|
154
|
+
"get_window_state",
|
|
155
|
+
{ pid: pid, window_id: wid, include_screenshot: include_screenshot }
|
|
156
|
+
)
|
|
157
|
+
text = text_of(payload)
|
|
158
|
+
raise EmptyAccessibilityTree.new(self) if text.match?(/elements=0/)
|
|
159
|
+
|
|
160
|
+
text
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Read the app's visible value via the clipboard (`Cmd+C`).
|
|
164
|
+
# The Calculator workaround: its display is `AXStaticText` with
|
|
165
|
+
# `elements=0`, so the AX tree can't be read — copy and read back.
|
|
166
|
+
#
|
|
167
|
+
# @return [String] clipboard text after copy
|
|
168
|
+
def display
|
|
169
|
+
d = @driver
|
|
170
|
+
begin
|
|
171
|
+
attempt_display(d)
|
|
172
|
+
rescue Ask::Computer::Error => e
|
|
173
|
+
raise friendly_error("display", e) unless fallbackable?(e)
|
|
174
|
+
|
|
175
|
+
@mutex.synchronize { @window_id = nil }
|
|
176
|
+
resolve_window!
|
|
177
|
+
attempt_display(d)
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Clear the app's entry field (`escape` by default).
|
|
182
|
+
def clear(key: "escape")
|
|
183
|
+
press(key)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
private
|
|
187
|
+
|
|
188
|
+
def attempt_display(driver)
|
|
189
|
+
wid = window_id || resolve_window!
|
|
190
|
+
driver.call_tool("hotkey", { pid: pid, window_id: wid, keys: %w[cmd c] })
|
|
191
|
+
sleep 0.2
|
|
192
|
+
read = driver.call_tool("clipboard_read", {})
|
|
193
|
+
clipboard_text(read) || raise(
|
|
194
|
+
Ask::Computer::Error,
|
|
195
|
+
"clipboard_read returned no text for pid #{pid}. " \
|
|
196
|
+
"The app may not support Cmd+C; use #state instead."
|
|
197
|
+
)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def live_window?(wid)
|
|
201
|
+
return false unless wid
|
|
202
|
+
|
|
203
|
+
payload = driver.call_tool(
|
|
204
|
+
"get_window_state",
|
|
205
|
+
{ pid: pid, window_id: wid, include_screenshot: false }
|
|
206
|
+
)
|
|
207
|
+
text_of(payload).match?(/elements=\d+/) &&
|
|
208
|
+
!text_of(payload).match?(/not a live window|stale/i)
|
|
209
|
+
rescue Ask::Computer::Error
|
|
210
|
+
false
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def fallbackable?(error)
|
|
214
|
+
error.message.match?(/off_space|not among|more than one eligible|Unknown tool/i)
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def friendly_error(tool, error)
|
|
218
|
+
message = error.message.to_s
|
|
219
|
+
hint =
|
|
220
|
+
if message.match?(/bare element_index/)
|
|
221
|
+
" Pass element_index together with the window's snapshot, or use pixel x/y on the window PNG."
|
|
222
|
+
elsif message.match?(/off_space|not among the process/)
|
|
223
|
+
" The window may be on another Space. Called bring_to_front first — verify the app is visible."
|
|
224
|
+
elsif message.match?(/elements=0|empty/i)
|
|
225
|
+
" The app renders custom content with no AX tree. Use #display (clipboard readback) instead of #state."
|
|
226
|
+
elsif message.match?(/Unknown tool/i)
|
|
227
|
+
" The driver build does not advertise this tool (stable vs nightly). Check history_available? / installer status."
|
|
228
|
+
end
|
|
229
|
+
Ask::Computer::Error.new("#{tool} failed for pid #{pid}: #{message}#{hint}")
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def text_of(result)
|
|
233
|
+
return result.to_s unless result.is_a?(Array)
|
|
234
|
+
|
|
235
|
+
result.map { |c| c[:text] || c["text"] || "" }.join("\n")
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def clipboard_text(result)
|
|
239
|
+
parts = result.is_a?(Array) ? result : [result]
|
|
240
|
+
parts.each do |part|
|
|
241
|
+
next unless part.is_a?(Hash)
|
|
242
|
+
|
|
243
|
+
text = part[:text] || part["text"]
|
|
244
|
+
return text unless text.nil? || text.match?(/omitted|privacy-sensitive/i)
|
|
245
|
+
sc = part[:structuredContent] || part["structuredContent"]
|
|
246
|
+
if sc.is_a?(Hash)
|
|
247
|
+
t = sc[:text] || sc["text"]
|
|
248
|
+
return t if t && !t.empty?
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
nil
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Raised by {App#state} when the app exposes `elements=0`
|
|
256
|
+
# (custom rendering, e.g. Calculator) — use {App#display} instead.
|
|
257
|
+
class EmptyAccessibilityTree < Error
|
|
258
|
+
def initialize(app)
|
|
259
|
+
super(
|
|
260
|
+
"pid #{app.pid} exposes elements=0 (custom rendering, e.g. Calculator). " \
|
|
261
|
+
"Use #display (Cmd+C clipboard readback) or vision on the window PNG instead of #state."
|
|
262
|
+
)
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
end
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
|
|
5
|
+
module Ask
|
|
6
|
+
module Computer
|
|
7
|
+
module CLI
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def run(argv = ARGV)
|
|
11
|
+
argv = argv.dup
|
|
12
|
+
command = argv.shift
|
|
13
|
+
|
|
14
|
+
case command
|
|
15
|
+
when "install" then cmd_install(argv)
|
|
16
|
+
when "setup" then cmd_setup(argv)
|
|
17
|
+
when "status" then cmd_status
|
|
18
|
+
when "history" then cmd_history(argv)
|
|
19
|
+
when "uninstall" then cmd_uninstall(argv)
|
|
20
|
+
when "help", "--help", "-h", nil then cmd_help
|
|
21
|
+
else
|
|
22
|
+
warn "Unknown command: #{command}"
|
|
23
|
+
cmd_help
|
|
24
|
+
1
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def cmd_install(argv)
|
|
29
|
+
opts = { channel: "stable", bin_dir: nil, no_modify_path: false, dry_run: false }
|
|
30
|
+
parser = OptionParser.new do |o|
|
|
31
|
+
o.banner = "Usage: ask-computer install [options]"
|
|
32
|
+
o.on("--channel NAME", "stable or nightly [stable]") { |v| opts[:channel] = v }
|
|
33
|
+
o.on("--bin-dir PATH", "Install dir (default ~/.local/bin)") { |v| opts[:bin_dir] = v }
|
|
34
|
+
o.on("--no-modify-path", "Do not append PATH export to shell rc") { opts[:no_modify_path] = true }
|
|
35
|
+
o.on("--dry-run", "Print the install command without running it") { opts[:dry_run] = true }
|
|
36
|
+
o.on("-h", "--help", "Show help") { puts o; return 0 }
|
|
37
|
+
end
|
|
38
|
+
parser.parse!(argv)
|
|
39
|
+
result = Installer.install(**opts)
|
|
40
|
+
puts result.message
|
|
41
|
+
warn result.stderr unless result.stderr.empty?
|
|
42
|
+
result.ok? ? 0 : 1
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def cmd_setup(argv)
|
|
46
|
+
opts = { channel: "stable", enable_history: false, bin_dir: nil, no_modify_path: false, dry_run: false }
|
|
47
|
+
parser = OptionParser.new do |o|
|
|
48
|
+
o.banner = "Usage: ask-computer setup [options]"
|
|
49
|
+
o.on("--channel NAME", "stable or nightly [stable]") { |v| opts[:channel] = v }
|
|
50
|
+
o.on("--with-history", "Enable Computer History after install") { opts[:enable_history] = true }
|
|
51
|
+
o.on("--bin-dir PATH", "Install dir") { |v| opts[:bin_dir] = v }
|
|
52
|
+
o.on("--no-modify-path", "Do not modify shell rc") { opts[:no_modify_path] = true }
|
|
53
|
+
o.on("--dry-run", "Print commands without running") { opts[:dry_run] = true }
|
|
54
|
+
o.on("-h", "--help", "Show help") { puts o; return 0 }
|
|
55
|
+
end
|
|
56
|
+
parser.parse!(argv)
|
|
57
|
+
steps = Installer.setup(**opts)
|
|
58
|
+
steps.each do |name, result|
|
|
59
|
+
label = name == :install ? "Install" : "History enable"
|
|
60
|
+
puts "#{label}: #{result.ok? ? "ok" : "failed"} — #{result.message}"
|
|
61
|
+
warn result.stderr unless result.stderr.empty?
|
|
62
|
+
end
|
|
63
|
+
steps.all? { |_, r| r.ok? } ? 0 : 1
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def cmd_status
|
|
67
|
+
s = Installer.status
|
|
68
|
+
puts "Installed: #{s[:installed] ? "yes (#{s[:bin]})" : "no"}"
|
|
69
|
+
puts "Version: #{s[:version] || "(unknown)"}" if s[:installed]
|
|
70
|
+
h = s[:history]
|
|
71
|
+
if h.is_a?(Hash) && h[:ok] == false
|
|
72
|
+
puts "History: #{h[:message] || "unavailable"}"
|
|
73
|
+
elsif h.is_a?(Hash)
|
|
74
|
+
puts "History: #{h["health"] || h[:health] || h.inspect[0,120]}"
|
|
75
|
+
else
|
|
76
|
+
puts "History: #{h.inspect[0,120]}"
|
|
77
|
+
end
|
|
78
|
+
0
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def cmd_history(argv)
|
|
82
|
+
sub = argv.shift
|
|
83
|
+
case sub
|
|
84
|
+
when "enable"
|
|
85
|
+
r = Installer.enable_history
|
|
86
|
+
puts r.message
|
|
87
|
+
r.ok? ? 0 : 1
|
|
88
|
+
when "disable"
|
|
89
|
+
r = Installer.disable_history
|
|
90
|
+
puts r.message
|
|
91
|
+
r.ok? ? 0 : 1
|
|
92
|
+
when "status"
|
|
93
|
+
r = Installer.history_status
|
|
94
|
+
puts r.stdout.empty? ? r.message : r.stdout
|
|
95
|
+
warn r.stderr unless r.stderr.empty?
|
|
96
|
+
r.ok? ? 0 : 1
|
|
97
|
+
else
|
|
98
|
+
puts "Usage: ask-computer history <enable|disable|status>"
|
|
99
|
+
1
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def cmd_uninstall(argv)
|
|
104
|
+
purge = argv.include?("--purge")
|
|
105
|
+
r = Installer.uninstall(purge: purge)
|
|
106
|
+
puts r.message
|
|
107
|
+
r.ok? ? 0 : 1
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def cmd_help
|
|
111
|
+
puts <<~HELP
|
|
112
|
+
ask-computer — Cua Driver helper for the ask-computer gem
|
|
113
|
+
|
|
114
|
+
Usage:
|
|
115
|
+
ask-computer install [--channel stable|nightly] [--dry-run]
|
|
116
|
+
ask-computer setup [--channel nightly] [--with-history] [--dry-run]
|
|
117
|
+
ask-computer status
|
|
118
|
+
ask-computer history <enable|disable|status>
|
|
119
|
+
ask-computer uninstall [--purge]
|
|
120
|
+
|
|
121
|
+
Examples:
|
|
122
|
+
ask-computer install # stable channel
|
|
123
|
+
ask-computer install --channel nightly # nightly (for Computer History preview)
|
|
124
|
+
ask-computer setup --channel nightly --with-history
|
|
125
|
+
ask-computer status
|
|
126
|
+
ask-computer history status --json
|
|
127
|
+
|
|
128
|
+
Ruby API:
|
|
129
|
+
Ask::Computer.install(channel: "nightly", dry_run: true)
|
|
130
|
+
Ask::Computer.setup(channel: "nightly", enable_history: true)
|
|
131
|
+
Ask::Computer.status
|
|
132
|
+
Ask::Computer::Installer.version
|
|
133
|
+
HELP
|
|
134
|
+
0
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ask
|
|
4
|
+
module Computer
|
|
5
|
+
class Config
|
|
6
|
+
attr_accessor :driver_command, :driver_args, :driver_url, :transport
|
|
7
|
+
attr_accessor :history_default_limit
|
|
8
|
+
|
|
9
|
+
def initialize
|
|
10
|
+
@driver_command = ENV.fetch("ASK_COMPUTER_DRIVER_COMMAND", "cua-driver")
|
|
11
|
+
@driver_args = ENV["ASK_COMPUTER_DRIVER_ARGS"] ? ENV["ASK_COMPUTER_DRIVER_ARGS"].split : ["mcp"]
|
|
12
|
+
@driver_url = ENV["ASK_COMPUTER_DRIVER_URL"]
|
|
13
|
+
@transport = :stdio
|
|
14
|
+
@history_default_limit = 50
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def driver_url?
|
|
18
|
+
driver_url && !driver_url.empty?
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ask
|
|
4
|
+
module Computer
|
|
5
|
+
DESCRIPTION = "Computer use via Cua Driver — drive desktop apps in the background, " \
|
|
6
|
+
"take screenshots, click, type, and query encrypted Computer History"
|
|
7
|
+
DOCS_URL = "https://cua.ai/docs"
|
|
8
|
+
DRIVER_INSTALL_URL = "https://cua.ai/driver/install.sh"
|
|
9
|
+
HISTORY_DOCS_URL = "https://github.com/trycua/cua/blob/main/libs/cua-driver/docs/computer-history-preview.md"
|
|
10
|
+
AUTH_NAME = :cua_driver
|
|
11
|
+
AUTH_HOW = "Install Cua Driver: /bin/bash -c \"$(curl -fsSL https://cua.ai/driver/install.sh)\" — " \
|
|
12
|
+
"see https://cua.ai/docs/tutorials/drive-your-first-app"
|
|
13
|
+
GEM_NAME = "cua-driver"
|
|
14
|
+
GEM_DOCS = "https://cua.ai/docs"
|
|
15
|
+
QUICK_START = <<~RUBY
|
|
16
|
+
require "ask-computer"
|
|
17
|
+
|
|
18
|
+
# Connect to the local Cua Driver daemon
|
|
19
|
+
driver = Ask::Computer.driver
|
|
20
|
+
driver.start
|
|
21
|
+
|
|
22
|
+
# Drive the desktop
|
|
23
|
+
driver.call_tool("computer_screenshot", {})
|
|
24
|
+
driver.call_tool("computer_click", { x: 100, y: 200 })
|
|
25
|
+
|
|
26
|
+
# Query encrypted Computer History
|
|
27
|
+
history = Ask::Computer.history
|
|
28
|
+
history.status
|
|
29
|
+
history.recent(limit: 20)
|
|
30
|
+
RUBY
|
|
31
|
+
end
|
|
32
|
+
end
|