wrangle 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +87 -0
- data/LICENSE.txt +48 -0
- data/README.md +473 -0
- data/exe/wrangle +374 -0
- data/lib/wrangle/action_space.rb +67 -0
- data/lib/wrangle/decider.rb +167 -0
- data/lib/wrangle/errors.rb +53 -0
- data/lib/wrangle/jev.rb +87 -0
- data/lib/wrangle/js/bridge.js +390 -0
- data/lib/wrangle/js/page.js +127 -0
- data/lib/wrangle/js/snapshot.js +118 -0
- data/lib/wrangle/jxa_bridge.rb +194 -0
- data/lib/wrangle/mcp_bridge.rb +300 -0
- data/lib/wrangle/observation.rb +50 -0
- data/lib/wrangle/run_loop.rb +298 -0
- data/lib/wrangle/safari.rb +404 -0
- data/lib/wrangle/session_server.rb +447 -0
- data/lib/wrangle/version.rb +5 -0
- data/lib/wrangle.rb +19 -0
- data/skills/wrangle/SKILL.md +206 -0
- metadata +69 -0
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "json"
|
|
5
|
+
require "socket"
|
|
6
|
+
|
|
7
|
+
require_relative "run_loop"
|
|
8
|
+
require_relative "safari"
|
|
9
|
+
|
|
10
|
+
module Wrangle
|
|
11
|
+
# A Safari session that outlives a single command.
|
|
12
|
+
#
|
|
13
|
+
# An agent working through a shell gets one process per command, but a browser session is only
|
|
14
|
+
# useful if it persists between them. This holds one `Wrangle::Safari` open behind a Unix socket so
|
|
15
|
+
# `observe`, `act`, and `act` again are three commands against the same window.
|
|
16
|
+
#
|
|
17
|
+
# Everything it returns is shaped for a reader who has to decide what to do next: what changed, what
|
|
18
|
+
# is now available, and — when something failed — whether the failure is worth retrying.
|
|
19
|
+
class SessionServer
|
|
20
|
+
# Polling for a page to stop moving, which is the server's own business: the run loop asks it to
|
|
21
|
+
# settle and says for how long, but never how.
|
|
22
|
+
STEADY_POLL = 0.05
|
|
23
|
+
SETTLE_POLL = 0.3
|
|
24
|
+
STABLE_ROUNDS = 2
|
|
25
|
+
# A click that navigates changes nothing for the first few hundred milliseconds. Without a floor,
|
|
26
|
+
# two identical reads arrive before the browser has begun and the page is declared settled, which
|
|
27
|
+
# reports a working action as "nothing changed" — worse than saying nothing at all.
|
|
28
|
+
QUIET_FLOOR = 1.5
|
|
29
|
+
|
|
30
|
+
def self.socket_path(name)
|
|
31
|
+
File.join(ENV["WRANGLE_HOME"] || File.join(Dir.home, ".wrangle"), "#{name}.sock")
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.run(socket_path, options)
|
|
35
|
+
new(socket_path, options).run
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# `session` is injectable so the protocol, the diffing, and the refusal shapes can be tested
|
|
39
|
+
# against a fake bridge instead of a browser.
|
|
40
|
+
def initialize(socket_path, options, session: nil, jev: nil)
|
|
41
|
+
@socket_path = socket_path
|
|
42
|
+
@options = options
|
|
43
|
+
@session = session
|
|
44
|
+
@jev = jev
|
|
45
|
+
@page = nil
|
|
46
|
+
@acted = 0
|
|
47
|
+
@history = []
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def run
|
|
51
|
+
FileUtils.mkdir_p(File.dirname(@socket_path))
|
|
52
|
+
FileUtils.rm_f(@socket_path)
|
|
53
|
+
@session ||= start_session
|
|
54
|
+
serve(UNIXServer.new(@socket_path))
|
|
55
|
+
ensure
|
|
56
|
+
shutdown
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# What the run loop is allowed to ask for. It decides what should happen; everything that reads
|
|
60
|
+
# or changes the page lives here, so there is one file to read when asking what Wrangle can do to
|
|
61
|
+
# a window.
|
|
62
|
+
attr_reader :page
|
|
63
|
+
|
|
64
|
+
# Proposes, and does nothing else. Acting on the proposal is a separate, explicit request.
|
|
65
|
+
def decide(request)
|
|
66
|
+
page = current
|
|
67
|
+
chooser = decider(request)
|
|
68
|
+
started = now
|
|
69
|
+
questions = chooser.questions(page)
|
|
70
|
+
answer = asked(request, chooser.state(page, @history), questions)
|
|
71
|
+
choice = chooser.resolve(answer, page)
|
|
72
|
+
|
|
73
|
+
{ "operation" => choice.operation, "action" => choice.label, "kind" => choice.action&.fetch("kind"),
|
|
74
|
+
"confidence" => choice.confidence, "target_confidence" => choice.target_confidence,
|
|
75
|
+
"decided_in_ms" => ((now - started) * 1000).round, "executed" => false,
|
|
76
|
+
"model" => answer["model"], "input_tokens" => answer.dig("usage", "input_tokens"),
|
|
77
|
+
"choice" => choice }
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Asks Jev, and watches the page while it thinks.
|
|
81
|
+
#
|
|
82
|
+
# Proving a page has stopped moving takes two looks a couple of hundred milliseconds apart, and
|
|
83
|
+
# Jev takes about 380ms to answer. Those used to be paid one after the other, which is why the
|
|
84
|
+
# settle was worth skipping and why it was in fact skipped for months. Run together the looks are
|
|
85
|
+
# free: they fit inside a wait the step was making anyway.
|
|
86
|
+
#
|
|
87
|
+
# None of this changes what the decision is about. The answer belongs to `page`, the snapshot it
|
|
88
|
+
# was asked about, and the guard at act time still has the last word on whether it may be used.
|
|
89
|
+
# What the watching buys is a fresh observation sitting ready the moment the answer lands, so a
|
|
90
|
+
# decision the page outran costs one more request instead of a read on top of it.
|
|
91
|
+
def asked(request, state, questions)
|
|
92
|
+
@watched = nil
|
|
93
|
+
thinking = Thread.new { jev(request).ask(state: state, questions: questions) }
|
|
94
|
+
thinking.report_on_exception = false
|
|
95
|
+
watch(thinking)
|
|
96
|
+
thinking.value
|
|
97
|
+
ensure
|
|
98
|
+
thinking&.kill
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Stops at the first pair of reads that agree: once the page has held still there is nothing
|
|
102
|
+
# further to learn, and Apple Events are not free even when nobody is waiting on them.
|
|
103
|
+
def watch(thinking)
|
|
104
|
+
seen = @page && @page["fingerprint"]
|
|
105
|
+
while thinking.alive?
|
|
106
|
+
sleep(STEADY_POLL)
|
|
107
|
+
break unless thinking.alive?
|
|
108
|
+
|
|
109
|
+
looked = @session.observe
|
|
110
|
+
# Stamped with the number of actions taken. A read is only ever worth promoting while that
|
|
111
|
+
# number still holds: after a mutation it describes a page that no longer exists, and the one
|
|
112
|
+
# way this could report the wrong thing is by outliving the page it was taken from.
|
|
113
|
+
@watched = [@acted, looked]
|
|
114
|
+
break if looked["fingerprint"] == seen
|
|
115
|
+
|
|
116
|
+
seen = looked["fingerprint"]
|
|
117
|
+
end
|
|
118
|
+
rescue Error
|
|
119
|
+
# A read that fails while the answer is still coming is not this step's problem to solve. The
|
|
120
|
+
# guard, or the next read, will run into whatever is wrong and report it in its own terms.
|
|
121
|
+
@watched = nil
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The run loop decides; this stays the only thing that touches the page, so there is one place to
|
|
125
|
+
# read when asking what Wrangle is allowed to do to a window.
|
|
126
|
+
def perform(choice, text)
|
|
127
|
+
before = @page
|
|
128
|
+
began = now
|
|
129
|
+
@session.act(choice.action, @page, text: text)
|
|
130
|
+
acted = now
|
|
131
|
+
@acted += 1
|
|
132
|
+
@page = @session.observe
|
|
133
|
+
changed = before["fingerprint"] != @page["fingerprint"]
|
|
134
|
+
@history << { "action" => choice.label, "kind" => choice.action["kind"], "text" => text,
|
|
135
|
+
"page_changed" => changed }
|
|
136
|
+
{ "executed" => true, "text" => text, "page_changed" => changed,
|
|
137
|
+
"act_ms" => ((acted - began) * 1000).round, "read_ms" => ((now - acted) * 1000).round }
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Promotes the read taken while Jev was thinking, if nothing has been acted on since. A rejected
|
|
141
|
+
# decision touches nothing, so that read is the same page a fresh one would return, a poll
|
|
142
|
+
# sooner. A read from before an action is not, and is dropped rather than reused.
|
|
143
|
+
def observe!
|
|
144
|
+
acted, looked = @watched
|
|
145
|
+
@watched = nil
|
|
146
|
+
@page = (looked if looked && acted == @acted) || @session.observe
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# A decision about a page that is still rendering is a decision thrown away: Jev answers in
|
|
150
|
+
# ~350ms, and the freshness check then rejects it. Autocomplete menus and calendars settle in far
|
|
151
|
+
# less than that, so a couple of cheap observations cost much less than the wasted call they avoid.
|
|
152
|
+
# This is a budget, not a floor — a page that is already still returns immediately after one poll.
|
|
153
|
+
def steady(budget)
|
|
154
|
+
return @page unless budget.positive?
|
|
155
|
+
|
|
156
|
+
deadline = now + budget
|
|
157
|
+
loop do
|
|
158
|
+
before = @page["fingerprint"]
|
|
159
|
+
sleep(STEADY_POLL)
|
|
160
|
+
@page = @session.observe
|
|
161
|
+
return @page if @page["fingerprint"] == before || now >= deadline
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Three actions in a row that changed nothing is not progress, whatever the model believes — and
|
|
166
|
+
# neither is clicking the same control three times while the page shuffles underneath. Both are
|
|
167
|
+
# the same failure: the run is circling, and a human or the calling model should look at it.
|
|
168
|
+
def stalled?
|
|
169
|
+
recent = @history.last(3)
|
|
170
|
+
return false unless recent.length == 3
|
|
171
|
+
|
|
172
|
+
recent.none? { |entry| entry["page_changed"] || entry["kind"] == "wait" } ||
|
|
173
|
+
(recent.map { |entry| entry["action"] }.uniq.one? && recent.none? { |e| e["kind"] == "wait" })
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
private
|
|
177
|
+
|
|
178
|
+
def mcp? = @options["backend"].to_s == "mcp"
|
|
179
|
+
|
|
180
|
+
def start_session
|
|
181
|
+
if @options["window_id"]
|
|
182
|
+
raise ArgumentError, "The MCP backend cannot attach to an existing window" if mcp?
|
|
183
|
+
|
|
184
|
+
Safari.attach(window_id: @options["window_id"], display: @options["display"])
|
|
185
|
+
else
|
|
186
|
+
Safari.open(@options.fetch("url"),
|
|
187
|
+
display: @options["display"], bounds: @options["bounds"],
|
|
188
|
+
**(mcp? ? { bridge: McpBridge.new } : {}))
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def serve(server)
|
|
193
|
+
File.chmod(0o600, @socket_path)
|
|
194
|
+
loop do
|
|
195
|
+
client = server.accept
|
|
196
|
+
line = client.gets
|
|
197
|
+
next client.close unless line
|
|
198
|
+
|
|
199
|
+
request = parse(line)
|
|
200
|
+
client.puts(JSON.generate(dispatch(request)))
|
|
201
|
+
client.close
|
|
202
|
+
break if request["op"] == "close"
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def parse(line)
|
|
207
|
+
JSON.parse(line)
|
|
208
|
+
rescue JSON::ParserError
|
|
209
|
+
{ "op" => "bad" }
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def dispatch(request)
|
|
213
|
+
{ "ok" => true, "value" => handle(request) }
|
|
214
|
+
rescue Wrangle::Error => e
|
|
215
|
+
refusal(e.class.name.split("::").last, e.message, terminal: e.is_a?(ScopeLost) || e.is_a?(DeliveryUnknown))
|
|
216
|
+
rescue ArgumentError => e
|
|
217
|
+
refusal("ArgumentError", e.message, terminal: false)
|
|
218
|
+
rescue StandardError => e
|
|
219
|
+
# A bug is still a reply. The client is blocked on a socket read, so a server that dies here
|
|
220
|
+
# hangs the caller forever instead of telling it anything — the session is suspect afterwards,
|
|
221
|
+
# so this is terminal, but it is reported rather than silently fatal.
|
|
222
|
+
refusal(e.class.name, "internal error: #{e.message} (#{e.backtrace&.first})", terminal: true)
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def handle(request)
|
|
226
|
+
case request["op"]
|
|
227
|
+
when "status" then status
|
|
228
|
+
when "observe" then observe(request["settle"].to_f)
|
|
229
|
+
when "act" then act(request)
|
|
230
|
+
when "text" then { "text" => fresh_text(request) }
|
|
231
|
+
when "decide" then decide(request)
|
|
232
|
+
when "run" then run_goal(request)
|
|
233
|
+
when "close" then { "closing" => true }
|
|
234
|
+
else raise ArgumentError, "Unknown op #{request["op"].inspect}"
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# A refusal says what to do about itself. `retryable` is the difference between "look again" and
|
|
239
|
+
# "this session is over", which is the single most useful bit for whoever is deciding next.
|
|
240
|
+
def refusal(name, message, terminal:)
|
|
241
|
+
{
|
|
242
|
+
"ok" => false, "class" => name, "error" => message, "terminal" => terminal,
|
|
243
|
+
"retryable" => !terminal,
|
|
244
|
+
"hint" => if terminal
|
|
245
|
+
"This session cannot continue. Start a new one."
|
|
246
|
+
else
|
|
247
|
+
"Run `wrangle observe` and choose an action from the fresh list."
|
|
248
|
+
end
|
|
249
|
+
}
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def status
|
|
253
|
+
{
|
|
254
|
+
"pid" => Process.pid, "backend" => @options["backend"] || "jxa",
|
|
255
|
+
"window_id" => @session.window_id, "mode" => @session.mode,
|
|
256
|
+
"owned" => @session.owned?, "url" => @session.expected_url, "actions_taken" => @acted,
|
|
257
|
+
"observed" => !@page.nil?
|
|
258
|
+
}
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def current
|
|
262
|
+
@page || (@page = @session.observe)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# Reading text off a cached observation silently reports the page as it was before the last
|
|
266
|
+
# navigation finished. Text is always read fresh.
|
|
267
|
+
def fresh_text(request)
|
|
268
|
+
settle = request["settle"].to_f
|
|
269
|
+
@page = settle.positive? ? settled(settle, @page) : @session.observe
|
|
270
|
+
@page["text"]
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def observe(settle)
|
|
274
|
+
before = @page
|
|
275
|
+
@page = settle.positive? ? settled(settle, before) : @session.observe
|
|
276
|
+
describe(before, @page)
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# Poll until the page stops changing, rather than sleeping a guessed number of seconds. Stability
|
|
280
|
+
# only counts once the page has moved, or once the floor has passed with it sitting still.
|
|
281
|
+
# When the caller has told us what they are waiting for, wait for exactly that and stop. Waiting
|
|
282
|
+
# for a page to go quiet is a proxy; waiting for the text you need is the real thing, and a busy
|
|
283
|
+
# page like a results list may never go quiet at all.
|
|
284
|
+
def settled(timeout, baseline, expect = nil)
|
|
285
|
+
page = @session.observe
|
|
286
|
+
return page if arrived?(page, expect)
|
|
287
|
+
|
|
288
|
+
deadline = now + timeout
|
|
289
|
+
floor = now + [QUIET_FLOOR, timeout].min
|
|
290
|
+
stable = 0
|
|
291
|
+
moved = departed?(baseline, page)
|
|
292
|
+
until now >= deadline || (stable >= STABLE_ROUNDS && (moved || now >= floor))
|
|
293
|
+
sleep SETTLE_POLL
|
|
294
|
+
nxt = @session.observe
|
|
295
|
+
stable = nxt["fingerprint"] == page["fingerprint"] ? stable + 1 : 0
|
|
296
|
+
moved ||= departed?(baseline, nxt)
|
|
297
|
+
page = nxt
|
|
298
|
+
break if arrived?(page, expect)
|
|
299
|
+
end
|
|
300
|
+
page
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
def arrived?(page, expect)
|
|
304
|
+
!expect.nil? && page["text"].match?(expect)
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
def departed?(baseline, page)
|
|
308
|
+
!baseline.nil? && baseline["fingerprint"] != page["fingerprint"]
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
def act(request)
|
|
312
|
+
raise ArgumentError, "Observe before acting" unless @page
|
|
313
|
+
|
|
314
|
+
action = chosen(request["ref"])
|
|
315
|
+
before = @page
|
|
316
|
+
@session.act(action, @page, text: request["text"])
|
|
317
|
+
@acted += 1
|
|
318
|
+
@page = settled(request.fetch("settle", 3).to_f, before, request["expect"])
|
|
319
|
+
describe(before, @page).merge("executed" => label(action), "kind" => action["kind"])
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
def chosen(ref)
|
|
323
|
+
available = @page["actions"]
|
|
324
|
+
unless ref.is_a?(Integer) && ref.positive? && ref <= available.size
|
|
325
|
+
raise ArgumentError, "No action ##{ref.inspect} in the last observation; it offered #{available.size}"
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
available[ref - 1]
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
def describe(before, after)
|
|
332
|
+
{
|
|
333
|
+
"url" => after["url"], "title" => after["title"],
|
|
334
|
+
"fingerprint" => after["fingerprint"][0, 12],
|
|
335
|
+
"scroll" => after["scroll"], "text_chars" => after["text"].length,
|
|
336
|
+
"actions" => after["actions"].each_with_index.map { |action, index| compact(action, index) },
|
|
337
|
+
"changed" => change(before, after)
|
|
338
|
+
}
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
def compact(action, index)
|
|
342
|
+
{
|
|
343
|
+
"ref" => index + 1, "kind" => action["kind"], "role" => action["role"],
|
|
344
|
+
"label" => label(action), "value" => presence(action["value"])
|
|
345
|
+
}.compact
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
# The whole point of the interactive mode: after an action, say what actually moved. An agent that
|
|
349
|
+
# can see "nothing changed" can course-correct; one that only gets a page dump has to guess.
|
|
350
|
+
def change(before, after)
|
|
351
|
+
return nil unless before
|
|
352
|
+
|
|
353
|
+
old = before["actions"].map { |action| label(action) }
|
|
354
|
+
new = after["actions"].map { |action| label(action) }
|
|
355
|
+
{
|
|
356
|
+
"same_page" => before["fingerprint"] == after["fingerprint"],
|
|
357
|
+
"url_changed" => before["url"] != after["url"],
|
|
358
|
+
"text_delta" => after["text"].length - before["text"].length,
|
|
359
|
+
"scroll_delta" => after["scroll"]["y"] - before["scroll"]["y"],
|
|
360
|
+
"appeared" => (new - old).uniq.first(20),
|
|
361
|
+
"disappeared" => (old - new).uniq.first(20)
|
|
362
|
+
}
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
def label(action) = action["label"].to_s
|
|
366
|
+
|
|
367
|
+
# --- deciding ----------------------------------------------------------------------------
|
|
368
|
+
|
|
369
|
+
def decider(request)
|
|
370
|
+
Decider.new(goal: request.fetch("goal"))
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
def jev(request)
|
|
374
|
+
@jev ||= Jev.from_env(endpoint: request["endpoint"], model: request["model"])
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
# No fixed pause anywhere. Waiting is an operation the model can choose when a control is missing
|
|
378
|
+
# or results are still loading, so a page that updates instantly costs nothing.
|
|
379
|
+
def run_goal(request)
|
|
380
|
+
raise ArgumentError, "run needs execute: true to touch the page" unless request["execute"]
|
|
381
|
+
|
|
382
|
+
expect = request["expect"] && Regexp.new(request["expect"], Regexp::IGNORECASE)
|
|
383
|
+
plan = Array(request["plan"]).filter_map { |goal| presence(goal) }
|
|
384
|
+
plan = [request["goal"]] if plan.empty?
|
|
385
|
+
summarise(request, RunLoop.new(self, request, expect).run(plan), expect)
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def summarise(request, steps, expect)
|
|
389
|
+
if steps.last&.fetch("operation", nil) == "RESTALE"
|
|
390
|
+
steps << { "operation" => "HANDOFF", "confidence" => 0.0,
|
|
391
|
+
"action" => "The page kept moving faster than a decision could be made; look at " \
|
|
392
|
+
"it yourself and act by ref" }
|
|
393
|
+
end
|
|
394
|
+
proven = expect ? fresh_text(request).match?(expect) : nil
|
|
395
|
+
{ "goal" => request["goal"], "legs" => Array(request["plan"]).length,
|
|
396
|
+
"steps" => steps, "stopped" => steps.last&.fetch("operation", nil),
|
|
397
|
+
"expected" => expect&.source, "proven" => proven }
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
def presence(value) = value.nil? || value.to_s.empty? ? nil : value.to_s
|
|
401
|
+
|
|
402
|
+
def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
403
|
+
|
|
404
|
+
def shutdown
|
|
405
|
+
begin
|
|
406
|
+
@session&.close
|
|
407
|
+
rescue Wrangle::Error
|
|
408
|
+
nil # A poisoned session refuses to close its window. That refusal is the correct outcome.
|
|
409
|
+
end
|
|
410
|
+
FileUtils.rm_f(@socket_path) if @socket_path
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
# Talks to a SessionServer over its socket. One request, one reply, one connection.
|
|
415
|
+
class SessionClient
|
|
416
|
+
def initialize(socket_path)
|
|
417
|
+
@socket_path = socket_path
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
def running?
|
|
421
|
+
File.socket?(@socket_path) && begin
|
|
422
|
+
call("status")
|
|
423
|
+
true
|
|
424
|
+
rescue Wrangle::Error
|
|
425
|
+
false
|
|
426
|
+
end
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
def call(op, **params)
|
|
430
|
+
socket = UNIXSocket.new(@socket_path)
|
|
431
|
+
socket.puts(JSON.generate(params.merge(op: op)))
|
|
432
|
+
reply = socket.gets
|
|
433
|
+
raise BridgeError, "The wrangle session closed without replying" unless reply
|
|
434
|
+
|
|
435
|
+
JSON.parse(reply)
|
|
436
|
+
rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ENOTCONN
|
|
437
|
+
# The session was there when the connection opened and gone before it answered. Same outcome as
|
|
438
|
+
# a reply that never came, and a caller should not have to know the difference at errno level —
|
|
439
|
+
# macOS reports this as any of three errnos depending on how far the write got.
|
|
440
|
+
raise BridgeError, "The wrangle session closed without replying"
|
|
441
|
+
rescue Errno::ENOENT, Errno::ECONNREFUSED
|
|
442
|
+
raise BridgeError, "No wrangle session at #{@socket_path}. Start one with `wrangle open <url>`."
|
|
443
|
+
ensure
|
|
444
|
+
socket&.close
|
|
445
|
+
end
|
|
446
|
+
end
|
|
447
|
+
end
|
data/lib/wrangle.rb
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "wrangle/version"
|
|
4
|
+
require_relative "wrangle/errors"
|
|
5
|
+
require_relative "wrangle/observation"
|
|
6
|
+
require_relative "wrangle/jxa_bridge"
|
|
7
|
+
require_relative "wrangle/mcp_bridge"
|
|
8
|
+
require_relative "wrangle/safari"
|
|
9
|
+
require_relative "wrangle/jev"
|
|
10
|
+
require_relative "wrangle/action_space"
|
|
11
|
+
require_relative "wrangle/decider"
|
|
12
|
+
require_relative "wrangle/session_server"
|
|
13
|
+
|
|
14
|
+
# Hand one Safari window to a program, and no more than that.
|
|
15
|
+
#
|
|
16
|
+
# Wrangle drives an ordinary Safari window through Apple Events. There is no automation session and
|
|
17
|
+
# no extension, so the window stays a real one the user can see, keep, and take back at any moment.
|
|
18
|
+
module Wrangle
|
|
19
|
+
end
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: wrangle
|
|
3
|
+
description: Drive a real Safari window from the shell - open a page, see what it offers, click, type, and read results back. Use for looking things up on sites that need a real logged-in browser (flights, prices, dashboards, portals), filling a web form, or when the user says "use my browser", "check this site", or "/wrangle". Not for fetching static pages; use a normal HTTP fetch for those.
|
|
4
|
+
compatibility: macOS with Safari, and `wrangle` on PATH. Requires Safari > Settings > Advanced > "Allow JavaScript from Apple Events", plus a one-time Apple Events permission prompt.
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Wrangle
|
|
8
|
+
|
|
9
|
+
Drive one real Safari window through a persistent session. Every command below is a shell command.
|
|
10
|
+
|
|
11
|
+
**Never write a Ruby script for this.** The CLI is the entire interface. If you catch yourself
|
|
12
|
+
writing `require "wrangle"`, stop — use `wrangle observe` and `wrangle act` instead.
|
|
13
|
+
|
|
14
|
+
## The loop
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
wrangle open "https://example.com" --display 1 --settle 8 # start; prints the first observation
|
|
18
|
+
wrangle observe --match 'passenger|search' # look, filtered
|
|
19
|
+
wrangle act 13 # do one thing, see what changed
|
|
20
|
+
wrangle text --match 'total|price' # read page content
|
|
21
|
+
wrangle close # always finish here
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
The session lives in a background process, so these are separate commands against the same window.
|
|
25
|
+
State persists between them. Use `--session NAME` to run more than one at a time.
|
|
26
|
+
|
|
27
|
+
## Reading the output
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
acted click "Add adult"
|
|
31
|
+
page "Oklahoma City to Denver | Google Flights" <https://...>
|
|
32
|
+
state 9f8aab03117c scroll 0/1623 1193 chars 72 actions
|
|
33
|
+
changed url changed, text +1114
|
|
34
|
+
appeared "Remove adult", "2 passengers"
|
|
35
|
+
gone "Add adult", "Done", "Cancel"
|
|
36
|
+
13 click button 2 passengers
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
- **`appeared` / `gone` is your feedback signal.** `"Remove adult"` appearing proves the count went
|
|
40
|
+
1 → 2. Check it after every action instead of re-reading the whole page.
|
|
41
|
+
- **`changed nothing — the page is byte-identical`** means your click did nothing. Do not repeat it.
|
|
42
|
+
Observe, and pick a different action.
|
|
43
|
+
- **The numbers on the left are refs, and they shift after every action.** Only ever use a ref from
|
|
44
|
+
the most recent output. A stale ref is refused, not silently mis-clicked.
|
|
45
|
+
|
|
46
|
+
## Finding the right action
|
|
47
|
+
|
|
48
|
+
`observe` truncates to 25 actions. Do not dump everything; filter:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
wrangle observe --match 'passenger|adult|done' # regex, case-insensitive
|
|
52
|
+
wrangle observe --all # everything, when you must
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
To read page content rather than controls:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
wrangle text # whole page text
|
|
59
|
+
wrangle text --match '\$[0-9]' # only matching lines
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Typing and waiting
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
wrangle act 7 --text "Lisbon" # a fill action needs --text
|
|
66
|
+
wrangle act 4 --settle 10 # wait longer for a slow update
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
`--settle` polls until the page stops changing, up to N seconds. It does not sleep blindly. Raise it
|
|
70
|
+
for searches and slow SPAs. If content looks half-loaded — placeholders, "Fetching results", counts
|
|
71
|
+
that disagree with each other — that is a settle problem: run `wrangle observe --settle 10` again
|
|
72
|
+
before concluding anything.
|
|
73
|
+
|
|
74
|
+
## Let Jev drive a whole sub-task
|
|
75
|
+
|
|
76
|
+
`wrangle run` decides and acts in a loop, using a small typed-choice model (Jev) that picks one of the
|
|
77
|
+
actions Wrangle observed. It is several times faster than stepping by hand, because it never waits for
|
|
78
|
+
you between clicks.
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
wrangle run --goal "Search flights: from OKC to DEN, depart 2026-10-12, return 2026-10-15." \
|
|
82
|
+
--literal 'where from=OKC' --literal 'where to=DEN' --steps 10 --execute
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
- `--execute` is required. Without it the loop only proposes.
|
|
86
|
+
- `--literal LABEL=VALUE` supplies text for a field whose label contains `LABEL`. **The model never
|
|
87
|
+
invents text.** If a fill has no matching literal, the run stops and asks you for it.
|
|
88
|
+
- `--steps N` caps the loop. `--min-confidence F` moves the bar (default 0.5).
|
|
89
|
+
|
|
90
|
+
### When it hands back to you
|
|
91
|
+
|
|
92
|
+
The run stops and prints `ask` when Jev is not confident enough, when a fill needs text, when it
|
|
93
|
+
notices itself circling, or when a leg kept claiming to be finished while the page disagreed.
|
|
94
|
+
**You are the fallback.** Do not just re-run the same goal — look, then either:
|
|
95
|
+
|
|
96
|
+
1. Re-run with a **narrower goal** naming the next concrete step ("The calendar is open; click
|
|
97
|
+
Thursday, October 15, 2026, then click Done"). This is usually right.
|
|
98
|
+
2. Verify the state yourself with `wrangle observe` / `wrangle text`, then `wrangle act REF` directly.
|
|
99
|
+
Do this when Jev names the right control but is unsure whether to act at all.
|
|
100
|
+
|
|
101
|
+
Narrow goals work far better than one big goal. Drive a form in legs: airports, then dates, then
|
|
102
|
+
passengers, then submit.
|
|
103
|
+
|
|
104
|
+
A leg only finishes when the page shows it finished. Alongside every decision, Jev is asked
|
|
105
|
+
separately whether the goal's outcome is actually visible, and a confident disagreement keeps the
|
|
106
|
+
leg working — you will see `Said done, but the page does not show ...` in the transcript. So write
|
|
107
|
+
each leg as an **outcome you could see**, not an action you could take: "the results are sorted by
|
|
108
|
+
price, low to high" checks better than "use the sort dropdown". A leg that cannot be seen on the
|
|
109
|
+
page cannot be confirmed, and will be handed back to you.
|
|
110
|
+
|
|
111
|
+
## Prefer a deep link over filling a form
|
|
112
|
+
|
|
113
|
+
If the site accepts search parameters in the URL, open that directly and skip the form entirely. One
|
|
114
|
+
`open` beats six `act`s and cannot mis-click. **Unless the user asked you to use the site's own UI** —
|
|
115
|
+
then fill the form with `wrangle run` and do not shortcut it. Example:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
wrangle open "https://www.google.com/travel/flights?q=Flights%20from%20OKC%20to%20DEN%20on%202026-10-12%20through%202026-10-15" --settle 8
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Then use `act` only for what the URL could not express.
|
|
122
|
+
|
|
123
|
+
## Windows
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
wrangle windows --titles # what Safari has open
|
|
127
|
+
wrangle displays # screen geometry for --display
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
- `wrangle open URL` creates a window Wrangle owns and will close.
|
|
131
|
+
- `wrangle attach WINDOW_ID` takes over a window the user already has open. **Ask first.** Wrangle
|
|
132
|
+
never closes or navigates an attached window, but it will scroll and click in it.
|
|
133
|
+
- Use `--display 1` (a second monitor) when available so you are not covering the user's work.
|
|
134
|
+
|
|
135
|
+
## When something fails
|
|
136
|
+
|
|
137
|
+
Exit codes: `0` ok, `2` usage, `3` stale — observe and retry, `4` the session is over, `5` no session.
|
|
138
|
+
|
|
139
|
+
| Message | What to do |
|
|
140
|
+
|---|---|
|
|
141
|
+
| `StalePage: Page changed since this decision` | `wrangle observe`, then act on a fresh ref. Normal. |
|
|
142
|
+
| `ScopeLost: ...` | Terminal. The user took their window back, or it gained a tab. Do not reopen without asking. |
|
|
143
|
+
| `DeliveryUnknown: ...` | Terminal. An action may or may not have landed. **Never repeat it.** Tell the user what is uncertain. |
|
|
144
|
+
| `No action #N in the last observation` | You used a stale ref. Observe again. |
|
|
145
|
+
| `N Safari processes are running` | Orphaned `safaridriver` instances. Ask the user before killing anything. |
|
|
146
|
+
|
|
147
|
+
## Rules
|
|
148
|
+
|
|
149
|
+
- One action per command when stepping by hand. Observe between actions.
|
|
150
|
+
- Prefer `wrangle run` for a multi-step sub-task; step by hand when it hands back to you.
|
|
151
|
+
- Never repeat a mutation to find out whether it worked — read the page instead.
|
|
152
|
+
- Never enter passwords, card numbers, or 2FA codes. Stop and hand back to the user.
|
|
153
|
+
- Do not buy, book, send, post, or delete anything without explicit confirmation of that exact step.
|
|
154
|
+
- `wrangle close` when you are done, even if the task failed.
|
|
155
|
+
- Report what you actually observed. If a value looks inconsistent, say so rather than smoothing it.
|
|
156
|
+
|
|
157
|
+
## Worked example
|
|
158
|
+
|
|
159
|
+
Flight prices, OKC → DEN, Oct 12–15, 2 adults, filling the site's own form. **One command drives the
|
|
160
|
+
whole form.** Hand-stepping this same task took ten separate commands and over two minutes, almost
|
|
161
|
+
all of it your own turn latency.
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
wrangle open "https://www.google.com/travel/flights" --display 1 --side left --settle 4
|
|
165
|
+
|
|
166
|
+
wrangle run --execute --steps 40 --min-confidence 0.4 \
|
|
167
|
+
--literal 'where from=OKC' --literal 'where to=DEN' \
|
|
168
|
+
--plan "Set the origin to OKC and the destination to DEN, choosing the matching airport from each autocomplete list." \
|
|
169
|
+
--plan "Open the Departure field to show the calendar, then click the day Monday, October 12, 2026." \
|
|
170
|
+
--plan "Click the day Thursday, October 15, 2026 for the return, then click Done to confirm the dates." \
|
|
171
|
+
--plan "Open the passenger selector, add a second adult, then click Done to confirm." \
|
|
172
|
+
--plan "Click the Search button to run the flight search." \
|
|
173
|
+
--expect 'taxes \+ fees for 2 adults'
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
```
|
|
177
|
+
== 4. Open the passenger selector, add a second adult, then click Done to confirm.
|
|
178
|
+
26. did 1 passenger (91% sure/100% target, jev 272ms, step 626ms)
|
|
179
|
+
27. did Add adult (95% sure/95% target, jev 318ms, step 458ms)
|
|
180
|
+
28. done DONE (53% sure, jev 342ms)
|
|
181
|
+
proven the page shows "taxes \+ fees for 2 adults"
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Then read the result and finish:
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
wrangle text --match 'Nonstop|^\$|2 adults'
|
|
188
|
+
wrangle close
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
If a leg hands back, do not re-run the same plan. Look at the page, then re-run **only the remaining
|
|
192
|
+
legs**, with the first one describing what you actually see:
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
wrangle run --execute --min-confidence 0.4 \
|
|
196
|
+
--plan "A passenger dialog is open. Click its Done button to close it." \
|
|
197
|
+
--plan "Click the Search button to run the flight search."
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## Rules of thumb for plans
|
|
201
|
+
|
|
202
|
+
- One concrete step per leg. "Click the day Monday, October 12, 2026" beats "pick the dates".
|
|
203
|
+
- Name the control the way the page labels it. The model is choosing from observed labels, not guessing.
|
|
204
|
+
- Legs are ordered and assumed: a leg that fails ends the plan, because the rest depend on it.
|
|
205
|
+
- `--expect RE` proves the outcome from page text. Use it — a plan that ran is not a plan that worked.
|
|
206
|
+
- Lower `--min-confidence` to ~0.4 when you have written explicit legs; keep the default otherwise.
|