dsh-mobilecode 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -1
- package/lib/android-stream.js +3 -4
- package/lib/client.js +44 -6
- package/lib/device-build.js +111 -18
- package/lib/index.js +112 -22
- package/lib/uitree.js +4 -4
- package/lib/vision.js +130 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -87,6 +87,11 @@ drawer:
|
|
|
87
87
|
- `device_apps` / `device_launch_app` — list installed packages (third-party by
|
|
88
88
|
default) so a package name is never guessed / launch one by package or a
|
|
89
89
|
unique substring, with `relaunch` for a cold start.
|
|
90
|
+
- `device_intent` — open anything by Android intent: an `action` (e.g.
|
|
91
|
+
`android.settings.WIFI_SETTINGS`), a deep-link `uri` (`geo:`, `https://`,
|
|
92
|
+
`file://`), or an explicit `component` (`pkg/.Activity`). Reaches screens no
|
|
93
|
+
tap can address. Values are single-quoted for the device shell (apostrophes
|
|
94
|
+
escaped, control characters refused), so metacharacters stay inert.
|
|
90
95
|
- `device_stream` — drive the live screen stream the panel shows: `start` an
|
|
91
96
|
online device (returns a signed `streamUrl`), `status`, or `stop`. Agents that
|
|
92
97
|
just need to see the screen should prefer `device_screen` / `device_ui_tree`.
|
|
@@ -115,6 +120,17 @@ The Devices pane shows a real-time mirror of the attached device. It is produced
|
|
|
115
120
|
key (`~/.dsh/mobilecode/stream-access.key`, `0600`), expiring within 10 minutes
|
|
116
121
|
and re-minted automatically. Coordinates are normalized 0..1 of the streamed
|
|
117
122
|
frame, so one mapping serves every rotation.
|
|
123
|
+
|
|
124
|
+
**Multimodal screenshots**
|
|
125
|
+
|
|
126
|
+
When the routed model declares image input, `device_screen` delivers the
|
|
127
|
+
screenshot **as an image block** — the model literally sees the screen instead of
|
|
128
|
+
reading a file path. This mirrors the in-tree `read_image` tool: the PNG is
|
|
129
|
+
committed to DSH's durable attachment store (`ctx.get('attachments').saveImage`)
|
|
130
|
+
and returned as a `{type:'image', attachment}` content block, gated on
|
|
131
|
+
`llm.resolveModelInfo(...).inputModalities`. It **degrades, never refuses**: a
|
|
132
|
+
text-only route, a headless profile, or a host without the attachment store keeps
|
|
133
|
+
the plain JSON summary (path + UI tree + OCR) with no new error.
|
|
118
134
|
- `device_log` — device logs: logcat `main`/`crash`/`events`/`kernel` buffers
|
|
119
135
|
(kernel = dmesg, needs adb root — works on emulators) with an optional
|
|
120
136
|
case-insensitive substring filter, capped line count.
|
|
@@ -137,6 +153,9 @@ The Devices pane shows a real-time mirror of the attached device. It is produced
|
|
|
137
153
|
Fix button for auto-fixable checks (PaddleOCR install).
|
|
138
154
|
- **PaddleOCR** — install status, progress log, and the install button.
|
|
139
155
|
- **AI Prompt** — the copyable agent prompt.
|
|
156
|
+
- **Connection** — the resolved adb binary path and every device `adb` sees
|
|
157
|
+
(serial, state dot, model, USB / Wi-Fi badge), straight from
|
|
158
|
+
`GET /connection`. Explains the Wi-Fi reconnect policy.
|
|
140
159
|
- The same settings appear as a **`MobileCode` page in the DSH Settings**
|
|
141
160
|
(registered as a `settings.section` slot, like the other installed plugins),
|
|
142
161
|
so they are reachable from Settings even when the Devices pane is closed.
|
|
@@ -148,7 +167,22 @@ The Devices pane shows a real-time mirror of the attached device. It is produced
|
|
|
148
167
|
`directory` + `platform` body, mirroring mobilecode's `server.devicePreview`
|
|
149
168
|
group — plus setup endpoints: `GET /welcome`, `POST /welcome/dismiss`,
|
|
150
169
|
`GET /doctor`, `POST /doctor/fix {id}`, `GET /ocr`, `POST /ocr/install`,
|
|
151
|
-
`GET/POST /settings`.
|
|
170
|
+
`GET/POST /settings`, `GET /connection`.
|
|
171
|
+
|
|
172
|
+
**One classified adb boundary** — every serial-targeted adb command runs through
|
|
173
|
+
`adbRun()` in `lib/device-build.js`:
|
|
174
|
+
|
|
175
|
+
- A transport failure (`device not found` / `offline` / `unauthorized` /
|
|
176
|
+
`closed` / `no devices`) on a **Wi-Fi serial** (`ip:port`) gets exactly ONE
|
|
177
|
+
`adb connect` retry. Read-only commands (the `replaySafeAdb` allowlist:
|
|
178
|
+
`exec-out`/`screencap`/`uiautomator`/`dumpsys`/`getprop`/`logcat`/`cat`/…)
|
|
179
|
+
then replay automatically; **side-effectful ones never do** — a replayed tap
|
|
180
|
+
could double-tap — they raise "reconnected — call again" instead.
|
|
181
|
+
- USB serials and non-transport failures (a real `am start` error, a
|
|
182
|
+
`SecurityException`) surface as classified, actionable errors — the old
|
|
183
|
+
silent `capture()` → `""` swallowing is gone on agent-facing paths.
|
|
184
|
+
- Tolerant internal callers (boot polling, IME checks) opt back into
|
|
185
|
+
best-effort with an explicit `.catch(() => "")`.
|
|
152
186
|
|
|
153
187
|
## How it works
|
|
154
188
|
|
package/lib/android-stream.js
CHANGED
|
@@ -247,7 +247,7 @@ export class AndroidStreamHost {
|
|
|
247
247
|
|
|
248
248
|
async getRotation(serial) {
|
|
249
249
|
try {
|
|
250
|
-
const value = Number((await DeviceBuild.
|
|
250
|
+
const value = Number((await DeviceBuild.adbRun(serial, ['shell', 'settings', 'get', 'system', 'user_rotation'])).trim())
|
|
251
251
|
return ROTATION_CYCLE.includes(value) ? value : 0
|
|
252
252
|
} catch {
|
|
253
253
|
return 0
|
|
@@ -263,8 +263,7 @@ export class AndroidStreamHost {
|
|
|
263
263
|
}
|
|
264
264
|
|
|
265
265
|
async #shell(serial, shell) {
|
|
266
|
-
|
|
267
|
-
if (code !== 0) throw new Error(`adb shell ${shell.join(' ')} failed (exit ${code})`)
|
|
266
|
+
await DeviceBuild.adbRun(serial, ['shell', ...shell], { timeoutMs: CONTROL_TIMEOUT_MS })
|
|
268
267
|
}
|
|
269
268
|
|
|
270
269
|
async #keepAliveTick() {
|
|
@@ -371,7 +370,7 @@ export class AndroidStreamHost {
|
|
|
371
370
|
|
|
372
371
|
/** Override-aware physical screen size for the non-streaming control fallback. */
|
|
373
372
|
async function screenSize(serial) {
|
|
374
|
-
const output = await DeviceBuild.
|
|
373
|
+
const output = await DeviceBuild.adbRun(serial, ['shell', 'wm', 'size'])
|
|
375
374
|
const match = /Override size:\s*(\d+)x(\d+)/.exec(output) ?? /Physical size:\s*(\d+)x(\d+)/.exec(output)
|
|
376
375
|
if (!match) throw new Error(`cannot read the screen size of ${serial}`)
|
|
377
376
|
return { width: Number(match[1]), height: Number(match[2]) }
|
package/lib/client.js
CHANGED
|
@@ -121,6 +121,13 @@ window.__ModuleLoader__.load({
|
|
|
121
121
|
.mc-live-nav button { border: 1px solid light-dark(rgba(0,0,0,.16), rgba(255,255,255,.18)); background: light-dark(#fff, #20242b); color: inherit; border-radius: 999px; width: 40px; height: 32px; cursor: pointer; font-size: 14px; }
|
|
122
122
|
.mc-live-nav button:hover { background: light-dark(rgba(0,0,0,.06), rgba(255,255,255,.1)); }
|
|
123
123
|
.mc-live-cap { font-size: 11px; color: light-dark(#6b7078, #9aa0a8); text-align: center; }
|
|
124
|
+
.mc-device-list { display: flex; flex-direction: column; gap: 4px; margin: 8px 0; }
|
|
125
|
+
.mc-device-row { display: flex; align-items: center; gap: 8px; padding: 6px 8px; border: 1px solid light-dark(rgba(0,0,0,.12), rgba(255,255,255,.14)); border-radius: 8px; }
|
|
126
|
+
.mc-dot { width: 8px; height: 8px; border-radius: 50%; flex: none; display: inline-block; background: #9e9e9e; }
|
|
127
|
+
.mc-dot[data-on="true"] { background: #4caf50; box-shadow: 0 0 6px rgba(76,175,80,.6); }
|
|
128
|
+
.mc-serial { font-weight: 600; font-family: ui-monospace, Consolas, monospace; font-size: 12px; }
|
|
129
|
+
.mc-badge { font-size: 11px; padding: 1px 7px; border-radius: 999px; background: light-dark(rgba(0,0,0,.07), rgba(255,255,255,.12)); }
|
|
130
|
+
.mc-warn { color: #b3261e; font-size: 12px; margin: 6px 0; }
|
|
124
131
|
`;
|
|
125
132
|
//#endregion
|
|
126
133
|
|
|
@@ -140,13 +147,13 @@ window.__ModuleLoader__.load({
|
|
|
140
147
|
}
|
|
141
148
|
|
|
142
149
|
const getInfo = async (directory) => {
|
|
143
|
-
const url = API_BASE + (directory ? "?directory=" + encodeURIComponent(directory) : "");
|
|
150
|
+
const url = location.origin + API_BASE + (directory ? "?directory=" + encodeURIComponent(directory) : "");
|
|
144
151
|
const response = await fetch(url);
|
|
145
152
|
if (!response.ok) throw new Error("HTTP " + response.status);
|
|
146
153
|
return response.json();
|
|
147
154
|
};
|
|
148
155
|
const post = async (path, body) => {
|
|
149
|
-
const response = await fetch(API_BASE + path, {
|
|
156
|
+
const response = await fetch(location.origin + API_BASE + path, {
|
|
150
157
|
method: "POST",
|
|
151
158
|
headers: { "content-type": "application/json" },
|
|
152
159
|
body: JSON.stringify(body ?? {}),
|
|
@@ -155,14 +162,14 @@ window.__ModuleLoader__.load({
|
|
|
155
162
|
return response.json();
|
|
156
163
|
};
|
|
157
164
|
const apiGet = async (path) => {
|
|
158
|
-
const response = await fetch(API_BASE + path);
|
|
165
|
+
const response = await fetch(location.origin + API_BASE + path);
|
|
159
166
|
if (!response.ok) throw new Error("HTTP " + response.status);
|
|
160
167
|
return response.json();
|
|
161
168
|
};
|
|
162
169
|
// POST that surfaces the server's JSON error copy (stream routes return
|
|
163
170
|
// human-readable 4xx bodies the generic post() would flatten to "HTTP 409").
|
|
164
171
|
const streamPost = async (path, body) => {
|
|
165
|
-
const response = await fetch(API_BASE + path, {
|
|
172
|
+
const response = await fetch(location.origin + API_BASE + path, {
|
|
166
173
|
method: "POST",
|
|
167
174
|
headers: { "content-type": "application/json" },
|
|
168
175
|
body: JSON.stringify(body ?? {}),
|
|
@@ -374,7 +381,9 @@ window.__ModuleLoader__.load({
|
|
|
374
381
|
if (!device) return;
|
|
375
382
|
try {
|
|
376
383
|
const r = await streamPost("/stream/grant", { device });
|
|
377
|
-
|
|
384
|
+
// Absolute URL: the plugin card can be hosted under a different
|
|
385
|
+
// origin/path than the API, and <img> src ignores fetch()'s base.
|
|
386
|
+
setStreamUrl(location.origin + r.streamUrl);
|
|
378
387
|
setSerial(r.device);
|
|
379
388
|
setError("");
|
|
380
389
|
clearTimeout(grantTimer.current);
|
|
@@ -575,14 +584,21 @@ window.__ModuleLoader__.load({
|
|
|
575
584
|
}
|
|
576
585
|
|
|
577
586
|
/** Settings dialog: tabs for Doctor / PaddleOCR / AI prompt. */
|
|
578
|
-
/** Shared settings tabs (Doctor / PaddleOCR / AI Prompt) — used by both the ⚙ dialog and the DSH Settings page. */
|
|
587
|
+
/** Shared settings tabs (Doctor / PaddleOCR / AI Prompt / Connection) — used by both the ⚙ dialog and the DSH Settings page. */
|
|
579
588
|
function SettingsTabs() {
|
|
580
589
|
const [tab, setTab] = useState("doctor");
|
|
581
590
|
const [prompt, setPrompt] = useState("");
|
|
582
591
|
const [copied, setCopied] = useState(false);
|
|
592
|
+
const [conn, setConn] = useState(null);
|
|
593
|
+
const [connError, setConnError] = useState("");
|
|
583
594
|
useEffect(() => {
|
|
584
595
|
apiGet("/welcome").then((w) => setPrompt(w.prompt ?? "")).catch(() => {});
|
|
585
596
|
}, []);
|
|
597
|
+
useEffect(() => {
|
|
598
|
+
if (tab !== "connection") return;
|
|
599
|
+
setConn(null); setConnError("");
|
|
600
|
+
apiGet("/connection").then(setConn).catch((e) => setConnError(e instanceof Error ? e.message : String(e)));
|
|
601
|
+
}, [tab]);
|
|
586
602
|
|
|
587
603
|
const copy = async () => {
|
|
588
604
|
const done = await copyText(prompt);
|
|
@@ -598,6 +614,7 @@ window.__ModuleLoader__.load({
|
|
|
598
614
|
tabButton("doctor", "Doctor"),
|
|
599
615
|
tabButton("ocr", "PaddleOCR"),
|
|
600
616
|
tabButton("prompt", "AI Prompt"),
|
|
617
|
+
tabButton("connection", "Connection"),
|
|
601
618
|
),
|
|
602
619
|
tab === "doctor" && h("div", { className: "mc-modal-body" }, h(DoctorView, { onFixed: () => {} })),
|
|
603
620
|
tab === "ocr" && h("div", { className: "mc-modal-body" }, h(OcrCard, {})),
|
|
@@ -611,6 +628,27 @@ window.__ModuleLoader__.load({
|
|
|
611
628
|
h("button", { className: "mc-btn mc-copy", onClick: copy }, copied ? "Copied ✓" : "Copy"),
|
|
612
629
|
),
|
|
613
630
|
),
|
|
631
|
+
tab === "connection" && h("div", { className: "mc-modal-body" },
|
|
632
|
+
h("div", null,
|
|
633
|
+
h("h3", null, "adb connection"),
|
|
634
|
+
connError !== "" && h("div", { className: "mc-warn" }, connError),
|
|
635
|
+
!conn && connError === "" && h("div", { className: "mc-hint" }, "Loading…"),
|
|
636
|
+
conn && h("div", null,
|
|
637
|
+
h("div", { className: "mc-hint" }, `adb: ${conn.adb}`),
|
|
638
|
+
h("div", { className: "mc-device-list" },
|
|
639
|
+
conn.devices.length === 0 && h("div", { className: "mc-hint" }, "No devices attached. Plug a phone (USB debug) or boot an emulator."),
|
|
640
|
+
conn.devices.map((d) => h("div", { className: "mc-device-row", key: d.serial },
|
|
641
|
+
h("span", { className: "mc-dot", "data-on": d.state === "device" ? "true" : undefined }),
|
|
642
|
+
h("span", { className: "mc-serial" }, d.serial),
|
|
643
|
+
d.model && h("span", { className: "mc-hint" }, d.model),
|
|
644
|
+
h("span", { className: "mc-badge" }, d.wifi ? "Wi-Fi" : "USB"),
|
|
645
|
+
h("span", { className: "mc-hint" }, d.state),
|
|
646
|
+
)),
|
|
647
|
+
),
|
|
648
|
+
h("div", { className: "mc-hint" }, "Wi-Fi serials look like 192.168.1.23:5555 — attach with `adb connect <ip>:5555`. A dropped Wi-Fi link is reconnected once automatically; read-only probes then replay, taps never do."),
|
|
649
|
+
),
|
|
650
|
+
),
|
|
651
|
+
),
|
|
614
652
|
];
|
|
615
653
|
}
|
|
616
654
|
|
package/lib/device-build.js
CHANGED
|
@@ -97,17 +97,20 @@ export function exec(command, args, options, onLine) {
|
|
|
97
97
|
return { child, exit }
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
-
/** Run a command
|
|
101
|
-
export async function
|
|
100
|
+
/** Run a command, capturing stdout AND stderr. Resolves {code, out, err}. */
|
|
101
|
+
export async function captureFull(command, args, options = {}) {
|
|
102
102
|
const out = []
|
|
103
|
+
const err = []
|
|
103
104
|
const running = launch(command, args, {
|
|
104
105
|
cwd: options.cwd,
|
|
105
106
|
env: options.env ? { ...process.env, ...options.env } : undefined,
|
|
106
|
-
stdio: ["ignore", "pipe", "
|
|
107
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
107
108
|
windowsHide: true,
|
|
108
109
|
})
|
|
109
110
|
running.stdout?.setEncoding("utf8")
|
|
111
|
+
running.stderr?.setEncoding("utf8")
|
|
110
112
|
running.stdout?.on("data", (chunk) => out.push(chunk))
|
|
113
|
+
running.stderr?.on("data", (chunk) => err.push(chunk))
|
|
111
114
|
let timer
|
|
112
115
|
if (options.timeoutMs) {
|
|
113
116
|
timer = setTimeout(() => running.kill(), options.timeoutMs)
|
|
@@ -118,9 +121,18 @@ export async function capture(command, args, options = {}) {
|
|
|
118
121
|
running.once("close", (value) => resolve(value ?? -1))
|
|
119
122
|
})
|
|
120
123
|
if (timer) clearTimeout(timer)
|
|
121
|
-
if (code !== 0) return ""
|
|
122
124
|
const text = out.join("")
|
|
123
|
-
return
|
|
125
|
+
return {
|
|
126
|
+
code,
|
|
127
|
+
out: options.maxBytes && Buffer.byteLength(text) > options.maxBytes ? text.slice(0, options.maxBytes) : text,
|
|
128
|
+
err: err.join(""),
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Run a command purely for its stdout, e.g. a `-json` query. Empty string on failure. */
|
|
133
|
+
export async function capture(command, args, options = {}) {
|
|
134
|
+
const result = await captureFull(command, args, options)
|
|
135
|
+
return result.code === 0 ? result.out : ""
|
|
124
136
|
}
|
|
125
137
|
|
|
126
138
|
/** Direct child pids of `pid`. POSIX only; returns nothing when pgrep is unavailable. */
|
|
@@ -715,6 +727,87 @@ export function adb() {
|
|
|
715
727
|
return exe ?? "adb"
|
|
716
728
|
}
|
|
717
729
|
|
|
730
|
+
// ── classified adb boundary (Wi-Fi resilience; pattern credited to boheastill/phone-eye) ──
|
|
731
|
+
|
|
732
|
+
/** Wi-Fi adb serials end in `ip:port` (`192.168.1.23:5555`); USB/emulator serials never do. */
|
|
733
|
+
export function isWifiSerial(serial) {
|
|
734
|
+
return /:\d+$/.test(serial)
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
const ADB_TRANSPORT_RE = /no devices|device (?:'.*?' )?not found|device offline|device unauthorized|device still connecting|error: closed|cannot connect to daemon|failed to start daemon/i
|
|
738
|
+
|
|
739
|
+
/** Why an adb command failed: 'multi-device', 'transport' (transient — a reconnect may fix it), or undefined (real command error). */
|
|
740
|
+
export function classifyAdbFailure(result) {
|
|
741
|
+
const text = `${result.err}\n${result.out}`
|
|
742
|
+
if (/more than one device/i.test(text)) return "multi-device"
|
|
743
|
+
if (ADB_TRANSPORT_RE.test(text)) return "transport"
|
|
744
|
+
return undefined
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
/**
|
|
748
|
+
* Commands that may be auto-replayed after a reconnect: read-only probes only.
|
|
749
|
+
* Anything else (input/am/pm-mutate/install/screencap-to-file) must never
|
|
750
|
+
* replay — a replayed `input tap` could double-tap a payment button.
|
|
751
|
+
*/
|
|
752
|
+
export function replaySafeAdb(args) {
|
|
753
|
+
return /^(exec-out (screencap|cat|uiautomator|logcat|getprop|dumpsys)|shell (screencap|uiautomator|dumpsys|getprop|wm|settings get|pm list|ime list|cat|df|dmesg|logcat|true)|logcat( |$)|emu avd name)/.test(args.join(" "))
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
function adbFailTail(result) {
|
|
757
|
+
const text = `${result.err}\n${result.out}`.trim().split(/\r?\n/).filter(Boolean).slice(-3).join(" | ")
|
|
758
|
+
return text.slice(-300)
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
async function adbConnect(serial) {
|
|
762
|
+
const result = await captureFull(adb(), ["connect", serial], { timeoutMs: 15_000 })
|
|
763
|
+
return /connected|already connected/i.test(`${result.out}\n${result.err}`)
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
/**
|
|
767
|
+
* The single classified boundary for every serial-targeted adb command.
|
|
768
|
+
* On a transient transport failure with a Wi-Fi serial (ip:port) it attempts
|
|
769
|
+
* exactly one `adb connect`; only read-only commands are then replayed, while
|
|
770
|
+
* side-effectful ones raise "reconnected — call again". USB serials get a
|
|
771
|
+
* classified, actionable message instead of an empty result.
|
|
772
|
+
*/
|
|
773
|
+
export async function adbRun(serial, args, options = {}) {
|
|
774
|
+
const argv = ["-s", serial, ...args]
|
|
775
|
+
let result = await captureFull(adb(), argv, options)
|
|
776
|
+
if (result.code === 0) return result.out
|
|
777
|
+
const kind = classifyAdbFailure(result)
|
|
778
|
+
if (kind === "multi-device") throw new Error("adb: more than one device/emulator attached — pass an explicit serial (see adb devices)")
|
|
779
|
+
if (kind !== "transport" || !isWifiSerial(serial)) {
|
|
780
|
+
throw new Error(kind === "transport"
|
|
781
|
+
? `device ${serial} unreachable (${adbFailTail(result)}). For Wi-Fi adb run: adb connect <ip>:5555`
|
|
782
|
+
: `adb ${String(args[0])} failed on ${serial} (exit ${result.code}): ${adbFailTail(result)}`)
|
|
783
|
+
}
|
|
784
|
+
if (!(await adbConnect(serial))) {
|
|
785
|
+
throw new Error(`device ${serial} unreachable — adb connect failed; check the phone's Wi-Fi IP or replug USB once`)
|
|
786
|
+
}
|
|
787
|
+
if (!replaySafeAdb(args)) {
|
|
788
|
+
throw new Error(`device ${serial} reconnected over Wi-Fi; command not auto-replayed (side effects) — call again`)
|
|
789
|
+
}
|
|
790
|
+
result = await captureFull(adb(), argv, options)
|
|
791
|
+
if (result.code !== 0) throw new Error(`device ${serial} still failing after reconnect (exit ${result.code}): ${adbFailTail(result)}`)
|
|
792
|
+
return result.out
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/** Quote one value for the device shell (`adb shell` rejoins argv through sh -c): '…' with ' → '\''; control chars refused. */
|
|
796
|
+
export function shQuoteDevice(text) {
|
|
797
|
+
if (/[\x00-\x1f]/.test(text)) throw new Error(`refusing control characters in adb shell argument: ${JSON.stringify(text.slice(0, 40))}`)
|
|
798
|
+
return `'${text.replace(/'/g, `'\\''`)}'`
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/** Build the `am start` argv from intent parts; values are device-shell quoted so metacharacters stay inert. */
|
|
802
|
+
export function adbIntentArgs({ action, uri, component } = {}) {
|
|
803
|
+
const parts = ["shell", "am", "start"]
|
|
804
|
+
for (const [flag, value] of [["-a", action], ["-d", uri], ["-n", component]]) {
|
|
805
|
+
if (value === undefined || String(value) === "") continue
|
|
806
|
+
parts.push(flag, shQuoteDevice(String(value)))
|
|
807
|
+
}
|
|
808
|
+
return parts
|
|
809
|
+
}
|
|
810
|
+
|
|
718
811
|
/** SDK emulator binary (emulator.exe on Windows), or undefined. */
|
|
719
812
|
export function emulatorBinary() {
|
|
720
813
|
const sdk = androidSdk()
|
|
@@ -736,13 +829,13 @@ export async function androidAvds() {
|
|
|
736
829
|
|
|
737
830
|
/** True once the serial has finished booting (sys.boot_completed == 1). */
|
|
738
831
|
export async function androidBooted(serial) {
|
|
739
|
-
const output = await
|
|
832
|
+
const output = await adbRun(serial, ["shell", "getprop", "sys.boot_completed"]).catch(() => "")
|
|
740
833
|
return output.trim() === "1"
|
|
741
834
|
}
|
|
742
835
|
|
|
743
836
|
/** AVD name of an emulator serial (`adb emu avd name`); undefined for physical/offline. */
|
|
744
837
|
export async function avdName(serial) {
|
|
745
|
-
const output = await
|
|
838
|
+
const output = await adbRun(serial, ["emu", "avd", "name"]).catch(() => "")
|
|
746
839
|
const line = output
|
|
747
840
|
.split(/\r?\n/)
|
|
748
841
|
.map((item) => item.trim())
|
|
@@ -771,14 +864,14 @@ export function bootEmulator(avd) {
|
|
|
771
864
|
|
|
772
865
|
/** True when the ADBKeyboard IME is installed (the only way to type non-ASCII over adb). */
|
|
773
866
|
export async function adbKeyboardReady(serial) {
|
|
774
|
-
const output = await
|
|
867
|
+
const output = await adbRun(serial, ["shell", "ime", "list", "-s"]).catch(() => "")
|
|
775
868
|
return /com\.android\.adbkeyboard/i.test(output)
|
|
776
869
|
}
|
|
777
870
|
|
|
778
871
|
/** Type one string via the ADBKeyboard broadcast (base64, so any codepoint survives the shell). */
|
|
779
872
|
export async function typeViaAdbKeyboard(serial, text) {
|
|
780
873
|
const msg = Buffer.from(text, "utf8").toString("base64")
|
|
781
|
-
await
|
|
874
|
+
await adbRun(serial, ["shell", "am", "broadcast", "-a", "ADB_INPUT_B64", "--es", "msg", msg])
|
|
782
875
|
}
|
|
783
876
|
|
|
784
877
|
/** True when every codepoint is safe for `adb shell input text` (printable ASCII, no shell metachars). */
|
|
@@ -819,14 +912,14 @@ export async function androidDevice() {
|
|
|
819
912
|
|
|
820
913
|
/** Primary CPU ABI of a device, e.g. `arm64-v8a`. */
|
|
821
914
|
export async function androidAbi(serial) {
|
|
822
|
-
const output = await
|
|
915
|
+
const output = await adbRun(serial, ["shell", "getprop", "ro.product.cpu.abi"]).catch(() => "")
|
|
823
916
|
const abi = output.trim()
|
|
824
917
|
return /^[a-z0-9_-]+$/i.test(abi) ? abi : undefined
|
|
825
918
|
}
|
|
826
919
|
|
|
827
920
|
/** Free space on the device's data partition in megabytes, when `df` reports it. */
|
|
828
921
|
export async function androidFreeMb(serial) {
|
|
829
|
-
return parseFreeMb(await
|
|
922
|
+
return parseFreeMb(await adbRun(serial, ["shell", "df", "-k", "/data"]).catch(() => ""))
|
|
830
923
|
}
|
|
831
924
|
|
|
832
925
|
/** Second line of `df -k`: Filesystem 1K-blocks Used Available Use% Mounted. */
|
|
@@ -931,7 +1024,7 @@ export async function devices(serial) {
|
|
|
931
1024
|
/** Local path of a fresh screenshot of the serial. undefined on failure. */
|
|
932
1025
|
export async function screenCapture(serial, outDir) {
|
|
933
1026
|
const remote = "/sdcard/dsh-mobilecode-shot.png"
|
|
934
|
-
await
|
|
1027
|
+
await adbRun(serial, ["shell", "screencap", "-p", remote])
|
|
935
1028
|
const name = `screen-${serial}-${Date.now()}.png`
|
|
936
1029
|
const local = path.join(outDir ?? os.tmpdir(), name)
|
|
937
1030
|
const pulled = await new Promise((resolve) => {
|
|
@@ -952,8 +1045,8 @@ export async function screenCapture(serial, outDir) {
|
|
|
952
1045
|
*/
|
|
953
1046
|
export async function uiDump(serial) {
|
|
954
1047
|
const remote = "/sdcard/dsh-mobilecode-ui.xml"
|
|
955
|
-
await
|
|
956
|
-
const xml = await
|
|
1048
|
+
await adbRun(serial, ["shell", "uiautomator", "dump", remote])
|
|
1049
|
+
const xml = await adbRun(serial, ["shell", "cat", remote])
|
|
957
1050
|
const items = []
|
|
958
1051
|
const node = /<node[^>]*>/g
|
|
959
1052
|
for (const match of xml.match(node) ?? []) {
|
|
@@ -975,7 +1068,7 @@ export async function uiDump(serial) {
|
|
|
975
1068
|
|
|
976
1069
|
/** Foreground activity, e.g. "com.foo/.MainActivity", or undefined. */
|
|
977
1070
|
export async function foregroundActivity(serial) {
|
|
978
|
-
const output = await
|
|
1071
|
+
const output = await adbRun(serial, ["shell", "dumpsys", "activity", "activities"]).catch(() => "")
|
|
979
1072
|
const line = output.split("\n").find((item) => /topResumedActivity|mResumedActivity/.test(item))
|
|
980
1073
|
const match = /ActivityRecord\{[^}]*\s([^\s}]+)\}/.exec(line ?? "")
|
|
981
1074
|
return match?.[1] ?? undefined
|
|
@@ -983,14 +1076,14 @@ export async function foregroundActivity(serial) {
|
|
|
983
1076
|
|
|
984
1077
|
/** Kernel log (dmesg). Requires adb root — works on emulators, usually not on real devices. */
|
|
985
1078
|
export async function dmesg(serial) {
|
|
986
|
-
return
|
|
1079
|
+
return adbRun(serial, ["shell", "dmesg"])
|
|
987
1080
|
}
|
|
988
1081
|
|
|
989
1082
|
/** logcat snapshot, filtered by buffer/level/package-like substring. */
|
|
990
1083
|
export async function logcat(serial, { buffer = "main", lines = 200, filter } = {}) {
|
|
991
|
-
const args = ["
|
|
1084
|
+
const args = ["logcat", "-d", "-t", String(lines)]
|
|
992
1085
|
if (buffer && buffer !== "all") args.push("-b", buffer)
|
|
993
|
-
let output = await
|
|
1086
|
+
let output = await adbRun(serial, args)
|
|
994
1087
|
if (filter) output = output.split("\n").filter((line) => line.toLowerCase().includes(filter.toLowerCase())).join("\n")
|
|
995
1088
|
return output
|
|
996
1089
|
}
|
package/lib/index.js
CHANGED
|
@@ -21,6 +21,7 @@ import * as UiTree from './uitree.js'
|
|
|
21
21
|
import * as FrameSource from './frame-source.js'
|
|
22
22
|
import * as StreamAccess from './stream-access.js'
|
|
23
23
|
import { AndroidStreamHost, ROTATION_CYCLE } from './android-stream.js'
|
|
24
|
+
import * as Vision from './vision.js'
|
|
24
25
|
import { DevicePreviewEngine } from './device-preview.js'
|
|
25
26
|
import * as Setup from './setup.js'
|
|
26
27
|
import { registerMobileSkill } from './skill.js'
|
|
@@ -219,6 +220,20 @@ function makeRoutes(engine, config, stream) {
|
|
|
219
220
|
writeJson(res, 200, { ok: true })
|
|
220
221
|
},
|
|
221
222
|
},
|
|
223
|
+
// GET /api/dsh-mobilecode/connection → {adb, devices:[{serial, state, model?, wifi}]} — connection card data.
|
|
224
|
+
{
|
|
225
|
+
kind: 'exact',
|
|
226
|
+
path: API_BASE + '/connection',
|
|
227
|
+
handler: async (req, res) => {
|
|
228
|
+
if (!guard(req, res)) return
|
|
229
|
+
if ((req.method ?? 'GET') !== 'GET') { writeJson(res, 405, { error: 'method not allowed' }); return }
|
|
230
|
+
try {
|
|
231
|
+
writeJson(res, 200, { adb: adbHostInfo(), devices: await connectionDevices() })
|
|
232
|
+
} catch (error) {
|
|
233
|
+
writeJson(res, 500, { error: error instanceof Error ? error.message : String(error) })
|
|
234
|
+
}
|
|
235
|
+
},
|
|
236
|
+
},
|
|
222
237
|
// GET /api/dsh-mobilecode/doctor → [{name, ok, detail, fix?}] — plugin health check.
|
|
223
238
|
{
|
|
224
239
|
kind: 'exact',
|
|
@@ -727,11 +742,10 @@ function deviceInputTool() {
|
|
|
727
742
|
async execute(args) {
|
|
728
743
|
const serial = await requireAndroidDevice(args.serial)
|
|
729
744
|
const action = args.action ?? 'tap'
|
|
730
|
-
const adbArgs = (shell) => ['-s', serial, 'shell', ...shell]
|
|
731
745
|
switch (action) {
|
|
732
746
|
case 'tap': {
|
|
733
747
|
if (typeof args.x !== 'number' || typeof args.y !== 'number') throw new Error('action=tap requires x and y (integer pixels).')
|
|
734
|
-
await DeviceBuild.
|
|
748
|
+
await DeviceBuild.adbRun(serial, ['shell', 'input', 'tap', String(args.x), String(args.y)])
|
|
735
749
|
return { serial, action, sent: `tap ${args.x},${args.y}` }
|
|
736
750
|
}
|
|
737
751
|
case 'swipe': {
|
|
@@ -739,13 +753,13 @@ function deviceInputTool() {
|
|
|
739
753
|
throw new Error('action=swipe requires x, y, x2, y2.')
|
|
740
754
|
}
|
|
741
755
|
const duration = args.duration ?? 200
|
|
742
|
-
await DeviceBuild.
|
|
756
|
+
await DeviceBuild.adbRun(serial, ['shell', 'input', 'swipe', String(args.x), String(args.y), String(args.x2), String(args.y2), String(duration)])
|
|
743
757
|
return { serial, action, sent: `swipe ${args.x},${args.y}→${args.x2},${args.y2} (${duration}ms)` }
|
|
744
758
|
}
|
|
745
759
|
case 'text': {
|
|
746
760
|
if (typeof args.text !== 'string' || args.text.length === 0) throw new Error('action=text requires a non-empty text string.')
|
|
747
761
|
if (DeviceBuild.isAsciiInput(args.text)) {
|
|
748
|
-
await DeviceBuild.
|
|
762
|
+
await DeviceBuild.adbRun(serial, ['shell', 'input', 'text', DeviceBuild.escapeInputText(args.text)])
|
|
749
763
|
return { serial, action, sent: `text "${args.text}"` }
|
|
750
764
|
}
|
|
751
765
|
// Non-ASCII (CJK, emoji, accented) cannot go through `input text`; the
|
|
@@ -764,7 +778,7 @@ function deviceInputTool() {
|
|
|
764
778
|
const raw = String(args.key ?? '')
|
|
765
779
|
const code = /^\d+$/.test(raw) ? Number(raw) : KEYCODES[raw.toLowerCase()]
|
|
766
780
|
if (!code) throw new Error(`unknown key "${raw}" — use a name from ${Object.keys(KEYCODES).join(', ')} or a raw keycode integer.`)
|
|
767
|
-
await DeviceBuild.
|
|
781
|
+
await DeviceBuild.adbRun(serial, ['shell', 'input', 'keyevent', String(code)])
|
|
768
782
|
return { serial, action, sent: `key ${raw} (${code})` }
|
|
769
783
|
}
|
|
770
784
|
default:
|
|
@@ -774,7 +788,71 @@ function deviceInputTool() {
|
|
|
774
788
|
})
|
|
775
789
|
}
|
|
776
790
|
|
|
777
|
-
/**
|
|
791
|
+
/** Plain-device adb diagnostics (no serial) for the settings Connection card. */
|
|
792
|
+
export function adbHostInfo() {
|
|
793
|
+
return DeviceBuild.adb()
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
/** Lines of `adb devices -l` after the header rows, parsed for the settings Connection card. */
|
|
797
|
+
export async function connectionDevices() {
|
|
798
|
+
const output = await DeviceBuild.capture(DeviceBuild.adb(), ["devices", "-l"])
|
|
799
|
+
return parseDeviceLongList(output)
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/** Parse `adb devices -l` into [{serial, state, model, wifi}]. */
|
|
803
|
+
export function parseDeviceLongList(output) {
|
|
804
|
+
return String(output)
|
|
805
|
+
.split(/\r?\n/)
|
|
806
|
+
.map((line) => line.trim())
|
|
807
|
+
.filter((line) => line && !/^List of devices/i.test(line) && !/^\* daemon/i.test(line))
|
|
808
|
+
.map((line) => {
|
|
809
|
+
const [serial, state, ...rest] = line.split(/\s+/)
|
|
810
|
+
const model = /model:(\S+)/.exec(rest.join(" "))?.[1]
|
|
811
|
+
const row = { serial, state: state || "unknown", wifi: DeviceBuild.isWifiSerial(serial) }
|
|
812
|
+
if (model) row.model = model
|
|
813
|
+
return row
|
|
814
|
+
})
|
|
815
|
+
.filter((row) => row.serial)
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
function deviceIntentTool() {
|
|
819
|
+
return defineTool({
|
|
820
|
+
name: 'device_intent',
|
|
821
|
+
description: 'Open anything on the device by Android intent: an action (e.g. android.settings.WIFI_SETTINGS), ' +
|
|
822
|
+
'a deep-link URI, or an explicit component (package/.Activity). Reaches screens no tap can address — deep ' +
|
|
823
|
+
'settings pages, app deep links, files. Values are device-shell quoted so metacharacters stay inert.',
|
|
824
|
+
parameters: {
|
|
825
|
+
serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
|
|
826
|
+
action: { type: 'string', description: 'Intent action, e.g. android.settings.WIFI_SETTINGS or android.intent.action.VIEW.' },
|
|
827
|
+
uri: { type: 'string', description: 'Data URI the intent carries, e.g. geo:0,0?q=Berlin or a https:// deep link.' },
|
|
828
|
+
component: { type: 'string', description: 'Explicit component, e.g. com.android.settings/.Settings.' },
|
|
829
|
+
},
|
|
830
|
+
output: {
|
|
831
|
+
schema: {
|
|
832
|
+
type: 'object',
|
|
833
|
+
additionalProperties: false,
|
|
834
|
+
properties: {
|
|
835
|
+
serial: { type: 'string', required: true },
|
|
836
|
+
intent: { type: 'string', required: true },
|
|
837
|
+
started: { type: 'boolean', required: true },
|
|
838
|
+
},
|
|
839
|
+
},
|
|
840
|
+
render: (_args, value) => [{ type: 'text', text: `Started ${value?.intent} on ${value?.serial}` }],
|
|
841
|
+
},
|
|
842
|
+
async execute(args) {
|
|
843
|
+
const serial = await requireAndroidDevice(args.serial)
|
|
844
|
+
const argv = DeviceBuild.adbIntentArgs({ action: args.action, uri: args.uri, component: args.component })
|
|
845
|
+
if (argv.length === 3) throw new Error('device_intent needs at least one of action, uri or component.')
|
|
846
|
+
const output = await DeviceBuild.adbRun(serial, argv)
|
|
847
|
+
if (/^Error/i.test(output.trim())) throw new Error(`am start refused the intent on ${serial}: ${output.trim().split(/\r?\n/)[0]}`)
|
|
848
|
+
const label = [["-a", args.action], ["-d", args.uri], ["-n", args.component]]
|
|
849
|
+
.filter(([, value]) => value !== undefined && value !== '')
|
|
850
|
+
.map(([flag, value]) => `${flag} ${value}`)
|
|
851
|
+
.join(' ')
|
|
852
|
+
return { serial, intent: label, started: true }
|
|
853
|
+
},
|
|
854
|
+
})
|
|
855
|
+
}
|
|
778
856
|
async function ocrHasText(serial, wantedLower) {
|
|
779
857
|
if (!DeviceBuild.ocrPython()) return undefined
|
|
780
858
|
const png = await DeviceBuild.screenCapture(serial)
|
|
@@ -930,7 +1008,7 @@ function deviceShutdownTool() {
|
|
|
930
1008
|
if (!isEmulator) {
|
|
931
1009
|
throw new Error(`device_shutdown refuses ${serial}: it is a physical device and adb has no power-off verb for phones — use its own power button.`)
|
|
932
1010
|
}
|
|
933
|
-
await DeviceBuild.
|
|
1011
|
+
await DeviceBuild.adbRun(serial, ['emu', 'kill'])
|
|
934
1012
|
return { serial, shutdown: true }
|
|
935
1013
|
},
|
|
936
1014
|
})
|
|
@@ -970,15 +1048,15 @@ function deviceActionTool() {
|
|
|
970
1048
|
const serial = await requireAndroidDevice(args.serial)
|
|
971
1049
|
const action = String(args.action ?? '')
|
|
972
1050
|
if (action === 'rotate') {
|
|
973
|
-
const current = Number(await DeviceBuild.
|
|
1051
|
+
const current = Number(await DeviceBuild.adbRun(serial, ['shell', 'settings', 'get', 'system', 'user_rotation']))
|
|
974
1052
|
const next = ((Number.isFinite(current) ? current : 0) + 1) % 4
|
|
975
|
-
await DeviceBuild.
|
|
976
|
-
await DeviceBuild.
|
|
1053
|
+
await DeviceBuild.adbRun(serial, ['shell', 'settings', 'put', 'system', 'accelerometer_rotation', '0'])
|
|
1054
|
+
await DeviceBuild.adbRun(serial, ['shell', 'settings', 'put', 'system', 'user_rotation', String(next)])
|
|
977
1055
|
return { serial, action, rotation: next }
|
|
978
1056
|
}
|
|
979
1057
|
const shell = DEVICE_ACTIONS[action]
|
|
980
1058
|
if (!shell) throw new Error(`unknown action "${action}" — use ${[...Object.keys(DEVICE_ACTIONS), 'rotate'].join(', ')}.`)
|
|
981
|
-
await DeviceBuild.
|
|
1059
|
+
await DeviceBuild.adbRun(serial, ['shell', ...shell])
|
|
982
1060
|
return { serial, action }
|
|
983
1061
|
},
|
|
984
1062
|
})
|
|
@@ -1011,7 +1089,7 @@ function deviceAppsTool() {
|
|
|
1011
1089
|
},
|
|
1012
1090
|
async execute(args) {
|
|
1013
1091
|
const serial = await requireAndroidDevice(args.serial)
|
|
1014
|
-
const output = await DeviceBuild.
|
|
1092
|
+
const output = await DeviceBuild.adbRun(serial, ['shell', 'pm', 'list', 'packages', ...(args.include_system ? [] : ['-3'])])
|
|
1015
1093
|
const packages = output
|
|
1016
1094
|
.split(/\r?\n/)
|
|
1017
1095
|
.map((line) => line.trim())
|
|
@@ -1049,7 +1127,7 @@ function deviceLaunchAppTool() {
|
|
|
1049
1127
|
const serial = await requireAndroidDevice(args.serial)
|
|
1050
1128
|
let pkg = String(args.package ?? '').trim()
|
|
1051
1129
|
if (pkg === '') throw new Error('device_launch_app requires a package name (or a unique substring).')
|
|
1052
|
-
const listOut = await DeviceBuild.
|
|
1130
|
+
const listOut = await DeviceBuild.adbRun(serial, ['shell', 'pm', 'list', 'packages'])
|
|
1053
1131
|
const all = listOut
|
|
1054
1132
|
.split(/\r?\n/)
|
|
1055
1133
|
.map((line) => line.trim())
|
|
@@ -1061,9 +1139,9 @@ function deviceLaunchAppTool() {
|
|
|
1061
1139
|
if (matches.length > 1) throw new Error(`"${pkg}" matches ${matches.length} packages (${matches.slice(0, 8).join(', ')}${matches.length > 8 ? ', …' : ''}) — be more specific.`)
|
|
1062
1140
|
pkg = matches[0]
|
|
1063
1141
|
}
|
|
1064
|
-
if (args.relaunch) await DeviceBuild.
|
|
1065
|
-
const
|
|
1066
|
-
if (
|
|
1142
|
+
if (args.relaunch) await DeviceBuild.adbRun(serial, ['shell', 'am', 'force-stop', pkg])
|
|
1143
|
+
const launched = await DeviceBuild.adbRun(serial, ['shell', 'monkey', '-p', pkg, '-c', 'android.intent.category.LAUNCHER', '1']).catch((error) => { throw new Error(`Could not launch ${pkg}: ${error instanceof Error ? error.message : error}`) })
|
|
1144
|
+
if (/no activities found|no events to send/i.test(launched)) throw new Error(`Could not launch ${pkg} (no launcher activity).`)
|
|
1067
1145
|
return { serial, package: pkg, launched: true }
|
|
1068
1146
|
},
|
|
1069
1147
|
})
|
|
@@ -1120,7 +1198,7 @@ function deviceStreamTool(host, access) {
|
|
|
1120
1198
|
})
|
|
1121
1199
|
}
|
|
1122
1200
|
|
|
1123
|
-
function deviceScreenTool(engine) {
|
|
1201
|
+
function deviceScreenTool(engine, vision) {
|
|
1124
1202
|
return defineTool({
|
|
1125
1203
|
name: 'device_screen',
|
|
1126
1204
|
description: 'See what is on an attached Android device right now: captures the screen as a PNG file, dumps the UI ' +
|
|
@@ -1177,6 +1255,7 @@ function deviceScreenTool(engine) {
|
|
|
1177
1255
|
},
|
|
1178
1256
|
},
|
|
1179
1257
|
ocrError: { type: 'string' },
|
|
1258
|
+
image: Vision.IMAGE_REF_SCHEMA,
|
|
1180
1259
|
},
|
|
1181
1260
|
},
|
|
1182
1261
|
render: (_args, value) => {
|
|
@@ -1200,10 +1279,13 @@ function deviceScreenTool(engine) {
|
|
|
1200
1279
|
if (v.ocr.length > 40) lines.push(` … and ${v.ocr.length - 40} more`)
|
|
1201
1280
|
}
|
|
1202
1281
|
if (!v.ui?.length && !v.ocr?.length) lines.push('No text found on screen.')
|
|
1203
|
-
|
|
1282
|
+
const blocks = [{ type: 'text', text: lines.join('\n') }]
|
|
1283
|
+
// When the routed model accepts images, the screenshot rides along as a
|
|
1284
|
+
// real image block so the model SEES the screen (see lib/vision.js).
|
|
1285
|
+
return Vision.appendImageBlock(blocks, v)
|
|
1204
1286
|
},
|
|
1205
1287
|
},
|
|
1206
|
-
async execute(args) {
|
|
1288
|
+
async execute(args, exec) {
|
|
1207
1289
|
const serial = await requireAndroidDevice(args.serial)
|
|
1208
1290
|
const png = await DeviceBuild.screenCapture(serial, args.directory)
|
|
1209
1291
|
const [ui, foreground, size] = await Promise.all([
|
|
@@ -1227,13 +1309,15 @@ function deviceScreenTool(engine) {
|
|
|
1227
1309
|
else out.ocrError = 'PaddleOCR returned no text (or failed silently)'
|
|
1228
1310
|
}
|
|
1229
1311
|
}
|
|
1312
|
+
const image = await Vision.maybeAttachScreenshot(vision, png, exec)
|
|
1313
|
+
if (image !== undefined) out.image = image
|
|
1230
1314
|
return out
|
|
1231
1315
|
},
|
|
1232
1316
|
})
|
|
1233
1317
|
}
|
|
1234
1318
|
|
|
1235
1319
|
async function captureScreenSize(serial) {
|
|
1236
|
-
const output = await DeviceBuild.
|
|
1320
|
+
const output = await DeviceBuild.adbRun(serial, ['shell', 'wm', 'size'])
|
|
1237
1321
|
// An `wm size` override wins over the physical panel — the input space is the override.
|
|
1238
1322
|
const override = /Override size:\s*(\d+)x(\d+)/.exec(output)
|
|
1239
1323
|
const match = override ?? /Physical size:\s*(\d+)x(\d+)/.exec(output)
|
|
@@ -1355,7 +1439,7 @@ function deviceTapElementTool() {
|
|
|
1355
1439
|
const selector = { identifier: args.resource_id, label: args.text }
|
|
1356
1440
|
const { node, matchedBy } = UiTree.resolveTapTarget(parsed.roots, selector, { tool: 'device_tap_element', allowOffscreen: args.allow_offscreen === true })
|
|
1357
1441
|
const center = UiTree.boundsCenter(node.bounds)
|
|
1358
|
-
await DeviceBuild.
|
|
1442
|
+
await DeviceBuild.adbRun(serial, ['shell', 'input', 'tap', String(center.x), String(center.y)])
|
|
1359
1443
|
const describe = () => {
|
|
1360
1444
|
const parts = []
|
|
1361
1445
|
if (node.resourceId) parts.push(`resource_id ${node.resourceId}`)
|
|
@@ -1572,6 +1656,10 @@ function guidance() {
|
|
|
1572
1656
|
'- device_action: notifications / quick_settings / collapse / lock / wake / assistant / rotate.',
|
|
1573
1657
|
'- device_boot / device_shutdown: boot an AVD by name and wait for boot / shut an emulator down (refuses physical devices).',
|
|
1574
1658
|
'- device_apps / device_launch_app: list installed packages (never guess a package name) / launch one by package or unique substring.',
|
|
1659
|
+
'- device_intent: open anything by Android intent (action, deep-link URI, or package/.Activity). Reaches screens no tap can address.',
|
|
1660
|
+
' Every adb command runs through one classified boundary: a dropped Wi-Fi connection (ip:port serial) gets exactly one',
|
|
1661
|
+
' `adb connect` attempt, read-only commands then replay automatically, and side-effectful ones refuse the replay —',
|
|
1662
|
+
' a replayed tap could double-tap — and raise "reconnected — call again" instead.',
|
|
1575
1663
|
'- device_stream: drive the live screen stream the Devices panel shows (status / start an online device / stop). Agents',
|
|
1576
1664
|
' that just need to see the screen should prefer device_screen or device_ui_tree.',
|
|
1577
1665
|
'- device_log: read device logs (logcat main/crash/events, kernel dmesg). Call it when a run fails or an app misbehaves.',
|
|
@@ -1604,6 +1692,7 @@ export function apply(ctx, config) {
|
|
|
1604
1692
|
const engine = new DevicePreviewEngine()
|
|
1605
1693
|
const streamHost = new AndroidStreamHost()
|
|
1606
1694
|
const streamAccess = new StreamAccess.StreamAccessController()
|
|
1695
|
+
const vision = Vision.resolveVisionServices(ctx)
|
|
1607
1696
|
const handle = {
|
|
1608
1697
|
engine,
|
|
1609
1698
|
stream: streamHost,
|
|
@@ -1644,7 +1733,7 @@ export function apply(ctx, config) {
|
|
|
1644
1733
|
const disposers = [
|
|
1645
1734
|
deviceRunTool(engine, config),
|
|
1646
1735
|
deviceDetectTool(engine, config),
|
|
1647
|
-
deviceScreenTool(engine),
|
|
1736
|
+
deviceScreenTool(engine, vision),
|
|
1648
1737
|
deviceUiTreeTool(),
|
|
1649
1738
|
deviceTapElementTool(),
|
|
1650
1739
|
deviceWaitForTool(),
|
|
@@ -1653,6 +1742,7 @@ export function apply(ctx, config) {
|
|
|
1653
1742
|
deviceActionTool(),
|
|
1654
1743
|
deviceAppsTool(),
|
|
1655
1744
|
deviceLaunchAppTool(),
|
|
1745
|
+
deviceIntentTool(),
|
|
1656
1746
|
deviceStreamTool(streamHost, streamAccess),
|
|
1657
1747
|
deviceLogTool(engine),
|
|
1658
1748
|
deviceStatusTool(engine),
|
package/lib/uitree.js
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* Ported design credit: ZSeven-W/dsh-android (MIT) src/uitree.ts.
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import { adb,
|
|
16
|
+
import { adb, adbRun, exec } from "./device-build.js"
|
|
17
17
|
|
|
18
18
|
const DUMP_TIMEOUT_MS = 60_000
|
|
19
19
|
const DUMP_MAX_BYTES = 8 * 1024 * 1024
|
|
@@ -260,7 +260,7 @@ export async function dumpUiTreeXml(serial) {
|
|
|
260
260
|
let primaryFailure
|
|
261
261
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
262
262
|
try {
|
|
263
|
-
const buffer = await
|
|
263
|
+
const buffer = await adbRun(serial, ["exec-out", "uiautomator", "dump", "/dev/tty"], options)
|
|
264
264
|
if (buffer === "") throw new Error("the device produced no output")
|
|
265
265
|
return extractHierarchyXml(buffer)
|
|
266
266
|
} catch (error) {
|
|
@@ -284,8 +284,8 @@ export async function dumpUiTreeXml(serial) {
|
|
|
284
284
|
}
|
|
285
285
|
const remotePath = "/sdcard/window_dump.xml"
|
|
286
286
|
try {
|
|
287
|
-
const notice = await
|
|
288
|
-
const buffer = await
|
|
287
|
+
const notice = await adbRun(serial, ["shell", "uiautomator", "dump", remotePath], options)
|
|
288
|
+
const buffer = await adbRun(serial, ["exec-out", "cat", remotePath], options)
|
|
289
289
|
const xml = extractHierarchyXml(buffer)
|
|
290
290
|
await exec(adb(), ["-s", serial, "shell", "rm", "-f", remotePath]).exit.catch(() => {})
|
|
291
291
|
if (xml.trim() === "") throw new Error(notice.trim() || "empty dump")
|
package/lib/vision.js
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-mobilecode — native multimodal delivery.
|
|
3
|
+
*
|
|
4
|
+
* When the routed model declares image input, the capture tools hand the model
|
|
5
|
+
* the screenshot ITSELF (a `{type:'image', attachment}` block) instead of only a
|
|
6
|
+
* file path it would have to open. DSH 0.1.1 carries images end to end: tool
|
|
7
|
+
* results may contain image blocks, bytes live in the durable attachment store
|
|
8
|
+
* (`ctx.get('attachments')`), and `llm.resolveModelInfo(...).inputModalities`
|
|
9
|
+
* says whether the routed model accepts images. This mirrors the in-tree
|
|
10
|
+
* `read_image` tool in dsh-tool-fs.
|
|
11
|
+
*
|
|
12
|
+
* The deliberate difference from `read_image`: where that tool REFUSES on a
|
|
13
|
+
* text-only route (the image is its whole point), the capture tools here
|
|
14
|
+
* DEGRADE. The primary output is always the JSON summary; the image block is an
|
|
15
|
+
* enhancement added only when (a) the attachment store is mounted, (b) the
|
|
16
|
+
* calling route's resolved model declares `image` input, and (c) admission
|
|
17
|
+
* succeeds. Any failure in that chain silently keeps the text-only behavior, so
|
|
18
|
+
* text-only routes, headless profiles, and older hosts never see a new error.
|
|
19
|
+
*
|
|
20
|
+
* Everything is typed structurally — the plugin is plain JS and must not depend
|
|
21
|
+
* on the host's attachment type exports.
|
|
22
|
+
* @module vision
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { readFile } from 'node:fs/promises'
|
|
26
|
+
import path from 'node:path'
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Resolve the optional vision services from the plugin context. Both come back
|
|
30
|
+
* absent on hosts that do not mount them; every consumer treats that as
|
|
31
|
+
* "stay text-only".
|
|
32
|
+
*/
|
|
33
|
+
export function resolveVisionServices(ctx) {
|
|
34
|
+
const get = typeof ctx?.get === 'function' ? ctx.get.bind(ctx) : undefined
|
|
35
|
+
if (get === undefined) return {}
|
|
36
|
+
const attachments = get('attachments')
|
|
37
|
+
const llm = get('llm')
|
|
38
|
+
return {
|
|
39
|
+
...(attachments !== undefined && typeof attachments.saveImage === 'function' ? { attachments } : {}),
|
|
40
|
+
...(llm !== undefined && typeof llm.resolveModelInfo === 'function' ? { llm } : {}),
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* True when the calling route's resolved model declares `image` input. Mirrors
|
|
46
|
+
* `read_image`'s gate (request-header config first, then agent options) but
|
|
47
|
+
* answers false instead of throwing: a tool result that enters durable history
|
|
48
|
+
* must not carry an image its route cannot replay.
|
|
49
|
+
*/
|
|
50
|
+
export async function imageInputActive(services, exec) {
|
|
51
|
+
if (services.llm === undefined || services.attachments === undefined) return false
|
|
52
|
+
try {
|
|
53
|
+
const routed = exec?.agent?.session?.requestHeader?.()?.config
|
|
54
|
+
const provider = routed?.provider ?? exec?.agent?.options?.provider
|
|
55
|
+
const model = routed?.model ?? exec?.agent?.options?.model
|
|
56
|
+
if (provider === undefined || model === undefined) return false
|
|
57
|
+
const info = await services.llm.resolveModelInfo(provider, model, exec?.signal)
|
|
58
|
+
return info?.inputModalities?.includes('image') === true
|
|
59
|
+
} catch {
|
|
60
|
+
return false
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Durably commit one screenshot PNG and return the plain reference for the
|
|
66
|
+
* result value, or undefined when the store is absent or admission fails
|
|
67
|
+
* (oversized, malformed) — never an error, per the degrade-not-refuse rule.
|
|
68
|
+
*/
|
|
69
|
+
export async function saveScreenshotAttachment(services, png, name) {
|
|
70
|
+
const attachments = services.attachments
|
|
71
|
+
if (attachments === undefined) return undefined
|
|
72
|
+
try {
|
|
73
|
+
const ref = await attachments.saveImage({ data: png, mediaType: 'image/png', name })
|
|
74
|
+
if (typeof ref?.attachmentId !== 'string' || ref.attachmentId === '') return undefined
|
|
75
|
+
return {
|
|
76
|
+
attachmentId: ref.attachmentId,
|
|
77
|
+
mediaType: ref.mediaType,
|
|
78
|
+
bytes: ref.bytes,
|
|
79
|
+
width: ref.width,
|
|
80
|
+
height: ref.height,
|
|
81
|
+
...(ref.name === undefined ? {} : { name: ref.name }),
|
|
82
|
+
}
|
|
83
|
+
} catch {
|
|
84
|
+
return undefined
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Convenience for the capture tools: gate on the route, read the file, and save
|
|
90
|
+
* the attachment — returning undefined (degrade) on any miss. Never throws.
|
|
91
|
+
*/
|
|
92
|
+
export async function maybeAttachScreenshot(services, filePath, exec) {
|
|
93
|
+
if (services.attachments === undefined || typeof filePath !== 'string' || filePath === '') return undefined
|
|
94
|
+
if (!(await imageInputActive(services, exec))) return undefined
|
|
95
|
+
try {
|
|
96
|
+
const data = await readFile(filePath)
|
|
97
|
+
return await saveScreenshotAttachment(services, data, path.basename(filePath))
|
|
98
|
+
} catch {
|
|
99
|
+
return undefined
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Output-schema fragment for the optional `image` result field. */
|
|
104
|
+
export const IMAGE_REF_SCHEMA = {
|
|
105
|
+
type: 'object',
|
|
106
|
+
additionalProperties: false,
|
|
107
|
+
description: 'Durable attachment reference for the screenshot delivered to the model as an image block '
|
|
108
|
+
+ '(present only when the routed model declares image input).',
|
|
109
|
+
properties: {
|
|
110
|
+
attachmentId: { type: 'string', required: true },
|
|
111
|
+
mediaType: { type: 'string', required: true },
|
|
112
|
+
bytes: { type: 'number', required: true },
|
|
113
|
+
width: { type: 'number', required: true },
|
|
114
|
+
height: { type: 'number', required: true },
|
|
115
|
+
name: { type: 'string' },
|
|
116
|
+
},
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Append the image block to a render's content blocks when the value carries an
|
|
121
|
+
* `image` ref — so an image-capable model SEES the screen. Returns the same
|
|
122
|
+
* array for chaining.
|
|
123
|
+
*/
|
|
124
|
+
export function appendImageBlock(blocks, value) {
|
|
125
|
+
const image = value?.image
|
|
126
|
+
if (image !== undefined && typeof image.attachmentId === 'string') {
|
|
127
|
+
blocks.push({ type: 'image', attachment: image })
|
|
128
|
+
}
|
|
129
|
+
return blocks
|
|
130
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-mobilecode",
|
|
3
|
-
"description": "MobileCode for the dsh web GUI: detect iOS/Android projects, run
|
|
4
|
-
"version": "0.
|
|
3
|
+
"description": "MobileCode for the dsh web GUI: detect iOS/Android projects, run preview servers, and drive the simulator/emulator from the session — 15 agent tools (device_run, device_screen, device_ui_tree, device_tap_element, device_input, device_intent, device_log, live screen stream, multimodal screenshots) with one classified adb boundary. Hot-pluggable — mounted via the profile bundle list + cordis.patch.yml, no dsh source changes.",
|
|
4
|
+
"version": "0.5.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "pnpm@11.22.0",
|
|
7
7
|
"engines": {
|