@anuraghq/mobius 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mobiusagent-driver-darwin-arm64 +0 -0
- package/dist/execution/observe.js +3 -1
- package/dist/execution/session.d.ts +16 -5
- package/dist/execution/session.js +43 -19
- package/dist/interface/mcp.js +1 -0
- package/dist/interface/mobius-tools.js +60 -20
- package/ios-agent/setup.sh +10 -4
- package/package.json +35 -6
|
Binary file
|
|
@@ -152,7 +152,9 @@ export class ObservationStore {
|
|
|
152
152
|
nextId() { return `o${++this.seq}`; }
|
|
153
153
|
add(obs, fps) {
|
|
154
154
|
this.history.push({ obs, fps });
|
|
155
|
-
|
|
155
|
+
// ponytail: settle/verify polls add an observation every ~100 ms, so a small window evicts the ids the agent
|
|
156
|
+
// just got from observe after one slow action. 64 covers a full verify timeout; the on-disk copy stays at 5.
|
|
157
|
+
if (this.history.length > 64)
|
|
156
158
|
this.history.shift();
|
|
157
159
|
if (!this.file)
|
|
158
160
|
return;
|
|
@@ -20,10 +20,15 @@ export declare const DEFAULT_ACTION_TIMEOUT_MS = 20000;
|
|
|
20
20
|
export declare const DEFAULT_EXPECT_TIMEOUT_MS = 6000;
|
|
21
21
|
/** Cold app launches (Flutter/RN on iOS) take longer than a tap to show; live run: Scapia was still on SpringBoard at 6 s. */
|
|
22
22
|
export declare const LAUNCH_EXPECT_TIMEOUT_MS = 20000;
|
|
23
|
-
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Where to tap an element: inside its on-screen part, off its own clickable children (a card's inner button),
|
|
25
|
+
* and off anything drawn over it (a floating toolbar, tab bar or sheet: a later element that isn't its ancestor).
|
|
26
|
+
* Throws when no point of it is visibly tappable, so a covered or off-screen target is never tapped "successfully".
|
|
27
|
+
*/
|
|
28
|
+
export declare const center: (e: Element, screen?: readonly Element[], size?: {
|
|
29
|
+
width: number;
|
|
30
|
+
height: number;
|
|
31
|
+
}) => {
|
|
27
32
|
x: number;
|
|
28
33
|
y: number;
|
|
29
34
|
};
|
|
@@ -32,6 +37,8 @@ export declare function expectationMet(obs: Observation, ex: Expect): boolean;
|
|
|
32
37
|
export declare class DeviceSession {
|
|
33
38
|
readonly connector: Connector;
|
|
34
39
|
readonly store: ObservationStore;
|
|
40
|
+
/** The observation last shown to the agent (observe / list / snapshot): short refs like @e5 point into it, not into internal polls. */
|
|
41
|
+
shown: string | null;
|
|
35
42
|
private readonly audit;
|
|
36
43
|
private readonly policy;
|
|
37
44
|
private readonly approver;
|
|
@@ -46,7 +53,11 @@ export declare class DeviceSession {
|
|
|
46
53
|
/** Serialize work per device: one action at a time, no interleaved taps. */
|
|
47
54
|
private exclusive;
|
|
48
55
|
observe(): Promise<Observation>;
|
|
49
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* Re-observe until two consecutive screens hash the same (UI settled), bounded. State-based, no fixed sleeps.
|
|
58
|
+
* Two identical frames of the *pre-action* screen only count after a grace window: a tap that launches an
|
|
59
|
+
* activity takes a beat before the window changes, and settling early reports the old screen as the result.
|
|
60
|
+
*/
|
|
50
61
|
private settle;
|
|
51
62
|
/** Every element containing the point on the latest screen (overlays, the button below them, its labels). */
|
|
52
63
|
elementsAt(x: number, y: number): Element[];
|
|
@@ -10,32 +10,49 @@ export const DEFAULT_EXPECT_TIMEOUT_MS = 6_000;
|
|
|
10
10
|
export const LAUNCH_EXPECT_TIMEOUT_MS = 20_000;
|
|
11
11
|
/** Pause between re-reads while waiting for the screen. */
|
|
12
12
|
const POLL_MS = 100;
|
|
13
|
+
/** How long an unchanged pre-action screen is waited on before settle accepts it (see settle). */
|
|
14
|
+
const SETTLE_GRACE_MS = 600;
|
|
13
15
|
/** Text read from pixels, as an element the policy can judge like any label. */
|
|
14
16
|
const drawnText = (text, bounds) => ({ id: 'drawn', role: 'text', text, label: null, resource_id: null,
|
|
15
17
|
clickable: false, long_clickable: false, enabled: true, focused: false, scrollable: false, editable: false, checked: null, selected: false,
|
|
16
18
|
sensitive: false, bounds, dup_index: 0 });
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
const has = (b, p) => p.x >= b.x && p.x <= b.x + b.width && p.y >= b.y && p.y <= b.y + b.height;
|
|
20
|
+
const holds = (o, i) => o.x <= i.x && o.y <= i.y && o.x + o.width >= i.x + i.width && o.y + o.height >= i.y + i.height;
|
|
21
|
+
/**
|
|
22
|
+
* Where to tap an element: inside its on-screen part, off its own clickable children (a card's inner button),
|
|
23
|
+
* and off anything drawn over it (a floating toolbar, tab bar or sheet: a later element that isn't its ancestor).
|
|
24
|
+
* Throws when no point of it is visibly tappable, so a covered or off-screen target is never tapped "successfully".
|
|
25
|
+
*/
|
|
26
|
+
export const center = (e, screen, size) => {
|
|
27
|
+
const W = size?.width || Infinity, H = size?.height || Infinity;
|
|
21
28
|
const x1 = Math.max(e.bounds.x, 0), y1 = Math.max(e.bounds.y, 0);
|
|
22
|
-
const x2 = Math.
|
|
29
|
+
const x2 = Math.min(e.bounds.x + e.bounds.width, W), y2 = Math.min(e.bounds.y + e.bounds.height, H);
|
|
30
|
+
if (x2 <= x1 || y2 <= y1) {
|
|
31
|
+
if (size)
|
|
32
|
+
throw new MobiusError('ELEMENT_NOT_FOUND', `${e.id} is off screen`, { hint: 'Scroll it into view, observe, then tap.' });
|
|
33
|
+
return { x: Math.max(x1, Math.min(x2, (x1 + x2) / 2)), y: Math.max(y1, Math.min(y2, (y1 + y2) / 2)) };
|
|
34
|
+
}
|
|
23
35
|
const mid = { x: (x1 + x2) / 2, y: (y1 + y2) / 2 };
|
|
24
36
|
if (!screen)
|
|
25
37
|
return mid;
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
const kids = screen.filter(o => o !== e && o.id !== e.id && (o.clickable || o.editable) &&
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
const
|
|
38
|
+
const idx = screen.findIndex(o => o === e || o.id === e.id);
|
|
39
|
+
const area = (b) => b.width * b.height;
|
|
40
|
+
const kids = screen.filter(o => o !== e && o.id !== e.id && (o.clickable || o.editable) && holds(e.bounds, o.bounds) && area(o.bounds) < area(e.bounds));
|
|
41
|
+
// Drawn on top: later in tree order, overlapping, neither its ancestor nor its descendant, and not a decoration
|
|
42
|
+
// that passes touches through (plain text/images inside the overlap still block a tap on what's below).
|
|
43
|
+
const over = idx < 0 ? [] : screen.slice(idx + 1).filter(o => !holds(o.bounds, e.bounds) && !holds(e.bounds, o.bounds)
|
|
44
|
+
&& (o.clickable || o.editable || o.scrollable || /toolbar|tab ?bar|navigation bar|sheet/i.test(`${o.label ?? ''} ${o.text ?? ''}`)));
|
|
45
|
+
const blocked = (p) => kids.some(k => has(k.bounds, p)) || over.some(o => has(o.bounds, p));
|
|
32
46
|
const at = (fx, fy) => ({ x: x1 + (x2 - x1) * fx, y: y1 + (y2 - y1) * fy });
|
|
33
47
|
for (const [fx, fy] of [[0.5, 0.5], [0.5, 0.12], [0.12, 0.5], [0.88, 0.5], [0.5, 0.88], [0.12, 0.12], [0.88, 0.12], [0.12, 0.88], [0.88, 0.88], [0.3, 0.12], [0.7, 0.12]]) {
|
|
34
48
|
const p = at(fx, fy);
|
|
35
|
-
if (!
|
|
49
|
+
if (!blocked(p))
|
|
36
50
|
return p;
|
|
37
51
|
}
|
|
38
|
-
|
|
52
|
+
if (over.length && [[0.5, 0.5], [0.12, 0.12], [0.88, 0.88]].every(([fx, fy]) => over.some(o => has(o.bounds, at(fx, fy))))) {
|
|
53
|
+
throw new MobiusError('ELEMENT_NOT_FOUND', `${e.id} is covered by ${over.map(o => o.label || o.text || o.role).slice(0, 2).join(', ')}`, { hint: 'Scroll it into view, observe, then tap.' });
|
|
54
|
+
}
|
|
55
|
+
return mid; // fully covered by its own children: the children are the target in practice
|
|
39
56
|
};
|
|
40
57
|
function withTimeout(p, ms, what) {
|
|
41
58
|
let timer;
|
|
@@ -62,6 +79,8 @@ export function expectationMet(obs, ex) {
|
|
|
62
79
|
export class DeviceSession {
|
|
63
80
|
connector;
|
|
64
81
|
store;
|
|
82
|
+
/** The observation last shown to the agent (observe / list / snapshot): short refs like @e5 point into it, not into internal polls. */
|
|
83
|
+
shown = null;
|
|
65
84
|
audit;
|
|
66
85
|
policy;
|
|
67
86
|
approver;
|
|
@@ -86,7 +105,7 @@ export class DeviceSession {
|
|
|
86
105
|
await this.observe();
|
|
87
106
|
const latest = this.store.latest();
|
|
88
107
|
let target = guard.targetId ? this.store.resolve(guard.targetId) : guard.point ? this.elementAt(guard.point.x, guard.point.y) : null;
|
|
89
|
-
const p = guard.point ?? (target ? center(target, latest?.elements) : null);
|
|
108
|
+
const p = guard.point ?? (target ? center(target, latest?.elements, latest?.screen) : null);
|
|
90
109
|
const under = p ? this.elementsAt(p.x, p.y) : [];
|
|
91
110
|
// Pixels, not just labels: on a screen showing an amount due, read the text actually drawn where the finger lands
|
|
92
111
|
// (OCR), so a Pay button can't hide behind an accessibility label that says "Close".
|
|
@@ -159,13 +178,18 @@ export class DeviceSession {
|
|
|
159
178
|
this.store.add(obs, fps);
|
|
160
179
|
return obs;
|
|
161
180
|
}
|
|
162
|
-
/**
|
|
163
|
-
|
|
181
|
+
/**
|
|
182
|
+
* Re-observe until two consecutive screens hash the same (UI settled), bounded. State-based, no fixed sleeps.
|
|
183
|
+
* Two identical frames of the *pre-action* screen only count after a grace window: a tap that launches an
|
|
184
|
+
* activity takes a beat before the window changes, and settling early reports the old screen as the result.
|
|
185
|
+
*/
|
|
186
|
+
async settle(deadline, preHash = null) {
|
|
187
|
+
const graceEnd = Date.now() + SETTLE_GRACE_MS;
|
|
164
188
|
let prev = await this.observe();
|
|
165
189
|
while (Date.now() < deadline) {
|
|
166
190
|
await new Promise(r => setTimeout(r, POLL_MS));
|
|
167
191
|
const next = await this.observe();
|
|
168
|
-
if (next.screen_hash === prev.screen_hash)
|
|
192
|
+
if (next.screen_hash === prev.screen_hash && (next.screen_hash !== preHash || Date.now() >= graceEnd))
|
|
169
193
|
return next;
|
|
170
194
|
prev = next;
|
|
171
195
|
}
|
|
@@ -199,7 +223,7 @@ export class DeviceSession {
|
|
|
199
223
|
const el = this.store.resolve(element_id);
|
|
200
224
|
if (!el.enabled)
|
|
201
225
|
throw new MobiusError('ELEMENT_NOT_FOUND', `${element_id} is disabled`, { hint: 'Wait for it to enable or pick another element.' });
|
|
202
|
-
return { ...center(el, this.store.latest()?.elements), el };
|
|
226
|
+
return { ...center(el, this.store.latest()?.elements, this.store.latest()?.screen), el };
|
|
203
227
|
}
|
|
204
228
|
if (x !== undefined && y !== undefined) {
|
|
205
229
|
const s = this.store.latest()?.screen;
|
|
@@ -235,7 +259,7 @@ export class DeviceSession {
|
|
|
235
259
|
const message = await withTimeout(exec(target), opts.timeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS, tool);
|
|
236
260
|
const verifying = !!opts.expect || !!opts.check;
|
|
237
261
|
const deadline = Date.now() + (verifying ? opts.verifyMs ?? DEFAULT_EXPECT_TIMEOUT_MS : 2_500);
|
|
238
|
-
let obs = await this.settle(Math.min(deadline, Date.now() + 2_500));
|
|
262
|
+
let obs = await this.settle(Math.min(deadline, Date.now() + 2_500), preHash);
|
|
239
263
|
let verified = null;
|
|
240
264
|
let why = '';
|
|
241
265
|
const evaluate = (o) => {
|
package/dist/interface/mcp.js
CHANGED
|
@@ -116,6 +116,7 @@ export function createServer(mobius) {
|
|
|
116
116
|
const obs = await s.observe();
|
|
117
117
|
if (s.isBlockedApp(obs.foreground_app))
|
|
118
118
|
return json(blockedView(obs));
|
|
119
|
+
s.shown = obs.observation_id;
|
|
119
120
|
const out = json(compactObservation(obs));
|
|
120
121
|
if (!screenshot)
|
|
121
122
|
return out;
|
|
@@ -9,6 +9,7 @@ import { driverPath } from "../execution/drivers/mobiusagent.js";
|
|
|
9
9
|
import { z } from 'zod';
|
|
10
10
|
import { MobiusError } from "../execution/connector.js";
|
|
11
11
|
import { scrub } from "../execution/observe.js";
|
|
12
|
+
import { center } from "../execution/session.js";
|
|
12
13
|
const device = z.string().describe('The device identifier. Use mobius_list_available_devices to find which devices are available.');
|
|
13
14
|
const xy = {
|
|
14
15
|
x: z.coerce.number().min(0).optional().describe('x in screen coordinates. Required unless ref is given.'),
|
|
@@ -29,15 +30,18 @@ export function registerMobiusTools(server, handlers, mobius, h) {
|
|
|
29
30
|
if (aliases)
|
|
30
31
|
server.registerTool(`mobile_${name}`, cfg, cb);
|
|
31
32
|
};
|
|
32
|
-
/**
|
|
33
|
+
/**
|
|
34
|
+
* "@e5" (short ref) → the id on the observation the agent last saw (list_elements / snapshot), so a ref the agent
|
|
35
|
+
* read is resolved by identity (or refused as stale) after the screen moves, never re-pointed at whatever is e5 now.
|
|
36
|
+
*/
|
|
33
37
|
const elementId = async (dev, ref) => {
|
|
34
38
|
if (!ref)
|
|
35
39
|
return undefined;
|
|
36
40
|
if (!ref.startsWith('@'))
|
|
37
41
|
return ref;
|
|
38
42
|
const s = await mobius.session(dev);
|
|
39
|
-
const
|
|
40
|
-
return `${
|
|
43
|
+
const shown = s.shown ?? s.store.latest()?.observation_id ?? (await s.observe()).observation_id;
|
|
44
|
+
return `${shown}:${ref.slice(1)}`;
|
|
41
45
|
};
|
|
42
46
|
/** A driver operation beyond the core set, through the same pipeline (gate + audit) as every other action. */
|
|
43
47
|
const extra = async (dev, tool, op, args, opts) => {
|
|
@@ -101,14 +105,49 @@ export function registerMobiusTools(server, handlers, mobius, h) {
|
|
|
101
105
|
const hay = [e.text, e.label].filter(Boolean).map(v => norm(String(v)));
|
|
102
106
|
return exact ? hay.some(h => h === norm(q)) : hay.some(h => h.includes(norm(q)));
|
|
103
107
|
};
|
|
108
|
+
/** Visibly tappable: on screen and not under a floating bar (the same rule the tap itself applies). */
|
|
109
|
+
const tappable = (e, o) => { try {
|
|
110
|
+
center(e, o.elements, o.screen);
|
|
111
|
+
return true;
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
return false;
|
|
115
|
+
} };
|
|
116
|
+
/** Scroll until an element showing `text` is tappable, or the list stops moving. */
|
|
117
|
+
const scrollTo = async (device, text, direction, max) => {
|
|
118
|
+
const s = await mobius.session(device);
|
|
119
|
+
let prev = '';
|
|
120
|
+
for (let i = 0; i <= max; i++) {
|
|
121
|
+
const o = await s.observe();
|
|
122
|
+
const el = o.elements.find(e => matches(e, text, false) && tappable(e, o));
|
|
123
|
+
if (el) {
|
|
124
|
+
s.shown = o.observation_id;
|
|
125
|
+
return { ...el, swipes: i };
|
|
126
|
+
}
|
|
127
|
+
if (o.screen_hash === prev)
|
|
128
|
+
return null; // end of the list
|
|
129
|
+
prev = o.screen_hash;
|
|
130
|
+
const r = await call('scroll', { device_id: device, direction });
|
|
131
|
+
if (r.isError)
|
|
132
|
+
throw new MobiusError(JSON.parse(r.content[0].text).error?.code ?? 'INTERNAL', JSON.parse(r.content[0].text).message ?? 'scroll failed');
|
|
133
|
+
}
|
|
134
|
+
return null;
|
|
135
|
+
};
|
|
104
136
|
tool('tap_text', 'Tap Text', 'Find the element showing this text (or accessibility label) on the current screen and tap it. Prefers buttons and clickable elements. Saves an observe + pick round-trip.', { device, text: z.string().min(1), exact: z.boolean().optional(), index: z.number().int().min(0).optional().describe('Which match, if several (default 0).') }, async ({ device, text, exact, index }) => {
|
|
105
137
|
const s = await mobius.session(device);
|
|
106
138
|
const o = await s.observe();
|
|
107
139
|
const hits = o.elements.filter(e => matches(e, text, !!exact));
|
|
108
140
|
const ranked = [...hits.filter(e => e.clickable || e.role === 'button'), ...hits.filter(e => !(e.clickable || e.role === 'button'))];
|
|
109
141
|
const el = ranked[index ?? 0];
|
|
110
|
-
if (el)
|
|
142
|
+
if (el && tappable(el, o))
|
|
111
143
|
return call('tap', { device_id: device, element_id: el.id });
|
|
144
|
+
if (el) {
|
|
145
|
+
// On screen in the tree but off its edge or under a floating bar: bring it into view, then tap the fresh match.
|
|
146
|
+
const found = await scrollTo(device, text, el.bounds.y > o.screen.height / 2 ? 'down' : 'up', 5);
|
|
147
|
+
if (found)
|
|
148
|
+
return call('tap', { device_id: device, element_id: found.id });
|
|
149
|
+
throw new MobiusError('ELEMENT_NOT_FOUND', `"${text}" is on this screen but covered or off its edge, and scrolling did not reveal it`, { hint: 'Scroll manually, observe, then tap.' });
|
|
150
|
+
}
|
|
112
151
|
// Not in the accessibility tree: read what's drawn on screen (OCR) and tap it by position. Coordinate taps go
|
|
113
152
|
// through the same gate (and count as unidentified targets where policy asks).
|
|
114
153
|
const ocr = s.connector.extra ? await s.connector.extra('ocr', {}).catch(() => null) : null;
|
|
@@ -119,20 +158,9 @@ export function registerMobiusTools(server, handlers, mobius, h) {
|
|
|
119
158
|
return call('tap', { device_id: device, x: x + w / 2, y: y + h / 2 });
|
|
120
159
|
});
|
|
121
160
|
tool('scroll_until_visible', 'Scroll Until Visible', 'Scroll until an element showing this text is on screen (or the list stops moving). Returns its ref.', { device, text: z.string().min(1), direction: z.enum(['up', 'down', 'left', 'right']).optional(), max_swipes: z.number().int().min(1).max(30).optional() }, async ({ device, text, direction, max_swipes }) => {
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
const o = await s.observe();
|
|
126
|
-
const el = o.elements.find(e => matches(e, text, false) && e.bounds.width > 0 && e.bounds.y >= 0 && e.bounds.y + e.bounds.height <= o.screen.height);
|
|
127
|
-
if (el)
|
|
128
|
-
return json({ found: true, ref: el.id, swipes: i, element: { role: el.role, text: el.text, label: el.label, box: [el.bounds.x, el.bounds.y, el.bounds.width, el.bounds.height] } });
|
|
129
|
-
if (o.screen_hash === prev)
|
|
130
|
-
break; // end of the list
|
|
131
|
-
prev = o.screen_hash;
|
|
132
|
-
const r = await call('scroll', { device_id: device, direction: direction ?? 'down' });
|
|
133
|
-
if (r.isError)
|
|
134
|
-
return r;
|
|
135
|
-
}
|
|
161
|
+
const el = await scrollTo(device, text, direction ?? 'down', max_swipes ?? 10);
|
|
162
|
+
if (el)
|
|
163
|
+
return json({ found: true, ref: el.id, swipes: el.swipes, element: { role: el.role, text: el.text, label: el.label, box: [el.bounds.x, el.bounds.y, el.bounds.width, el.bounds.height] } });
|
|
136
164
|
return json({ found: false, hint: `"${text}" did not appear; try the other direction or check the screen.` });
|
|
137
165
|
}, true);
|
|
138
166
|
tool('wait_for', 'Wait For', 'Wait until text appears, disappears, or an app is in the foreground (polls the screen, no fixed sleeps).', { device, text_visible: z.string().optional(), text_gone: z.string().optional(), app: z.string().optional(), timeout_ms: z.number().int().min(500).max(120_000).optional() }, ({ device, ...rest }) => call('wait_for', { device_id: device, ...rest }), true);
|
|
@@ -181,7 +209,18 @@ export function registerMobiusTools(server, handlers, mobius, h) {
|
|
|
181
209
|
drv('performance_report', 'Performance Report', 'Full performance report for an app: cold/warm launch time, CPU, memory (with leak trend), threads, frame rate and jank, battery, crashes during the session. Optional budgets give pass/fail (e.g. {"launch_cold_ms":2000,"jank_percent":5,"memory_max_mb":400}). Writes an HTML report when `output` ends in .html.', { packageName: app, seconds: z.number().int().min(3).max(1800).optional(), budget: z.record(z.string(), z.number()).optional(), output: z.string().optional() }, 'perf_report', a => ({ app: a.packageName, seconds: a.seconds ?? 30, ...(a.budget ? { budget: a.budget } : {}), ...(a.output ? { output: a.output } : {}) }), { mutating: true, app: a => a.packageName });
|
|
182
210
|
drv('accessibility_audit', 'Accessibility Audit', 'Audit the current screen: unlabeled controls, touch targets below 44pt/48dp, duplicate labels, with a score.', {}, 'a11y_audit', () => ({}), { readOnly: true, scrub: true });
|
|
183
211
|
drv('screen_diff', 'Screen Diff', 'What changed on screen since the last snapshot or diff (added/removed elements). Read the change, not the whole tree.', {}, 'diff', () => ({}), { readOnly: true, scrub: true });
|
|
184
|
-
|
|
212
|
+
tool('snapshot', 'Text Snapshot', 'The screen as compact text, one element per line with refs (@e5) usable by click tools.', { device }, async ({ device }) => {
|
|
213
|
+
const s = await mobius.session(device);
|
|
214
|
+
const o = await s.observe();
|
|
215
|
+
s.shown = o.observation_id;
|
|
216
|
+
const lines = o.elements.map(e => {
|
|
217
|
+
const words = [e.text, e.label && e.label !== e.text ? e.label : ''].filter(Boolean).map(t => JSON.stringify(t)).join(' ');
|
|
218
|
+
const rid = e.resource_id ? ` #${e.resource_id.replace(/^[\w.]+:id\//, '')}` : '';
|
|
219
|
+
const flags = [e.clickable && 'clickable', e.editable && 'editable', e.scrollable && 'scrollable', !e.enabled && 'disabled', e.focused && 'focused', e.sensitive && 'sensitive'].filter(Boolean);
|
|
220
|
+
return `@${e.id.split(':')[1]} [${e.role}] ${words}${rid}${flags.length ? ` (${flags.join(', ')})` : ''}`.replace(/ +/g, ' ');
|
|
221
|
+
});
|
|
222
|
+
return json({ app: o.foreground_app, observation_id: o.observation_id, snapshot: lines.join('\n') });
|
|
223
|
+
}, true);
|
|
185
224
|
drv('wait_until_idle', 'Wait Until Idle', 'Wait until the screen stops changing.', { timeout_ms: z.number().int().min(200).max(60_000).optional() }, 'wait_idle', a => ({ timeout_ms: a.timeout_ms ?? 5000 }), { readOnly: true });
|
|
186
225
|
drv('read_screen_text', 'Read Screen Text (OCR)', 'Every piece of text drawn on screen with its box, including unlabeled buttons and images. Secrets are masked.', {}, 'ocr', () => ({}), { readOnly: true, scrub: true });
|
|
187
226
|
drv('set_dark_mode', 'Set Dark Mode', 'Turn dark mode on or off (Android, iOS simulators/devices).', { on: z.boolean() }, 'dark_mode', a => ({ on: a.on }), { mutating: true });
|
|
@@ -201,7 +240,8 @@ export function registerMobiusTools(server, handlers, mobius, h) {
|
|
|
201
240
|
const s = await mobius.session(a.device);
|
|
202
241
|
if ((await s.observe()).elements.some(e => e.sensitive))
|
|
203
242
|
throw new MobiusError('POLICY_BLOCKED', 'A password, PIN, OTP or card number is on screen, so no screenshot was taken');
|
|
204
|
-
const
|
|
243
|
+
const el = s.store.resolve((await elementId(a.device, a.ref)));
|
|
244
|
+
const { data } = await extra(a.device, 'element_screenshot', 'element_screenshot', { box: [el.bounds.x, el.bounds.y, el.bounds.width, el.bounds.height] }, { mutating: false, describe: 'Element screenshot' });
|
|
205
245
|
return { content: [{ type: 'image', data: data.data, mimeType: data.mimeType }] };
|
|
206
246
|
}, true);
|
|
207
247
|
drv('play_gps_route', 'Play GPS Route', 'Move the device along waypoints ("lat,lon;lat,lon") at a speed, for ride and delivery flows.', { points: z.string(), speed_kmh: z.number().min(1).max(300).optional(), interval_s: z.number().min(0.2).max(10).optional() }, 'gps_route', a => ({ points: a.points, speed_kmh: a.speed_kmh ?? 30, interval_s: a.interval_s ?? 1 }), { mutating: true });
|
package/ios-agent/setup.sh
CHANGED
|
@@ -15,12 +15,18 @@ cd "$DIR"
|
|
|
15
15
|
# Mobius fast touch (tap/swipe without WebDriverAgent's app lookup and settle wait). BSD-3 allows shipping a modified build.
|
|
16
16
|
PATCH="$HERE/wda-mobius-fast-touch.patch"
|
|
17
17
|
git apply --check "$PATCH" 2>/dev/null && git apply "$PATCH" && echo "applied Mobius fast-touch patch"
|
|
18
|
-
xcodebuild
|
|
18
|
+
# Run xcodebuild into a log and stop on failure: piping it through grep would hide a failed build, and a stale
|
|
19
|
+
# .xctestrun from an earlier build would then pass the final check.
|
|
20
|
+
build() { # build <log name> <xcodebuild args...>
|
|
21
|
+
log="$DIR/$1.log"; shift
|
|
22
|
+
if xcodebuild "$@" >"$log" 2>&1; then grep -E "\*\* " "$log" || true; else grep -E "error:|\*\* " "$log"; echo "xcodebuild failed (full log: $log)" >&2; exit 1; fi
|
|
23
|
+
}
|
|
24
|
+
build build-device build-for-testing -project WebDriverAgent.xcodeproj -scheme WebDriverAgentRunner \
|
|
19
25
|
-destination 'generic/platform=iOS' DEVELOPMENT_TEAM="$TEAM" CODE_SIGN_STYLE=Automatic \
|
|
20
26
|
PRODUCT_BUNDLE_IDENTIFIER="${MOBIUS_WDA_BUNDLE:-dev.mobius.wda}" \
|
|
21
|
-
-allowProvisioningUpdates -allowProvisioningDeviceRegistration -derivedDataPath "$DIR/build"
|
|
27
|
+
-allowProvisioningUpdates -allowProvisioningDeviceRegistration -derivedDataPath "$DIR/build"
|
|
22
28
|
"$HERE/brand.sh"
|
|
23
29
|
# Simulators: an unsigned simulator build (no team needed).
|
|
24
|
-
|
|
25
|
-
-destination 'generic/platform=iOS Simulator' -derivedDataPath "$DIR/build-sim" CODE_SIGNING_ALLOWED=NO
|
|
30
|
+
build build-sim build-for-testing -project WebDriverAgent.xcodeproj -scheme WebDriverAgentRunner \
|
|
31
|
+
-destination 'generic/platform=iOS Simulator' -derivedDataPath "$DIR/build-sim" CODE_SIGNING_ALLOWED=NO
|
|
26
32
|
ls "$DIR"/build/Build/Products/*.xctestrun >/dev/null && echo "WebDriverAgent ready in $DIR"
|
package/package.json
CHANGED
|
@@ -1,18 +1,47 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@anuraghq/mobius",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Let AI agents use a real phone, safely. MCP server + its own fast device driver for Android, iPhones and iOS simulators: 75 tools, approvals, policy, masking, signed audit, performance and accessibility reports.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
7
|
-
"bin": {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
"
|
|
7
|
+
"bin": {
|
|
8
|
+
"mobius": "dist/cli.js"
|
|
9
|
+
},
|
|
10
|
+
"files": [
|
|
11
|
+
"dist",
|
|
12
|
+
"bin",
|
|
13
|
+
"ios-agent",
|
|
14
|
+
"README.md",
|
|
15
|
+
"LICENSE"
|
|
16
|
+
],
|
|
17
|
+
"engines": {
|
|
18
|
+
"node": ">=22.5"
|
|
19
|
+
},
|
|
20
|
+
"keywords": [
|
|
21
|
+
"mcp",
|
|
22
|
+
"model-context-protocol",
|
|
23
|
+
"android",
|
|
24
|
+
"ios",
|
|
25
|
+
"iphone",
|
|
26
|
+
"simulator",
|
|
27
|
+
"emulator",
|
|
28
|
+
"mobile",
|
|
29
|
+
"ai-agent",
|
|
30
|
+
"agent-safety",
|
|
31
|
+
"claude",
|
|
32
|
+
"automation",
|
|
33
|
+
"testing",
|
|
34
|
+
"qa",
|
|
35
|
+
"performance",
|
|
36
|
+
"accessibility"
|
|
37
|
+
],
|
|
11
38
|
"scripts": {
|
|
12
39
|
"build": "rm -rf dist && tsc -p tsconfig.build.json",
|
|
13
40
|
"prepack": "npm run build && node scripts/prepack.mjs"
|
|
14
41
|
},
|
|
15
|
-
"publishConfig": {
|
|
42
|
+
"publishConfig": {
|
|
43
|
+
"access": "public"
|
|
44
|
+
},
|
|
16
45
|
"repository": "github:Anu-Code07/mobius",
|
|
17
46
|
"homepage": "https://github.com/Anu-Code07/mobius#readme",
|
|
18
47
|
"dependencies": {
|