@anuraghq/mobius 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +371 -2
- package/bin/mobiusagent-driver-darwin-arm64 +0 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +151 -0
- package/dist/core/protocol.d.ts +294 -0
- package/dist/core/protocol.js +82 -0
- package/dist/execution/connector.d.ts +82 -0
- package/dist/execution/connector.js +11 -0
- package/dist/execution/drivers/mobiusagent.d.ts +57 -0
- package/dist/execution/drivers/mobiusagent.js +165 -0
- package/dist/execution/drivers/nodes.d.ts +47 -0
- package/dist/execution/drivers/nodes.js +104 -0
- package/dist/execution/observe.d.ts +48 -0
- package/dist/execution/observe.js +244 -0
- package/dist/execution/session.d.ts +101 -0
- package/dist/execution/session.js +304 -0
- package/dist/infrastructure/plane.d.ts +25 -0
- package/dist/infrastructure/plane.js +16 -0
- package/dist/infrastructure/registry.d.ts +19 -0
- package/dist/infrastructure/registry.js +49 -0
- package/dist/interface/engine.d.ts +24 -0
- package/dist/interface/engine.js +38 -0
- package/dist/interface/http.d.ts +4 -0
- package/dist/interface/http.js +85 -0
- package/dist/interface/mcp.d.ts +21 -0
- package/dist/interface/mcp.js +377 -0
- package/dist/interface/mobius-tools.d.ts +10 -0
- package/dist/interface/mobius-tools.js +288 -0
- package/dist/observability/audit.d.ts +52 -0
- package/dist/observability/audit.js +98 -0
- package/dist/observability/signing.d.ts +37 -0
- package/dist/observability/signing.js +87 -0
- package/dist/policy/policy.d.ts +147 -0
- package/dist/policy/policy.js +347 -0
- package/ios-agent/brand.py +62 -0
- package/ios-agent/brand.sh +19 -0
- package/ios-agent/setup.sh +26 -0
- package/ios-agent/wda-mobius-fast-touch.patch +73 -0
- package/package.json +20 -4
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import type { ActionResult, Element, Expect, Observation } from '../core/protocol.ts';
|
|
2
|
+
import type { AuditLog } from '../observability/audit.ts';
|
|
3
|
+
import { type PlaneContext } from '../infrastructure/plane.ts';
|
|
4
|
+
import { type Connector } from './connector.ts';
|
|
5
|
+
import { ObservationStore } from './observe.ts';
|
|
6
|
+
import { type Approver, type Policy, type PolicyContext } from '../policy/policy.ts';
|
|
7
|
+
/** Policy hook for one action. `targetId` is re-resolved after approval (screen may have changed). */
|
|
8
|
+
export interface Guard {
|
|
9
|
+
ctx: Omit<PolicyContext, 'app' | 'target'>;
|
|
10
|
+
targetId?: string | undefined;
|
|
11
|
+
/** Coordinate actions: the point the finger lands on (resolved against the fresh screen). */
|
|
12
|
+
point?: {
|
|
13
|
+
x: number;
|
|
14
|
+
y: number;
|
|
15
|
+
} | undefined;
|
|
16
|
+
/** Human-readable action built from DEVICE state (element text), never from agent prose. */
|
|
17
|
+
describe: (target: Element | null) => string;
|
|
18
|
+
}
|
|
19
|
+
export declare const DEFAULT_ACTION_TIMEOUT_MS = 20000;
|
|
20
|
+
export declare const DEFAULT_EXPECT_TIMEOUT_MS = 6000;
|
|
21
|
+
/** Cold app launches (Flutter/RN on iOS) take longer than a tap to show; live run: Scapia was still on SpringBoard at 6 s. */
|
|
22
|
+
export declare const LAUNCH_EXPECT_TIMEOUT_MS = 20000;
|
|
23
|
+
/** Where to touch an element: the center of its on-screen part (an element partly above/left of the screen has negative
|
|
24
|
+
* bounds; its raw center can be off-screen). Given the screen, a point not covered by a smaller clickable element inside
|
|
25
|
+
* it: tapping a big card's center must not hit the item drawn there (live run: Swiggy's restaurant card opened a burger). */
|
|
26
|
+
export declare const center: (e: Element, screen?: readonly Element[]) => {
|
|
27
|
+
x: number;
|
|
28
|
+
y: number;
|
|
29
|
+
};
|
|
30
|
+
export declare function expectationMet(obs: Observation, ex: Expect): boolean;
|
|
31
|
+
/** One device: observation history + serialized action pipeline. */
|
|
32
|
+
export declare class DeviceSession {
|
|
33
|
+
readonly connector: Connector;
|
|
34
|
+
readonly store: ObservationStore;
|
|
35
|
+
private readonly audit;
|
|
36
|
+
private readonly policy;
|
|
37
|
+
private readonly approver;
|
|
38
|
+
private queue;
|
|
39
|
+
private recent;
|
|
40
|
+
private readonly plane;
|
|
41
|
+
private verdict;
|
|
42
|
+
constructor(connector: Connector, audit: AuditLog, policy: Policy, approver: Approver, plane?: PlaneContext, persistIds?: boolean);
|
|
43
|
+
isBlockedApp(app: string | null): boolean;
|
|
44
|
+
/** Policy + approval gate. Returns the (re-resolved) target to act on. Throws MobiusError to stop. */
|
|
45
|
+
private gate;
|
|
46
|
+
/** Serialize work per device: one action at a time, no interleaved taps. */
|
|
47
|
+
private exclusive;
|
|
48
|
+
observe(): Promise<Observation>;
|
|
49
|
+
/** Re-observe until two consecutive screens hash the same (UI settled), bounded. State-based, no fixed sleeps. */
|
|
50
|
+
private settle;
|
|
51
|
+
/** Every element containing the point on the latest screen (overlays, the button below them, its labels). */
|
|
52
|
+
elementsAt(x: number, y: number): Element[];
|
|
53
|
+
/** Smallest clickable/editable element containing the point on the latest screen. */
|
|
54
|
+
elementAt(x: number, y: number): Element | null;
|
|
55
|
+
/** After tapping a field: wait (state-based, ≤2.5 s) until an editable element at that point has focus. */
|
|
56
|
+
waitForFocus(x: number, y: number): Promise<void>;
|
|
57
|
+
focusedField(): Element | null;
|
|
58
|
+
resolveTarget(element_id: string | undefined, x: number | undefined, y: number | undefined): {
|
|
59
|
+
el: {
|
|
60
|
+
id: string;
|
|
61
|
+
role: "button" | "textbox" | "checkbox" | "switch" | "radio" | "image" | "text" | "list" | "tab" | "view";
|
|
62
|
+
text: string | null;
|
|
63
|
+
label: string | null;
|
|
64
|
+
resource_id: string | null;
|
|
65
|
+
clickable: boolean;
|
|
66
|
+
long_clickable: boolean;
|
|
67
|
+
enabled: boolean;
|
|
68
|
+
focused: boolean;
|
|
69
|
+
scrollable: boolean;
|
|
70
|
+
editable: boolean;
|
|
71
|
+
checked: boolean | null;
|
|
72
|
+
selected: boolean;
|
|
73
|
+
sensitive: boolean;
|
|
74
|
+
bounds: {
|
|
75
|
+
x: number;
|
|
76
|
+
y: number;
|
|
77
|
+
width: number;
|
|
78
|
+
height: number;
|
|
79
|
+
};
|
|
80
|
+
dup_index: number;
|
|
81
|
+
};
|
|
82
|
+
x: number;
|
|
83
|
+
y: number;
|
|
84
|
+
} | {
|
|
85
|
+
x: number;
|
|
86
|
+
y: number;
|
|
87
|
+
el: null;
|
|
88
|
+
};
|
|
89
|
+
/**
|
|
90
|
+
* The single action pipeline: execute → settle → verify `expect` → audit.
|
|
91
|
+
* Never retries: the agent decides after seeing the fresh observation.
|
|
92
|
+
*/
|
|
93
|
+
run(tool: string, auditParams: Record<string, unknown>, exec: (target: Element | null) => Promise<string>, opts?: {
|
|
94
|
+
expect?: Expect | undefined;
|
|
95
|
+
timeoutMs?: number | undefined;
|
|
96
|
+
guard?: Guard;
|
|
97
|
+
verifyMs?: number;
|
|
98
|
+
/** Built-in post-condition (e.g. typed text landed); combined with `expect`. Returns a failure reason or null. */
|
|
99
|
+
check?: (obs: Observation) => string | null;
|
|
100
|
+
}): Promise<ActionResult>;
|
|
101
|
+
}
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { isRevoked, killSwitchPath, newPlaneContext } from "../infrastructure/plane.js";
|
|
4
|
+
import { MobiusError } from "./connector.js";
|
|
5
|
+
import { buildObservation, ObservationStore } from "./observe.js";
|
|
6
|
+
import { moneyOnScreen, newApprovalId } from "../policy/policy.js";
|
|
7
|
+
export const DEFAULT_ACTION_TIMEOUT_MS = 20_000;
|
|
8
|
+
export const DEFAULT_EXPECT_TIMEOUT_MS = 6_000;
|
|
9
|
+
/** Cold app launches (Flutter/RN on iOS) take longer than a tap to show; live run: Scapia was still on SpringBoard at 6 s. */
|
|
10
|
+
export const LAUNCH_EXPECT_TIMEOUT_MS = 20_000;
|
|
11
|
+
/** Pause between re-reads while waiting for the screen. */
|
|
12
|
+
const POLL_MS = 100;
|
|
13
|
+
/** Text read from pixels, as an element the policy can judge like any label. */
|
|
14
|
+
const drawnText = (text, bounds) => ({ id: 'drawn', role: 'text', text, label: null, resource_id: null,
|
|
15
|
+
clickable: false, long_clickable: false, enabled: true, focused: false, scrollable: false, editable: false, checked: null, selected: false,
|
|
16
|
+
sensitive: false, bounds, dup_index: 0 });
|
|
17
|
+
/** Where to touch an element: the center of its on-screen part (an element partly above/left of the screen has negative
|
|
18
|
+
* bounds; its raw center can be off-screen). Given the screen, a point not covered by a smaller clickable element inside
|
|
19
|
+
* it: tapping a big card's center must not hit the item drawn there (live run: Swiggy's restaurant card opened a burger). */
|
|
20
|
+
export const center = (e, screen) => {
|
|
21
|
+
const x1 = Math.max(e.bounds.x, 0), y1 = Math.max(e.bounds.y, 0);
|
|
22
|
+
const x2 = Math.max(e.bounds.x + e.bounds.width, x1), y2 = Math.max(e.bounds.y + e.bounds.height, y1);
|
|
23
|
+
const mid = { x: (x1 + x2) / 2, y: (y1 + y2) / 2 };
|
|
24
|
+
if (!screen)
|
|
25
|
+
return mid;
|
|
26
|
+
const inside = (o) => o.bounds.x >= e.bounds.x && o.bounds.y >= e.bounds.y && o.bounds.x + o.bounds.width <= e.bounds.x + e.bounds.width
|
|
27
|
+
&& o.bounds.y + o.bounds.height <= e.bounds.y + e.bounds.height && o.bounds.width * o.bounds.height < e.bounds.width * e.bounds.height;
|
|
28
|
+
const kids = screen.filter(o => o !== e && o.id !== e.id && (o.clickable || o.editable) && inside(o));
|
|
29
|
+
if (!kids.length)
|
|
30
|
+
return mid;
|
|
31
|
+
const hit = (p) => kids.some(k => p.x >= k.bounds.x && p.x <= k.bounds.x + k.bounds.width && p.y >= k.bounds.y && p.y <= k.bounds.y + k.bounds.height);
|
|
32
|
+
const at = (fx, fy) => ({ x: x1 + (x2 - x1) * fx, y: y1 + (y2 - y1) * fy });
|
|
33
|
+
for (const [fx, fy] of [[0.5, 0.5], [0.5, 0.12], [0.12, 0.5], [0.88, 0.5], [0.5, 0.88], [0.12, 0.12], [0.88, 0.12], [0.12, 0.88], [0.88, 0.88], [0.3, 0.12], [0.7, 0.12]]) {
|
|
34
|
+
const p = at(fx, fy);
|
|
35
|
+
if (!hit(p))
|
|
36
|
+
return p;
|
|
37
|
+
}
|
|
38
|
+
return mid; // fully covered: the children are the target in practice
|
|
39
|
+
};
|
|
40
|
+
function withTimeout(p, ms, what) {
|
|
41
|
+
let timer;
|
|
42
|
+
return Promise.race([
|
|
43
|
+
p.finally(() => clearTimeout(timer)),
|
|
44
|
+
new Promise((_, rej) => {
|
|
45
|
+
timer = setTimeout(() => rej(new MobiusError('OUTCOME_UNKNOWN', `${what} did not finish within ${ms} ms`, {
|
|
46
|
+
hint: 'The action may or may not have happened. Call observe before retrying.',
|
|
47
|
+
})), ms);
|
|
48
|
+
}),
|
|
49
|
+
]);
|
|
50
|
+
}
|
|
51
|
+
export function expectationMet(obs, ex) {
|
|
52
|
+
const has = (t) => obs.elements.some(e => (e.text ?? '').includes(t) || (e.label ?? '').includes(t));
|
|
53
|
+
if (ex.text_visible !== undefined && !has(ex.text_visible))
|
|
54
|
+
return false;
|
|
55
|
+
if (ex.app !== undefined && obs.foreground_app !== ex.app)
|
|
56
|
+
return false;
|
|
57
|
+
if (ex.element_gone !== undefined && obs.elements.some(e => e.id === ex.element_gone))
|
|
58
|
+
return false;
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
/** One device: observation history + serialized action pipeline. */
|
|
62
|
+
export class DeviceSession {
|
|
63
|
+
connector;
|
|
64
|
+
store;
|
|
65
|
+
audit;
|
|
66
|
+
policy;
|
|
67
|
+
approver;
|
|
68
|
+
queue = Promise.resolve();
|
|
69
|
+
recent = [];
|
|
70
|
+
plane;
|
|
71
|
+
verdict = null;
|
|
72
|
+
constructor(connector, audit, policy, approver, plane = newPlaneContext(), persistIds = false) {
|
|
73
|
+
this.store = new ObservationStore(persistIds ? connector.deviceId : undefined);
|
|
74
|
+
this.connector = connector;
|
|
75
|
+
this.plane = plane;
|
|
76
|
+
this.audit = audit;
|
|
77
|
+
this.policy = policy;
|
|
78
|
+
this.approver = approver;
|
|
79
|
+
}
|
|
80
|
+
isBlockedApp(app) { return this.policy.appBlocked(app); }
|
|
81
|
+
/** Policy + approval gate. Returns the (re-resolved) target to act on. Throws MobiusError to stop. */
|
|
82
|
+
async gate(tool, guard) {
|
|
83
|
+
// Time of check = time of use: the app may have changed the screen since the agent read it ("Add to cart" becomes
|
|
84
|
+
// "Pay ₹999"). Re-read now and decide on what the finger will actually hit. A changed target is refused as stale.
|
|
85
|
+
if (guard.ctx.mutating && (guard.targetId || guard.point))
|
|
86
|
+
await this.observe();
|
|
87
|
+
const latest = this.store.latest();
|
|
88
|
+
let target = guard.targetId ? this.store.resolve(guard.targetId) : guard.point ? this.elementAt(guard.point.x, guard.point.y) : null;
|
|
89
|
+
const p = guard.point ?? (target ? center(target, latest?.elements) : null);
|
|
90
|
+
const under = p ? this.elementsAt(p.x, p.y) : [];
|
|
91
|
+
// Pixels, not just labels: on a screen showing an amount due, read the text actually drawn where the finger lands
|
|
92
|
+
// (OCR), so a Pay button can't hide behind an accessibility label that says "Close".
|
|
93
|
+
if (guard.ctx.mutating && p && this.connector.extra && moneyOnScreen(latest?.elements)) {
|
|
94
|
+
const drawn = await this.connector.extra('ocr', {}).catch(() => null);
|
|
95
|
+
const b = target?.bounds;
|
|
96
|
+
for (const t of drawn?.text ?? []) {
|
|
97
|
+
const [x = 0, y = 0, w = 0, h = 0] = (t.box ?? []);
|
|
98
|
+
const cx = x + w / 2, cy = y + h / 2;
|
|
99
|
+
const hit = b ? cx >= b.x && cx <= b.x + b.width && cy >= b.y && cy <= b.y + b.height
|
|
100
|
+
: p.x >= x - 16 && p.x <= x + w + 16 && p.y >= y - 16 && p.y <= y + h + 16;
|
|
101
|
+
if (hit && t.text)
|
|
102
|
+
under.push(drawnText(String(t.text), { x, y, width: w, height: h }));
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (guard.ctx.mutating) {
|
|
106
|
+
if (existsSync(killSwitchPath()))
|
|
107
|
+
throw new MobiusError('POLICY_BLOCKED', 'Kill switch engaged: all agent actions are stopped', { hint: 'A human ran `mobius stop`. Stop and tell the user.' });
|
|
108
|
+
const pl = this.plane;
|
|
109
|
+
if (pl.actions >= pl.max_actions)
|
|
110
|
+
throw new MobiusError('POLICY_BLOCKED', `Action quota reached (${pl.max_actions} per session)`, { hint: 'Stop and report progress to the user.' });
|
|
111
|
+
const ses = pl.session();
|
|
112
|
+
if (ses && 'invalid' in ses)
|
|
113
|
+
throw new MobiusError('POLICY_BLOCKED', `Agent session token rejected: ${ses.invalid}`, { hint: 'Ask the operator for a fresh session token.' });
|
|
114
|
+
if (!ses && pl.require_session)
|
|
115
|
+
throw new MobiusError('POLICY_BLOCKED', 'This Mobius requires a signed agent session', { hint: 'Ask the operator for a session token (mobius session issue).' });
|
|
116
|
+
if (ses && isRevoked(ses.jti))
|
|
117
|
+
throw new MobiusError('POLICY_BLOCKED', 'Agent session revoked by the operator', { hint: 'Stop and tell the user.' });
|
|
118
|
+
if (ses && ses.exp <= Date.now() / 1000)
|
|
119
|
+
throw new MobiusError('POLICY_BLOCKED', 'Agent session expired', { hint: 'Ask the operator for a fresh session token.' });
|
|
120
|
+
if (ses?.policy_version && ses.policy_version !== this.policy.config.version) {
|
|
121
|
+
throw new MobiusError('POLICY_BLOCKED', `Session was issued for policy ${ses.policy_version}, but Mobius runs ${this.policy.config.version}`, { hint: 'The operator must re-issue the session for the current policy.' });
|
|
122
|
+
}
|
|
123
|
+
if (pl.max_minutes && Date.now() - pl.started > pl.max_minutes * 60_000)
|
|
124
|
+
throw new MobiusError('POLICY_BLOCKED', `Session time limit reached (${pl.max_minutes} min)`, { hint: 'Stop and report progress to the user.' });
|
|
125
|
+
}
|
|
126
|
+
const v = this.policy.evaluate({ ...guard.ctx, tool, app: latest?.foreground_app ?? null, target, screen: latest?.elements, under });
|
|
127
|
+
this.verdict = v;
|
|
128
|
+
const { decision, reason, action_class } = v;
|
|
129
|
+
if (decision === 'blocked')
|
|
130
|
+
throw new MobiusError('POLICY_BLOCKED', `Blocked by policy: ${reason}`, { hint: 'Ask the user to do this step themselves.' });
|
|
131
|
+
if (decision === 'allow')
|
|
132
|
+
return target;
|
|
133
|
+
const req = { id: newApprovalId(), device_id: this.connector.deviceId, app: latest?.foreground_app ?? null, tool, summary: guard.describe(target), reason, action_class, virtual_device: this.connector.virtual };
|
|
134
|
+
const outcome = await this.approver.request(req, this.policy.config.approval_timeout_s);
|
|
135
|
+
if (outcome === 'rejected')
|
|
136
|
+
throw new MobiusError('APPROVAL_REJECTED', `User rejected: ${req.summary}`, { hint: 'Do not retry. Ask the user what they want instead.' });
|
|
137
|
+
if (outcome === 'expired')
|
|
138
|
+
throw new MobiusError('APPROVAL_EXPIRED', `No decision within ${this.policy.config.approval_timeout_s}s: ${req.summary}`, { retryable: true });
|
|
139
|
+
// Time passed while the human decided: confirm the approved target is still the one on screen.
|
|
140
|
+
if (guard.targetId) {
|
|
141
|
+
await this.observe();
|
|
142
|
+
target = this.store.resolve(guard.targetId);
|
|
143
|
+
if (guard.describe(target) !== req.summary) {
|
|
144
|
+
throw new MobiusError('STALE_ELEMENT', 'Screen changed after approval; the approved target is no longer the same', { retryable: true, hint: 'Observe again; a new approval will be requested.' });
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return target;
|
|
148
|
+
}
|
|
149
|
+
/** Serialize work per device: one action at a time, no interleaved taps. */
|
|
150
|
+
exclusive(fn) {
|
|
151
|
+
const next = this.queue.then(fn, fn);
|
|
152
|
+
this.queue = next.catch(() => undefined);
|
|
153
|
+
return next;
|
|
154
|
+
}
|
|
155
|
+
async observe() {
|
|
156
|
+
const screen = await this.connector.dump();
|
|
157
|
+
const fw = screen.foregroundApp ? await this.connector.framework(screen.foregroundApp) : 'unknown';
|
|
158
|
+
const { obs, fps } = buildObservation(screen, this.connector.deviceId, this.connector.platform, this.store.nextId(), fw);
|
|
159
|
+
this.store.add(obs, fps);
|
|
160
|
+
return obs;
|
|
161
|
+
}
|
|
162
|
+
/** Re-observe until two consecutive screens hash the same (UI settled), bounded. State-based, no fixed sleeps. */
|
|
163
|
+
async settle(deadline) {
|
|
164
|
+
let prev = await this.observe();
|
|
165
|
+
while (Date.now() < deadline) {
|
|
166
|
+
await new Promise(r => setTimeout(r, POLL_MS));
|
|
167
|
+
const next = await this.observe();
|
|
168
|
+
if (next.screen_hash === prev.screen_hash)
|
|
169
|
+
return next;
|
|
170
|
+
prev = next;
|
|
171
|
+
}
|
|
172
|
+
return prev;
|
|
173
|
+
}
|
|
174
|
+
/** Every element containing the point on the latest screen (overlays, the button below them, its labels). */
|
|
175
|
+
elementsAt(x, y) {
|
|
176
|
+
return (this.store.latest()?.elements ?? []).filter(e => x >= e.bounds.x && x <= e.bounds.x + e.bounds.width && y >= e.bounds.y && y <= e.bounds.y + e.bounds.height);
|
|
177
|
+
}
|
|
178
|
+
/** Smallest clickable/editable element containing the point on the latest screen. */
|
|
179
|
+
elementAt(x, y) {
|
|
180
|
+
const hits = (this.store.latest()?.elements ?? []).filter(e => (e.clickable || e.editable || e.long_clickable)
|
|
181
|
+
&& x >= e.bounds.x && x <= e.bounds.x + e.bounds.width && y >= e.bounds.y && y <= e.bounds.y + e.bounds.height);
|
|
182
|
+
return hits.sort((a, b) => a.bounds.width * a.bounds.height - b.bounds.width * b.bounds.height)[0] ?? null;
|
|
183
|
+
}
|
|
184
|
+
/** After tapping a field: wait (state-based, ≤2.5 s) until an editable element at that point has focus. */
|
|
185
|
+
async waitForFocus(x, y) {
|
|
186
|
+
const until = Date.now() + 2_500;
|
|
187
|
+
while (Date.now() < until) {
|
|
188
|
+
const o = await this.observe();
|
|
189
|
+
const f = o.elements.find(e => e.editable && e.focused && x >= e.bounds.x && x <= e.bounds.x + e.bounds.width && y >= e.bounds.y && y <= e.bounds.y + e.bounds.height);
|
|
190
|
+
if (f)
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
focusedField() {
|
|
195
|
+
return this.store.latest()?.elements.find(e => e.editable && e.focused) ?? null;
|
|
196
|
+
}
|
|
197
|
+
resolveTarget(element_id, x, y) {
|
|
198
|
+
if (element_id) {
|
|
199
|
+
const el = this.store.resolve(element_id);
|
|
200
|
+
if (!el.enabled)
|
|
201
|
+
throw new MobiusError('ELEMENT_NOT_FOUND', `${element_id} is disabled`, { hint: 'Wait for it to enable or pick another element.' });
|
|
202
|
+
return { ...center(el, this.store.latest()?.elements), el };
|
|
203
|
+
}
|
|
204
|
+
if (x !== undefined && y !== undefined) {
|
|
205
|
+
const s = this.store.latest()?.screen;
|
|
206
|
+
if (s && s.width && (x < 0 || y < 0 || x > s.width || y > s.height)) {
|
|
207
|
+
throw new MobiusError('INVALID_INPUT', `(${x}, ${y}) is outside the ${s.width}x${s.height} screen`);
|
|
208
|
+
}
|
|
209
|
+
return { x, y, el: null };
|
|
210
|
+
}
|
|
211
|
+
throw new MobiusError('INVALID_INPUT', 'Give element_id, or both x and y');
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* The single action pipeline: execute → settle → verify `expect` → audit.
|
|
215
|
+
* Never retries: the agent decides after seeing the fresh observation.
|
|
216
|
+
*/
|
|
217
|
+
run(tool, auditParams, exec, opts = {}) {
|
|
218
|
+
return this.exclusive(async () => {
|
|
219
|
+
const action_id = `act_${randomUUID().slice(0, 8)}`;
|
|
220
|
+
const t0 = Date.now();
|
|
221
|
+
let preHash = this.store.latest()?.screen_hash ?? null;
|
|
222
|
+
const loopKey = `${tool}|${JSON.stringify(auditParams)}|${preHash}`;
|
|
223
|
+
let result;
|
|
224
|
+
try {
|
|
225
|
+
// Loop guard: same action on the same unchanged screen 3 times in a row.
|
|
226
|
+
const last2 = this.recent.slice(-2);
|
|
227
|
+
if (last2.length === 2 && last2.every(r => r.key === loopKey && r.changed === false)) {
|
|
228
|
+
throw new MobiusError('LOOP_DETECTED', `${tool} repeated 3x on an unchanged screen`, { hint: 'Change strategy: observe, scroll, take a screenshot, or ask the user.' });
|
|
229
|
+
}
|
|
230
|
+
this.verdict = null;
|
|
231
|
+
const target = opts.guard ? await this.gate(tool, opts.guard) : null;
|
|
232
|
+
if (opts.guard?.ctx.mutating)
|
|
233
|
+
this.plane.actions++;
|
|
234
|
+
preHash = this.store.latest()?.screen_hash ?? preHash;
|
|
235
|
+
const message = await withTimeout(exec(target), opts.timeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS, tool);
|
|
236
|
+
const verifying = !!opts.expect || !!opts.check;
|
|
237
|
+
const deadline = Date.now() + (verifying ? opts.verifyMs ?? DEFAULT_EXPECT_TIMEOUT_MS : 2_500);
|
|
238
|
+
let obs = await this.settle(Math.min(deadline, Date.now() + 2_500));
|
|
239
|
+
let verified = null;
|
|
240
|
+
let why = '';
|
|
241
|
+
const evaluate = (o) => {
|
|
242
|
+
const bad = opts.check?.(o) ?? null;
|
|
243
|
+
if (bad) {
|
|
244
|
+
why = bad;
|
|
245
|
+
return false;
|
|
246
|
+
}
|
|
247
|
+
if (opts.expect && !expectationMet(o, opts.expect)) {
|
|
248
|
+
why = `expectation not met: ${JSON.stringify(opts.expect)}`;
|
|
249
|
+
return false;
|
|
250
|
+
}
|
|
251
|
+
return true;
|
|
252
|
+
};
|
|
253
|
+
if (verifying) {
|
|
254
|
+
verified = evaluate(obs);
|
|
255
|
+
while (!verified && Date.now() < deadline) {
|
|
256
|
+
await new Promise(r => setTimeout(r, POLL_MS)); // yield: a fast device must not starve timers (and the CPU)
|
|
257
|
+
obs = await this.observe();
|
|
258
|
+
verified = evaluate(obs);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
result = {
|
|
262
|
+
ok: verified !== false,
|
|
263
|
+
action_id,
|
|
264
|
+
status: verified === false ? 'failed' : 'succeeded',
|
|
265
|
+
duration_ms: Date.now() - t0,
|
|
266
|
+
state_changed: preHash === null ? null : preHash !== obs.screen_hash,
|
|
267
|
+
foreground_app: obs.foreground_app,
|
|
268
|
+
observation_id: obs.observation_id,
|
|
269
|
+
verified,
|
|
270
|
+
message: verified === false ? `${message}, but ${why}` : message,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
catch (e) {
|
|
274
|
+
const err = e instanceof MobiusError ? e : new MobiusError('INTERNAL', e.message);
|
|
275
|
+
result = {
|
|
276
|
+
ok: false, action_id,
|
|
277
|
+
status: err.code === 'OUTCOME_UNKNOWN' ? 'outcome_unknown'
|
|
278
|
+
: ['POLICY_BLOCKED', 'APPROVAL_REJECTED', 'APPROVAL_EXPIRED'].includes(err.code) ? 'blocked' : 'failed',
|
|
279
|
+
duration_ms: Date.now() - t0, state_changed: null, foreground_app: null, observation_id: null, verified: null,
|
|
280
|
+
message: err.message,
|
|
281
|
+
error: { code: err.code, message: err.message, retryable: err.retryable, ...(err.hint ? { hint: err.hint } : {}) },
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
this.recent.push({ key: loopKey, changed: result.state_changed });
|
|
285
|
+
if (this.recent.length > 10)
|
|
286
|
+
this.recent.shift();
|
|
287
|
+
try {
|
|
288
|
+
this.audit.record({
|
|
289
|
+
action_id, device_id: this.connector.deviceId, tool, params: auditParams, status: result.status,
|
|
290
|
+
error_code: result.error?.code ?? null, duration_ms: result.duration_ms,
|
|
291
|
+
agent: (() => { const ses = this.plane.session(); return ses && !('invalid' in ses) ? ses.sub : this.plane.agent(); })(),
|
|
292
|
+
agent_session: (() => { const ses = this.plane.session(); return ses && !('invalid' in ses) ? ses.jti : null; })(),
|
|
293
|
+
session_id: this.plane.session_id,
|
|
294
|
+
decision: this.verdict?.decision ?? null, rule_id: this.verdict?.rule_id ?? null,
|
|
295
|
+
action_class: this.verdict?.action_class ?? null, amount: this.verdict?.amount ?? null,
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
catch (e) {
|
|
299
|
+
process.stderr.write(JSON.stringify({ level: 'error', msg: 'audit write failed', err: String(e) }) + '\n');
|
|
300
|
+
}
|
|
301
|
+
return result;
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { SessionClaims } from '../observability/signing.ts';
|
|
2
|
+
/** Shared across all devices of one server process: who the agent is, and its limits. */
|
|
3
|
+
export interface PlaneContext {
|
|
4
|
+
session_id: string;
|
|
5
|
+
/** MCP client name/version once the agent has connected. */
|
|
6
|
+
agent: () => string | null;
|
|
7
|
+
/** Mutating actions so far, and when the session began. */
|
|
8
|
+
actions: number;
|
|
9
|
+
started: number;
|
|
10
|
+
max_actions: number;
|
|
11
|
+
/** 0 = no limit. */
|
|
12
|
+
max_minutes: number;
|
|
13
|
+
/** The verified signed session of the agent making the current call (or an invalid-token reason). */
|
|
14
|
+
session: () => SessionClaims | {
|
|
15
|
+
invalid: string;
|
|
16
|
+
} | null;
|
|
17
|
+
/** Refuse state-changing actions from agents without a valid signed session (MOBIUS_REQUIRE_SESSION=1). */
|
|
18
|
+
require_session: boolean;
|
|
19
|
+
}
|
|
20
|
+
export declare const newPlaneContext: (agent?: () => string | null) => PlaneContext;
|
|
21
|
+
/** `mobius stop` creates this file; every state-changing action is refused until `mobius resume`. */
|
|
22
|
+
export declare const killSwitchPath: () => string;
|
|
23
|
+
/** `mobius session revoke <jti>` appends here; a revoked session can't act, even before it expires. */
|
|
24
|
+
export declare const revokedPath: () => string;
|
|
25
|
+
export declare const isRevoked: (jti: string) => boolean;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// Infrastructure: session-wide limits and the kill switch, shared by every device a server drives.
|
|
2
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { mobiusHome } from "../observability/audit.js";
|
|
6
|
+
export const newPlaneContext = (agent = () => null) => ({
|
|
7
|
+
session_id: `ses_${randomUUID().slice(0, 8)}`, agent, actions: 0, started: Date.now(),
|
|
8
|
+
max_actions: Number(process.env.MOBIUS_MAX_ACTIONS ?? 1000), max_minutes: Number(process.env.MOBIUS_MAX_SESSION_MIN ?? 0),
|
|
9
|
+
session: () => null, require_session: process.env.MOBIUS_REQUIRE_SESSION === '1',
|
|
10
|
+
});
|
|
11
|
+
/** `mobius stop` creates this file; every state-changing action is refused until `mobius resume`. */
|
|
12
|
+
export const killSwitchPath = () => join(mobiusHome(), 'STOP');
|
|
13
|
+
/** `mobius session revoke <jti>` appends here; a revoked session can't act, even before it expires. */
|
|
14
|
+
export const revokedPath = () => join(mobiusHome(), 'revoked');
|
|
15
|
+
// ponytail: re-read per check (a few lines, like the kill switch); index it if the list ever reaches thousands.
|
|
16
|
+
export const isRevoked = (jti) => existsSync(revokedPath()) && readFileSync(revokedPath(), 'utf8').split('\n').some(l => l.trim() === jti);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { type Connector } from '../execution/connector.ts';
|
|
2
|
+
import { DeviceSession } from '../execution/session.ts';
|
|
3
|
+
import type { AuditLog } from '../observability/audit.ts';
|
|
4
|
+
import type { Approver, Policy } from '../policy/policy.ts';
|
|
5
|
+
import { type PlaneContext } from './plane.ts';
|
|
6
|
+
/** Device registry. `discover` is injectable so tests run without real devices. */
|
|
7
|
+
export declare class Mobius {
|
|
8
|
+
private sessions;
|
|
9
|
+
private readonly discover;
|
|
10
|
+
readonly audit: AuditLog;
|
|
11
|
+
readonly policy: Policy;
|
|
12
|
+
readonly approver: Approver;
|
|
13
|
+
/** Real CLI only: keep element ids resolvable across Mobius restarts (identity hashes on disk, never screen text). */
|
|
14
|
+
persistIds: boolean;
|
|
15
|
+
readonly plane: PlaneContext;
|
|
16
|
+
constructor(discover: () => Promise<Connector[]>, audit: AuditLog, policy: Policy, approver: Approver, plane?: PlaneContext);
|
|
17
|
+
refresh(): Promise<DeviceSession[]>;
|
|
18
|
+
session(deviceId?: string): Promise<DeviceSession>;
|
|
19
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// Infrastructure: which devices exist and one session per device. Drivers are discovered through `discover`.
|
|
2
|
+
import { MobiusError } from "../execution/connector.js";
|
|
3
|
+
import { DeviceSession } from "../execution/session.js";
|
|
4
|
+
import { newPlaneContext } from "./plane.js";
|
|
5
|
+
/** Device registry. `discover` is injectable so tests run without real devices. */
|
|
6
|
+
export class Mobius {
|
|
7
|
+
sessions = new Map();
|
|
8
|
+
discover;
|
|
9
|
+
audit;
|
|
10
|
+
policy;
|
|
11
|
+
approver;
|
|
12
|
+
/** Real CLI only: keep element ids resolvable across Mobius restarts (identity hashes on disk, never screen text). */
|
|
13
|
+
persistIds = false;
|
|
14
|
+
plane;
|
|
15
|
+
constructor(discover, audit, policy, approver, plane = newPlaneContext()) {
|
|
16
|
+
this.plane = plane;
|
|
17
|
+
this.discover = discover;
|
|
18
|
+
this.audit = audit;
|
|
19
|
+
this.policy = policy;
|
|
20
|
+
this.approver = approver;
|
|
21
|
+
}
|
|
22
|
+
async refresh() {
|
|
23
|
+
const found = await this.discover();
|
|
24
|
+
const ids = new Set(found.map(c => c.deviceId));
|
|
25
|
+
for (const id of this.sessions.keys())
|
|
26
|
+
if (!ids.has(id))
|
|
27
|
+
this.sessions.delete(id);
|
|
28
|
+
for (const c of found)
|
|
29
|
+
if (!this.sessions.has(c.deviceId))
|
|
30
|
+
this.sessions.set(c.deviceId, new DeviceSession(c, this.audit, this.policy, this.approver, this.plane, this.persistIds));
|
|
31
|
+
return [...this.sessions.values()];
|
|
32
|
+
}
|
|
33
|
+
async session(deviceId) {
|
|
34
|
+
let s = deviceId ? this.sessions.get(deviceId) : undefined;
|
|
35
|
+
if (!s) {
|
|
36
|
+
const all = await this.refresh();
|
|
37
|
+
s = deviceId ? this.sessions.get(deviceId) : all.length === 1 ? all[0] : undefined;
|
|
38
|
+
if (!s) {
|
|
39
|
+
if (deviceId)
|
|
40
|
+
throw new MobiusError('DEVICE_NOT_FOUND', `No device "${deviceId}"`, { hint: `Available: ${all.map(a => a.connector.deviceId).join(', ') || 'none'}` });
|
|
41
|
+
throw new MobiusError(all.length ? 'INVALID_INPUT' : 'DEVICE_NOT_FOUND', all.length ? `${all.length} devices connected; pass device_id` : 'No devices connected', { hint: all.length ? `One of: ${all.map(a => a.connector.deviceId).join(', ')}` : 'Start an emulator/simulator or plug in a device, then call device_list.' });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
// A fresh process: read the screen once, so element ids from an earlier process can be resolved right away.
|
|
45
|
+
if (this.persistIds && !s.store.latest())
|
|
46
|
+
await s.observe();
|
|
47
|
+
return s;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Mobius } from '../infrastructure/registry.ts';
|
|
2
|
+
import type { SessionClaims } from '../observability/signing.ts';
|
|
3
|
+
export interface ToolInfo {
|
|
4
|
+
name: string;
|
|
5
|
+
description?: string;
|
|
6
|
+
inputSchema: unknown;
|
|
7
|
+
}
|
|
8
|
+
export interface EngineResult {
|
|
9
|
+
ok: boolean;
|
|
10
|
+
result: unknown;
|
|
11
|
+
image?: {
|
|
12
|
+
data: string;
|
|
13
|
+
mimeType: string;
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
export interface Engine {
|
|
17
|
+
tools(): Promise<ToolInfo[]>;
|
|
18
|
+
/** Run one tool as `agent` (recorded in the audit trail). */
|
|
19
|
+
call(tool: string, args: Record<string, unknown>, agent?: string, session?: SessionClaims | {
|
|
20
|
+
invalid: string;
|
|
21
|
+
} | null): Promise<EngineResult>;
|
|
22
|
+
close(): Promise<void>;
|
|
23
|
+
}
|
|
24
|
+
export declare function createEngine(mobius: Mobius): Promise<Engine>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// Agent interface: one engine behind every front door (MCP, REST, SDKs). Each call goes through the same tool layer,
|
|
2
|
+
// so policy, approvals, verification and audit are identical whichever way an agent connects.
|
|
3
|
+
// ponytail: the engine reuses the MCP tool handlers in-process; split them into plain functions if a front door ever
|
|
4
|
+
// needs something MCP can't express.
|
|
5
|
+
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
|
6
|
+
import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js';
|
|
7
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
8
|
+
import { createServer } from "./mcp.js";
|
|
9
|
+
export async function createEngine(mobius) {
|
|
10
|
+
const server = createServer(mobius);
|
|
11
|
+
const [a, b] = InMemoryTransport.createLinkedPair();
|
|
12
|
+
const client = new Client({ name: 'mobius-engine', version: '0.1.0' });
|
|
13
|
+
await Promise.all([server.connect(a), client.connect(b)]);
|
|
14
|
+
// Identity per call, carried with the call itself (not a shared variable), so calls on different devices run at the
|
|
15
|
+
// same time and each audit entry still names its own agent. Each device's own queue keeps its actions in order.
|
|
16
|
+
const who = new AsyncLocalStorage();
|
|
17
|
+
mobius.plane.agent = () => who.getStore()?.agent ?? null;
|
|
18
|
+
mobius.plane.session = () => who.getStore()?.session ?? null;
|
|
19
|
+
return {
|
|
20
|
+
async tools() { return (await client.listTools()).tools.map(t => ({ name: t.name, ...(t.description ? { description: t.description } : {}), inputSchema: t.inputSchema })); },
|
|
21
|
+
call(tool, args, agent, session) {
|
|
22
|
+
return who.run({ agent: agent ?? null, session: session ?? null }, async () => {
|
|
23
|
+
const r = await client.callTool({ name: tool, arguments: args }, undefined, { timeout: 300_000 });
|
|
24
|
+
const text = r.content?.find((c) => c.type === 'text')?.text ?? '{}';
|
|
25
|
+
const img = r.content?.find((c) => c.type === 'image');
|
|
26
|
+
let result;
|
|
27
|
+
try {
|
|
28
|
+
result = JSON.parse(text);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
result = { raw: text };
|
|
32
|
+
}
|
|
33
|
+
return { ok: !r.isError, result, ...(img ? { image: { data: img.data, mimeType: img.mimeType } } : {}) };
|
|
34
|
+
});
|
|
35
|
+
},
|
|
36
|
+
async close() { await client.close(); },
|
|
37
|
+
};
|
|
38
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type IncomingMessage, type ServerResponse } from 'node:http';
|
|
2
|
+
import type { AuditLog } from '../observability/audit.ts';
|
|
3
|
+
import type { Engine } from './engine.ts';
|
|
4
|
+
export declare function createHttpServer(engine: Engine, audit: AuditLog, token: string, sessionKey?: string): import("http").Server<typeof IncomingMessage, typeof ServerResponse>;
|