dsh-mobile-plugin 0.5.24 → 0.5.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/knowledge/apps/alipay.md +30 -0
- package/knowledge/apps/android-settings.md +30 -0
- package/knowledge/apps/bilibili.md +30 -0
- package/knowledge/apps/boss-zhipin.md +32 -0
- package/knowledge/apps/douyin.md +32 -0
- package/knowledge/apps/taobao.md +33 -0
- package/knowledge/apps/wechat.md +33 -0
- package/knowledge/apps/weibo.md +31 -0
- package/knowledge/apps/xiaohongshu.md +29 -0
- package/knowledge/apps/zhihu.md +30 -0
- package/knowledge/sites/bilibili.com.md +27 -0
- package/knowledge/sites/github.com.md +27 -0
- package/knowledge/sites/x.com.md +28 -0
- package/knowledge/sites/xiaohongshu.com.md +27 -0
- package/knowledge/sites/youtube.com.md +28 -0
- package/knowledge/sites/zhihu.com.md +27 -0
- package/lib/impl.js +9 -0
- package/lib/mobile/cli/acp.js +3 -2
- package/lib/mobile/phone/confirm.js +189 -0
- package/lib/mobile/phone/index.js +21 -1
- package/lib/mobile/phone/irreversible.js +146 -0
- package/lib/mobile/phone/pacing.js +173 -0
- package/lib/mobile/phone/route-hint.js +164 -0
- package/lib/mobile/phone/routes-memory.js +133 -0
- package/lib/mobile/phone/routes.js +84 -0
- package/lib/mobile/phone/service.js +90 -5
- package/lib/mobile/phone/tools.js +1190 -64
- package/lib/mobile/phone-knowledge.js +301 -0
- package/lib/mobile/schedule.js +64 -7
- package/lib/mobile/sessions.js +40 -5
- package/lib/mobile/wechat.js +26 -1
- package/lib/p2p/rendezvous.cjs +118 -0
- package/lib/p2p/session.cjs +595 -0
- package/lib/p2p/wiring.cjs +117 -0
- package/lib/panel/index.html +10 -10
- package/lib/relay/status.js +2 -0
- package/lib/routes/control-plane.js +24 -1
- package/lib/routes/mobile-api.js +20 -1
- package/lib/shared/e2ee-request.js +27 -7
- package/lib/tailcat/host-process.cjs +54 -1
- package/package.json +4 -2
|
@@ -24,9 +24,340 @@
|
|
|
24
24
|
|
|
25
25
|
const binding = require('./binding.js');
|
|
26
26
|
|
|
27
|
+
/**
|
|
28
|
+
* 本轮并行加的三个邻居:不可逆判定、节奏、App 知识库。
|
|
29
|
+
*
|
|
30
|
+
* 它们各自是别人的一次提交,可能还没合并进来。这里一律按「拿得到就用、拿不到就当没有」,
|
|
31
|
+
* 因为少一层提示 / 少一段停留都不该让工具的整条注册失败 —— 那会把八个旧工具也一起拖没。
|
|
32
|
+
*/
|
|
33
|
+
let irreversible = null;
|
|
34
|
+
try { irreversible = require('./irreversible.js'); } catch { irreversible = null; }
|
|
35
|
+
let pacing = null;
|
|
36
|
+
try { pacing = require('./pacing.js'); } catch { pacing = null; }
|
|
37
|
+
let phoneKnowledge = null;
|
|
38
|
+
try { phoneKnowledge = require('../phone-knowledge.js'); } catch { phoneKnowledge = null; }
|
|
39
|
+
// 付款确认:走宿主审批通道的那条路。它拿不到审批服务 / 会话时一律回 unavailable,
|
|
40
|
+
// 所以这里直接 require 也没关系 —— 确认要不到只会让「付款那一步不执行」,不会挂住工具。
|
|
41
|
+
const confirm = require('./confirm.js');
|
|
42
|
+
|
|
27
43
|
/** 元素的短标签:模型看到的是「第几个」,不是像素坐标。 */
|
|
28
44
|
const MAX_TEXT = 120;
|
|
29
45
|
|
|
46
|
+
/* ── 守卫账 ─────────────────────────────────────────────────────────────── */
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* 每个插件上下文一份:一轮里 phone_* 的调用次数、同一个 ref 的连点、同一屏的连读、同一批的失败。
|
|
50
|
+
* 按 ctx 隔离,热替换之后旧账不会串到新实例上。
|
|
51
|
+
*/
|
|
52
|
+
const guardStates = new WeakMap();
|
|
53
|
+
|
|
54
|
+
function guardStateFor(ctx) {
|
|
55
|
+
let state = guardStates.get(ctx);
|
|
56
|
+
if (!state) {
|
|
57
|
+
state = {
|
|
58
|
+
budget: new Map(), taps: new Map(), screens: new Map(), batches: new Map(),
|
|
59
|
+
autoShot: new Map(), revisions: new Map(),
|
|
60
|
+
// 上一次读屏的元素表:stale-ref 重试要按「旧文字 + 旧位置」把同一个元素认回来,
|
|
61
|
+
// 没有这一份就只能靠序号猜。只留最近一屏,一条会话一份。
|
|
62
|
+
lastElements: new Map(),
|
|
63
|
+
};
|
|
64
|
+
guardStates.set(ctx, state);
|
|
65
|
+
}
|
|
66
|
+
return state;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** 一轮里允许调用 phone_* 的次数:到了软上限就催收尾,越过硬上限直接拒。 */
|
|
70
|
+
const CALL_SOFT_LIMIT = 60;
|
|
71
|
+
const CALL_HARD_LIMIT = 90;
|
|
72
|
+
/**
|
|
73
|
+
* 一轮里工具结果累计的字节数:超过软线提醒收尾,越过硬线直接拒。
|
|
74
|
+
*
|
|
75
|
+
* 次数拦得住「调用很密」,拦不住「读得很肥」:一次 `phone_steps` 回来两百个元素、
|
|
76
|
+
* 再带一整段知识正文,几轮下来上下文就被界面文字占满了。体积按序列化后的 JSON 估,
|
|
77
|
+
* 不追求精确 —— 目的是拦住「反复读整屏」这种浪费,而不是量到字节。
|
|
78
|
+
*/
|
|
79
|
+
const BYTES_SOFT_LIMIT = 400 * 1024;
|
|
80
|
+
const BYTES_HARD_LIMIT = 800 * 1024;
|
|
81
|
+
/** 拿不到轮次号时,靠「多久没动过」兜底清账,免得同一会话把上限永远吃满。 */
|
|
82
|
+
const IDLE_RESET_MS = 10 * 60 * 1000;
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* 这一发属于会话的哪一轮。
|
|
86
|
+
*
|
|
87
|
+
* 复用直播过程那套轮次口径:`turn/start` 会把当前 turn 号写进 liveSteps 桶,工具执行时读得到。
|
|
88
|
+
* 读不到就退到一个固定键,再由「空闲超时」负责清账(宁可多给几次,也别把还在干活的一轮提前掐死)。
|
|
89
|
+
*/
|
|
90
|
+
function turnKeyOf(ctx, sessionId) {
|
|
91
|
+
try {
|
|
92
|
+
const { liveStepsTurnOf } = require('../live-steps.js');
|
|
93
|
+
const turn = liveStepsTurnOf(ctx, sessionId);
|
|
94
|
+
if (typeof turn === 'number') return `t${turn}`;
|
|
95
|
+
} catch { /* 没有直播模块就走兜底 */ }
|
|
96
|
+
return 'u';
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* 这一轮的那本账:次数与体积记在同一条目里。
|
|
101
|
+
*
|
|
102
|
+
* 两笔账共用一条记录、共用同一次重置(轮次变了就一起清零),因为「这一轮」是同一个概念 ——
|
|
103
|
+
* 分成两个 Map 就得写两遍重置条件,早晚有一边忘了清,于是体积账在新的一轮里还挂着旧数,
|
|
104
|
+
* 第一次读屏就被判超限。
|
|
105
|
+
*/
|
|
106
|
+
function budgetEntry(ctx, sessionId) {
|
|
107
|
+
const state = guardStateFor(ctx);
|
|
108
|
+
const now = Date.now();
|
|
109
|
+
const key = turnKeyOf(ctx, sessionId);
|
|
110
|
+
let entry = state.budget.get(sessionId);
|
|
111
|
+
if (!entry || entry.turnKey !== key || (key === 'u' && now - entry.at > IDLE_RESET_MS)) {
|
|
112
|
+
entry = { turnKey: key, count: 0, bytes: 0, at: now };
|
|
113
|
+
state.budget.set(sessionId, entry);
|
|
114
|
+
}
|
|
115
|
+
entry.at = now;
|
|
116
|
+
return entry;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** 记一次调用并判断是否该拦。返回值里的 count 供软提醒用,bytes 是上一发为止的累计体积。 */
|
|
120
|
+
function checkBudget(ctx, sessionId) {
|
|
121
|
+
const entry = budgetEntry(ctx, sessionId);
|
|
122
|
+
const bytes = entry.bytes;
|
|
123
|
+
entry.count += 1;
|
|
124
|
+
return { blocked: entry.count > CALL_HARD_LIMIT, count: entry.count, bytes };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* 记一发结果的体积,并判断越过了哪条线。
|
|
129
|
+
*
|
|
130
|
+
* 在结果**定稿之后**才记账:工具把自己的 note 补完、把知识附进去之后那份才是真交给模型的
|
|
131
|
+
* 那一段。硬线在下一发调用开头生效(`checkBudget` 带回来的是记账前的数),因为「这一发已经
|
|
132
|
+
* 做完了」再拦它没有意义,能省的是后面那一发。
|
|
133
|
+
*/
|
|
134
|
+
function noteBytes(ctx, sessionId, value) {
|
|
135
|
+
const entry = budgetEntry(ctx, sessionId);
|
|
136
|
+
const size = byteSizeOf(value);
|
|
137
|
+
entry.bytes += size;
|
|
138
|
+
if (entry.bytes > BYTES_HARD_LIMIT) return { over: 'hard', bytes: entry.bytes };
|
|
139
|
+
if (entry.bytes > BYTES_SOFT_LIMIT) return { over: 'soft', bytes: entry.bytes };
|
|
140
|
+
return { over: '', bytes: entry.bytes };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** 序列化后有多少字节。算不出来就按 0 记 —— 体积账坏了不该把工具弄挂。 */
|
|
144
|
+
function byteSizeOf(value) {
|
|
145
|
+
try {
|
|
146
|
+
const json = JSON.stringify(value);
|
|
147
|
+
return typeof json === 'string' ? Buffer.byteLength(json, 'utf8') : 0;
|
|
148
|
+
} catch {
|
|
149
|
+
return 0;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function sleep(ms) {
|
|
154
|
+
if (!Number.isFinite(ms) || ms <= 0) return Promise.resolve();
|
|
155
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function betweenDelay() {
|
|
159
|
+
if (!pacing || typeof pacing.get !== 'function' || typeof pacing.paceFor !== 'function') return 0;
|
|
160
|
+
try {
|
|
161
|
+
const range = pacing.paceFor(pacing.get()).betweenActionsMs;
|
|
162
|
+
if (!Array.isArray(range)) return 0;
|
|
163
|
+
const lo = Number(range[0]) || 0;
|
|
164
|
+
const hi = Number(range[1]) || lo;
|
|
165
|
+
return lo + Math.round(Math.random() * Math.max(0, hi - lo));
|
|
166
|
+
} catch {
|
|
167
|
+
return 0;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** 这台手机报没报某项能力;拿不到能力接口时按「支持」放过,别凭空白挡一条路径。 */
|
|
172
|
+
function supports(service, deviceId, cap) {
|
|
173
|
+
if (!service || typeof service.hasCapability !== 'function') return true;
|
|
174
|
+
try { return service.hasCapability(deviceId, cap) === true; } catch { return true; }
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function unsupportedNote(feature) {
|
|
178
|
+
return `这台手机还不支持${feature},需要把手机上的 App 升级到最新。`;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* `phone_open_link` 的两张封闭白名单。uri 只放行常见入口,settings 只放行这些设置页。
|
|
183
|
+
* 白名单而不是「拼字符串」:拼的话就回到了任意外部输入直达系统的口子。
|
|
184
|
+
*/
|
|
185
|
+
const URI_SCHEMES = new Set([
|
|
186
|
+
'http', 'https', 'tel', 'mailto', 'sms', 'smsto', 'geo', 'market',
|
|
187
|
+
'package', 'weixin', 'alipays', 'taobao', 'bilibili', 'zhihu', 'sinaweibo', 'snssdk1128',
|
|
188
|
+
]);
|
|
189
|
+
|
|
190
|
+
const SETTINGS_ACTIONS = new Set([
|
|
191
|
+
'android.settings.SETTINGS',
|
|
192
|
+
'android.settings.WIFI_SETTINGS',
|
|
193
|
+
'android.settings.BLUETOOTH_SETTINGS',
|
|
194
|
+
'android.settings.DATA_ROAMING_SETTINGS',
|
|
195
|
+
'android.settings.NFC_SETTINGS',
|
|
196
|
+
'android.settings.AIRPLANE_MODE_SETTINGS',
|
|
197
|
+
'android.settings.DISPLAY_SETTINGS',
|
|
198
|
+
'android.settings.SOUND_SETTINGS',
|
|
199
|
+
'android.settings.BATTERY_SAVER_SETTINGS',
|
|
200
|
+
'android.settings.LOCATION_SOURCE_SETTINGS',
|
|
201
|
+
'android.settings.PRIVACY_SETTINGS',
|
|
202
|
+
'android.settings.SECURITY_SETTINGS',
|
|
203
|
+
'android.settings.ACCESSIBILITY_SETTINGS',
|
|
204
|
+
'android.settings.APPLICATION_SETTINGS',
|
|
205
|
+
'android.settings.MANAGE_APPLICATIONS_SETTINGS',
|
|
206
|
+
'android.settings.APPLICATION_DETAILS_SETTINGS',
|
|
207
|
+
'android.settings.NOTIFICATION_SETTINGS',
|
|
208
|
+
'android.settings.ACTION_NOTIFICATION_LISTENER_SETTINGS',
|
|
209
|
+
'android.settings.USAGE_ACCESS_SETTINGS',
|
|
210
|
+
]);
|
|
211
|
+
|
|
212
|
+
/** phone_steps 里每一步允许的动作:单动作,不能再嵌套 batch。 */
|
|
213
|
+
const STEP_ACTIONS = new Set(['tap', 'type', 'swipe', 'key', 'app.open', 'app.close', 'screen.read', 'wait', 'gesture']);
|
|
214
|
+
|
|
215
|
+
/** 等待条件的默认与上限,和协议 1.2 一致。 */
|
|
216
|
+
const WAIT_DEFAULT_MS = 8000;
|
|
217
|
+
const WAIT_MAX_MS = 20000;
|
|
218
|
+
|
|
219
|
+
function clampTimeoutMs(value) {
|
|
220
|
+
if (!Number.isFinite(value)) return WAIT_DEFAULT_MS;
|
|
221
|
+
return Math.max(0, Math.min(WAIT_MAX_MS, Math.round(value)));
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** 判一下这一步是不是不可逆;判定模块没就绪就返回 null(安静跳过)。 */
|
|
225
|
+
function judgeIrreversible(action, args, screenText) {
|
|
226
|
+
if (!irreversible || typeof irreversible.judge !== 'function') return null;
|
|
227
|
+
try { return irreversible.judge({ action, args, screenText }); } catch { return null; }
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** 把一段提示补进结果已有的 note 里;没有 note 或没有提示就原样不动。 */
|
|
231
|
+
function appendNote(value, notes) {
|
|
232
|
+
if (!value || typeof value !== 'object') return value;
|
|
233
|
+
const extra = (Array.isArray(notes) ? notes : [notes]).filter((n) => typeof n === 'string' && n);
|
|
234
|
+
if (extra.length === 0) return value;
|
|
235
|
+
const existing = typeof value.note === 'string' ? value.note : '';
|
|
236
|
+
value.note = existing ? `${existing} ${extra.join(' ')}` : extra.join(' ');
|
|
237
|
+
return value;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function elementText(el) {
|
|
241
|
+
if (!el || typeof el !== 'object') return '';
|
|
242
|
+
for (const key of ['text', 'name', 'label', 'description', 'contentDescription', 'hint']) {
|
|
243
|
+
const value = el[key];
|
|
244
|
+
if (typeof value === 'string' && value.trim()) return value;
|
|
245
|
+
}
|
|
246
|
+
return '';
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** 一屏的指纹:包名 + 前若干个元素的「ref:文字」。用来判断「这块屏一直没变」。 */
|
|
250
|
+
function screenSignature(value) {
|
|
251
|
+
if (!value || typeof value !== 'object') return '';
|
|
252
|
+
const pkg = typeof value.packageName === 'string' ? value.packageName : '';
|
|
253
|
+
const elements = Array.isArray(value.elements) ? value.elements : [];
|
|
254
|
+
const parts = elements.slice(0, 80).map((el) => {
|
|
255
|
+
const ref = el && el.ref ? String(el.ref) : '';
|
|
256
|
+
return `${ref}:${elementText(el).slice(0, 40)}`;
|
|
257
|
+
});
|
|
258
|
+
return `${pkg}|${parts.join('|')}`;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** 元素在哪:手机给的是 `[left, top, right, bottom]` 百分比。读不出来就返回 null。 */
|
|
262
|
+
function centerOf(el) {
|
|
263
|
+
const bounds = el && Array.isArray(el.bounds) ? el.bounds : null;
|
|
264
|
+
if (!bounds || bounds.length < 4) return null;
|
|
265
|
+
const nums = bounds.slice(0, 4).map((n) => Number(n));
|
|
266
|
+
if (!nums.every((n) => Number.isFinite(n))) return null;
|
|
267
|
+
return { x: (nums[0] + nums[2]) / 2, y: (nums[1] + nums[3]) / 2 };
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* 在一屏元素里找回「刚才那个元素」:先按原文文字找,文字一样的有多个时按离原位置最近的算。
|
|
272
|
+
*
|
|
273
|
+
* 为什么按文字 + 相对位置,而不是只按位置:列表滚过一屏之后,同一个位置上是别的条目;
|
|
274
|
+
* 而只按文字又会撞上「发送」这种一屏里出现两次的按钮。两个条件一起用,才既能认出
|
|
275
|
+
* 「同一个控件挪了位置」,也能认出「同一个控件还在原地」。
|
|
276
|
+
*/
|
|
277
|
+
function relocateIn(elements, wanted) {
|
|
278
|
+
const list = Array.isArray(elements) ? elements : [];
|
|
279
|
+
const text = wanted && typeof wanted.text === 'string' ? wanted.text.trim() : '';
|
|
280
|
+
if (!text) return null;
|
|
281
|
+
const same = list.filter((el) => elementText(el).trim() === text);
|
|
282
|
+
if (same.length === 0) return null;
|
|
283
|
+
if (same.length === 1) return same[0];
|
|
284
|
+
const anchor = wanted.center;
|
|
285
|
+
if (!anchor) return same[0];
|
|
286
|
+
let best = null;
|
|
287
|
+
let bestDist = Infinity;
|
|
288
|
+
for (const el of same) {
|
|
289
|
+
const center = centerOf(el);
|
|
290
|
+
if (!center) continue;
|
|
291
|
+
const dist = (center.x - anchor.x) ** 2 + (center.y - anchor.y) ** 2;
|
|
292
|
+
if (dist < bestDist) { bestDist = dist; best = el; }
|
|
293
|
+
}
|
|
294
|
+
return best || same[0];
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* 认一下「这一屏里叫 ref 的那个元素」。读屏结果的元素表就是手机刚吐出来的那一屏,
|
|
299
|
+
* 按 ref 找即可;找不到说明它已经不在这一屏上了。
|
|
300
|
+
*/
|
|
301
|
+
function elementByRef(elements, ref) {
|
|
302
|
+
const wanted = typeof ref === 'string' ? ref.trim() : '';
|
|
303
|
+
if (!wanted) return null;
|
|
304
|
+
const list = Array.isArray(elements) ? elements : [];
|
|
305
|
+
return list.find((el) => el && String(el.ref) === wanted) || null;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* 死循环守卫:同一 ref 连点、同一屏连读、同一批连败,都在结果里加一句该换思路的话。
|
|
310
|
+
* 只提醒、不拦截 —— 拦截会把「界面本来就慢」误判成模型在瞎试。
|
|
311
|
+
*/
|
|
312
|
+
function deadLoopNotes(ctx, sessionId, name, args, value) {
|
|
313
|
+
const state = guardStateFor(ctx);
|
|
314
|
+
const notes = [];
|
|
315
|
+
|
|
316
|
+
if (name === 'phone_tap') {
|
|
317
|
+
const ref = args && typeof args.ref === 'string' ? args.ref.trim() : '';
|
|
318
|
+
if (ref) {
|
|
319
|
+
let entry = state.taps.get(sessionId);
|
|
320
|
+
if (!entry || entry.ref !== ref) entry = { ref, count: 0 };
|
|
321
|
+
entry.count += 1;
|
|
322
|
+
state.taps.set(sessionId, entry);
|
|
323
|
+
if (entry.count >= 4) {
|
|
324
|
+
notes.push(`同一个元素(${ref})已经连点 ${entry.count} 次,几乎肯定点错了。别再点它:重新读屏,换一个元素或换一条路径。`);
|
|
325
|
+
} else if (entry.count === 3) {
|
|
326
|
+
notes.push(`同一个元素(${ref})已经点了 3 次,屏没变就别再点,先读屏看看实际停在哪。`);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
if (name === 'phone_screen') {
|
|
332
|
+
const sig = screenSignature(value);
|
|
333
|
+
if (sig) {
|
|
334
|
+
let entry = state.screens.get(sessionId);
|
|
335
|
+
if (!entry || entry.sig !== sig) entry = { sig, count: 0 };
|
|
336
|
+
entry.count += 1;
|
|
337
|
+
state.screens.set(sessionId, entry);
|
|
338
|
+
if (entry.count >= 5) {
|
|
339
|
+
notes.push(`这块屏已经连续读了 ${entry.count} 次,内容一直没变。别再用 phone_screen 反复读,换个做法。`);
|
|
340
|
+
} else if (entry.count === 4) {
|
|
341
|
+
notes.push('这块屏连续读了 4 次没变化,再读一次也不会变,考虑换个做法。');
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
if (name === 'phone_steps') {
|
|
347
|
+
const failed = Boolean(value && (value.ok === false || value.stopReason === 'step-failed'));
|
|
348
|
+
const entry = state.batches.get(sessionId) || { count: 0 };
|
|
349
|
+
entry.count = failed ? entry.count + 1 : 0;
|
|
350
|
+
state.batches.set(sessionId, entry);
|
|
351
|
+
if (entry.count >= 3) {
|
|
352
|
+
notes.push(`这一批已经失败 ${entry.count} 次,原样重试大概率还是失败。换一种做法:拆成单步,先读屏确认。`);
|
|
353
|
+
} else if (entry.count === 2) {
|
|
354
|
+
notes.push('这一批已经失败 2 次,原样重试大概率还是失败,换个思路。');
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
return notes;
|
|
359
|
+
}
|
|
360
|
+
|
|
30
361
|
function textOf(value, max) {
|
|
31
362
|
if (typeof value !== 'string') return '';
|
|
32
363
|
const trimmed = value.trim();
|
|
@@ -60,13 +391,21 @@ async function run(service, deviceId, sessionId, action, args, signal) {
|
|
|
60
391
|
} catch (error) {
|
|
61
392
|
const code = error && error.code ? error.code : 'failed';
|
|
62
393
|
const message = error && error.message ? error.message : '手机那边没能完成这一步';
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
394
|
+
// 抛出去的时候把手机侧的 code 带在 error 上:`stale-ref` 要在这里被上层认出来做一次
|
|
395
|
+
// 自动重试,而「人话」那段文案里不带机器码。别的地方照旧只看 message。
|
|
396
|
+
const withCode = (text) => {
|
|
397
|
+
const wrapped = new Error(text);
|
|
398
|
+
wrapped.code = code;
|
|
399
|
+
return wrapped;
|
|
400
|
+
};
|
|
401
|
+
if (code === 'offline') throw withCode('那台手机在这期间断开了连接,这一步没有做。');
|
|
402
|
+
if (code === 'timeout') throw withCode(`那台手机 ${message}。可以再试一次,或者先 phone_screen 看看它现在停在哪。`);
|
|
403
|
+
if (code === 'unsupported') throw withCode(`那台手机上装的 App 版本还不支持 ${action},需要把手机上的 App 升级到最新。`);
|
|
404
|
+
if (code === 'adb-unavailable') throw withCode('那台手机的无线调试没连上(可能被系统关了),让用户在手机上重新打开无线调试。');
|
|
405
|
+
if (code === 'warming') throw withCode('手机正在准备后台屏,大约还要几秒,稍等再试。');
|
|
406
|
+
if (code === 'aborted') throw withCode('这一步被取消了。');
|
|
407
|
+
if (code === 'stale-ref') throw withCode(`${message}(界面在点之前变了)。`);
|
|
408
|
+
throw withCode(message);
|
|
70
409
|
}
|
|
71
410
|
}
|
|
72
411
|
|
|
@@ -84,7 +423,37 @@ function summarizeScreen(data) {
|
|
|
84
423
|
const count = Array.isArray(data.elements) ? data.elements.length : 0;
|
|
85
424
|
parts.push(`可见元素 ${count} 个`);
|
|
86
425
|
if (data.treeUnreadable) parts.push('没读到应用窗口(可能是自绘界面),坐标只能看截图');
|
|
87
|
-
|
|
426
|
+
// 元素表要写进这段文本里:模型看到的是 render 出来的文字,不是工具结果里的 json 字段。
|
|
427
|
+
// 只给「可见元素 61 个」而不给这 61 个是什么,等于告诉模型「屏上有东西但我不告诉你」——
|
|
428
|
+
// 它就只能去截图猜、或者拿 ref 瞎试。
|
|
429
|
+
const lines = elementLines(data.elements);
|
|
430
|
+
if (lines.length > 0) parts.push('', ...lines);
|
|
431
|
+
return parts.join('\n');
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/** 把元素表写成一行一个的短文本;每个都带 ref,模型据此点。 */
|
|
435
|
+
function elementLines(elements) {
|
|
436
|
+
const list = Array.isArray(elements) ? elements : [];
|
|
437
|
+
const out = [];
|
|
438
|
+
for (const el of list) {
|
|
439
|
+
if (!el || typeof el !== 'object') continue;
|
|
440
|
+
const ref = el.ref === undefined || el.ref === null ? '' : String(el.ref);
|
|
441
|
+
if (!ref) continue;
|
|
442
|
+
const bits = [];
|
|
443
|
+
if (el.role) bits.push(String(el.role));
|
|
444
|
+
const label = el.text || el.name || el.description || '';
|
|
445
|
+
if (label) bits.push(`"${String(label).replace(/\s+/g, ' ').slice(0, 80)}"`);
|
|
446
|
+
if (el.clickable) bits.push('可点');
|
|
447
|
+
if (el.editable) bits.push('可输入');
|
|
448
|
+
if (el.scrollable) bits.push('可滚动');
|
|
449
|
+
if (el.checked === true) bits.push('已勾选');
|
|
450
|
+
if (el.secret) bits.push('密码框');
|
|
451
|
+
if (el.enabled === false) bits.push('不可用');
|
|
452
|
+
const b = Array.isArray(el.bounds) ? el.bounds : null;
|
|
453
|
+
if (b && b.length === 4) bits.push(`[${b.join(',')}]`);
|
|
454
|
+
out.push(`${ref}: ${bits.join(' ')}`);
|
|
455
|
+
}
|
|
456
|
+
return out;
|
|
88
457
|
}
|
|
89
458
|
|
|
90
459
|
/**
|
|
@@ -106,7 +475,48 @@ function installPhoneTools(ctx, deps) {
|
|
|
106
475
|
}
|
|
107
476
|
|
|
108
477
|
const build = (scoped) => {
|
|
109
|
-
|
|
478
|
+
/**
|
|
479
|
+
* 统一注册口:每条工具都套上「步数预算」与「死循环」两道软守卫。
|
|
480
|
+
*
|
|
481
|
+
* 包在注册这一层而不是每条 execute 里各写一遍 —— 守卫要对**所有** phone_* 生效,
|
|
482
|
+
* 漏一条就等于漏一条腿。预算越界直接拒(返回前就抛),软提醒补进结果已有的 note。
|
|
483
|
+
*/
|
|
484
|
+
const register = (tool) => {
|
|
485
|
+
const inner = tool.execute;
|
|
486
|
+
scoped.tools.register(defineTool({
|
|
487
|
+
...tool,
|
|
488
|
+
execute: async (args, exec) => {
|
|
489
|
+
const sessionId = sessionOf(exec);
|
|
490
|
+
const gate = checkBudget(ctx, sessionId);
|
|
491
|
+
if (gate.blocked) {
|
|
492
|
+
throw new Error(`这一轮操作手机已经到上限(约 ${CALL_HARD_LIMIT} 次),不能继续试了。把现在的进度告诉用户,需要的话用 end_not_done 收尾。`);
|
|
493
|
+
}
|
|
494
|
+
// 体积硬线:上一发之后累计已经越过 800KB 就直接拒。这一发还没跑,拦下来才有意义;
|
|
495
|
+
// 预算按「这一轮」算,轮次变了由 `budgetEntry` 一起清掉(次数与体积同一次重置)。
|
|
496
|
+
if (gate.bytes > BYTES_HARD_LIMIT) {
|
|
497
|
+
throw new Error(`这一轮手机界面读回来的内容已经超过 ${Math.round(BYTES_HARD_LIMIT / 1024)}KB,上下文塞不下了,不能继续读。把现在的进度告诉用户,需要的话用 end_not_done 收尾。`);
|
|
498
|
+
}
|
|
499
|
+
const value = await inner(args, exec);
|
|
500
|
+
// 顺手记住这次结果里的元素表:stale-ref 重试靠它把「上一次那个元素」认回来。
|
|
501
|
+
// 只在真的读到元素时更新 —— 无障碍没开时是一张空表,拿它覆盖会把上一屏一起忘掉。
|
|
502
|
+
if (value && Array.isArray(value.elements) && value.elements.length > 0) {
|
|
503
|
+
guardStateFor(ctx).lastElements.set(sessionId, value.elements);
|
|
504
|
+
}
|
|
505
|
+
const notes = [];
|
|
506
|
+
if (gate.count > CALL_SOFT_LIMIT) {
|
|
507
|
+
notes.push(`这一轮操作手机的次数已经超过 ${CALL_SOFT_LIMIT} 次(现在 ${gate.count} 次),请尽快收尾或结束这一步,别再零碎地反复试。`);
|
|
508
|
+
}
|
|
509
|
+
notes.push(...deadLoopNotes(ctx, sessionId, tool.name, args, value));
|
|
510
|
+
const withNotes = appendNote(value, notes);
|
|
511
|
+
// 体积账在结果定稿之后才记:附进去的那些提示本身也是交给模型的字节。
|
|
512
|
+
const size = noteBytes(ctx, sessionId, withNotes);
|
|
513
|
+
if (size.over === 'soft') {
|
|
514
|
+
appendNote(withNotes, `这一轮读到的手机界面已经累计约 ${Math.round(size.bytes / 1024)}KB,读得太多了。该收尾了:把结论说给用户,别再整屏整屏地读。`);
|
|
515
|
+
}
|
|
516
|
+
return withNotes;
|
|
517
|
+
},
|
|
518
|
+
}));
|
|
519
|
+
};
|
|
110
520
|
|
|
111
521
|
/**
|
|
112
522
|
* 宿主的附件库:截图要交给它存,编号才是宿主认得的那种(见 `phone_screenshot` 的说明)。
|
|
@@ -124,6 +534,108 @@ function installPhoneTools(ctx, deps) {
|
|
|
124
534
|
return null;
|
|
125
535
|
};
|
|
126
536
|
|
|
537
|
+
/**
|
|
538
|
+
* 截一张后台屏并存进宿主附件库,返回可直接塞进结果的那个图片块。
|
|
539
|
+
*
|
|
540
|
+
* 截图有两条用途:模型主动要看(`phone_screenshot`),以及读屏读不出应用窗口时自动补一张
|
|
541
|
+
* (`phone_screen`)。存图这件事只写一遍 —— 引用编号必须是宿主那种 `sha256:` 形状,写错地方
|
|
542
|
+
* 下一轮整条请求会挂(2026-11-04 真机)。
|
|
543
|
+
*/
|
|
544
|
+
const captureImage = async (service, deviceId, sessionId, target, signal) => {
|
|
545
|
+
const data = await run(service, deviceId, sessionId, 'screen.screenshot', {}, signal);
|
|
546
|
+
const blob = data && typeof data.blob === 'string' ? data.blob : '';
|
|
547
|
+
if (!blob) throw new Error('没截到画面:手机那边没有回图片。');
|
|
548
|
+
const store = attachmentsOf();
|
|
549
|
+
if (!store) throw new Error('宿主没有附件库,这张图存不进去。');
|
|
550
|
+
let ref;
|
|
551
|
+
try {
|
|
552
|
+
ref = await store.saveImage({
|
|
553
|
+
data: Buffer.from(blob, 'base64'),
|
|
554
|
+
mediaType: data && data.format === 'jpeg' ? 'image/jpeg' : 'image/png',
|
|
555
|
+
name: textOf(target, MAX_TEXT) || 'phone-screen',
|
|
556
|
+
});
|
|
557
|
+
} catch (error) {
|
|
558
|
+
throw new Error(`截到的画面存不下来:${String((error && error.message) || error).slice(0, 200)}`);
|
|
559
|
+
}
|
|
560
|
+
return {
|
|
561
|
+
width: Number(ref.width) || 0,
|
|
562
|
+
height: Number(ref.height) || 0,
|
|
563
|
+
image: {
|
|
564
|
+
type: 'image',
|
|
565
|
+
attachment: {
|
|
566
|
+
attachmentId: ref.attachmentId,
|
|
567
|
+
mediaType: ref.mediaType,
|
|
568
|
+
bytes: ref.bytes,
|
|
569
|
+
width: ref.width,
|
|
570
|
+
height: ref.height,
|
|
571
|
+
...(ref.name ? { name: ref.name } : {}),
|
|
572
|
+
},
|
|
573
|
+
},
|
|
574
|
+
};
|
|
575
|
+
};
|
|
576
|
+
|
|
577
|
+
/** 在一屏元素里对一下等待条件,成立就返回。 */
|
|
578
|
+
const matchCondition = (elements, pkg, until) => {
|
|
579
|
+
const hay = (Array.isArray(elements) ? elements : []).map(elementText).join('\n').toLowerCase();
|
|
580
|
+
const pkgLower = String(pkg || '').toLowerCase();
|
|
581
|
+
if (until.appears && hay.includes(String(until.appears).toLowerCase())) return { met: true, stopped: null };
|
|
582
|
+
if (until.gone && !hay.includes(String(until.gone).toLowerCase())) return { met: true, stopped: null };
|
|
583
|
+
if (until.app && pkgLower.includes(String(until.app).toLowerCase())) return { met: true, stopped: null };
|
|
584
|
+
return null;
|
|
585
|
+
};
|
|
586
|
+
|
|
587
|
+
/**
|
|
588
|
+
* 原生的 screen.wait 用不了时的兜底:自己按 500ms 读一屏比对。
|
|
589
|
+
* 慢,但比「假装等到了」或直接报不支持强 —— 能力差异不该让用户的任务断掉。
|
|
590
|
+
*/
|
|
591
|
+
const pollCondition = async (service, deviceId, sessionId, until, timeoutMs, signal) => {
|
|
592
|
+
const start = Date.now();
|
|
593
|
+
const deadline = start + timeoutMs;
|
|
594
|
+
let lastSig = '';
|
|
595
|
+
let unchanged = 0;
|
|
596
|
+
for (;;) {
|
|
597
|
+
const data = await run(service, deviceId, sessionId, 'screen.read', { includeText: true }, signal);
|
|
598
|
+
const elements = Array.isArray(data && data.elements) ? data.elements : [];
|
|
599
|
+
const hit = matchCondition(elements, data && data.packageName, until);
|
|
600
|
+
if (hit) return { met: true, stopped: null, waitedMs: Date.now() - start };
|
|
601
|
+
if (until.stable === true) {
|
|
602
|
+
const sig = screenSignature({ packageName: data && data.packageName, elements });
|
|
603
|
+
if (sig === lastSig) {
|
|
604
|
+
unchanged += 1;
|
|
605
|
+
if (unchanged >= 1) return { met: true, stopped: null, waitedMs: Date.now() - start };
|
|
606
|
+
} else {
|
|
607
|
+
unchanged = 0;
|
|
608
|
+
lastSig = sig;
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
if (Date.now() >= deadline) return { met: false, stopped: 'timeout', waitedMs: Date.now() - start };
|
|
612
|
+
await sleep(Math.max(50, Math.min(500, deadline - Date.now())));
|
|
613
|
+
}
|
|
614
|
+
};
|
|
615
|
+
|
|
616
|
+
/** 等一个条件:原生 screen.wait 可用就走它,否则退回轮询。 */
|
|
617
|
+
const waitCondition = async (service, deviceId, sessionId, until, signal, canNative) => {
|
|
618
|
+
const timeoutMs = clampTimeoutMs(until && until.timeoutMs);
|
|
619
|
+
const cond = { timeoutMs };
|
|
620
|
+
if (until && typeof until.appears === 'string' && until.appears.trim()) cond.appears = textOf(until.appears, 200);
|
|
621
|
+
if (until && typeof until.gone === 'string' && until.gone.trim()) cond.gone = textOf(until.gone, 200);
|
|
622
|
+
if (until && typeof until.app === 'string' && until.app.trim()) cond.app = textOf(until.app, 200);
|
|
623
|
+
if (until && until.stable === true) cond.stable = true;
|
|
624
|
+
const hasCondition = cond.appears !== undefined || cond.gone !== undefined || cond.app !== undefined || cond.stable === true;
|
|
625
|
+
if (!hasCondition) return { met: true, stopped: null, waitedMs: 0 };
|
|
626
|
+
if (canNative) {
|
|
627
|
+
try {
|
|
628
|
+
const data = await run(service, deviceId, sessionId, 'screen.wait', cond, signal);
|
|
629
|
+
return {
|
|
630
|
+
met: Boolean(data && data.met === true),
|
|
631
|
+
stopped: data && data.stopped != null ? data.stopped : null,
|
|
632
|
+
waitedMs: Number(data && data.waitedMs) || 0,
|
|
633
|
+
};
|
|
634
|
+
} catch { /* 原生等失败就退回轮询 */ }
|
|
635
|
+
}
|
|
636
|
+
return pollCondition(service, deviceId, sessionId, cond, timeoutMs, signal);
|
|
637
|
+
};
|
|
638
|
+
|
|
127
639
|
/**
|
|
128
640
|
* 所有工具都要的那一步:从执行上下文里认出这条会话。
|
|
129
641
|
*
|
|
@@ -136,6 +648,120 @@ function installPhoneTools(ctx, deps) {
|
|
|
136
648
|
return typeof id === 'string' ? id : '';
|
|
137
649
|
};
|
|
138
650
|
|
|
651
|
+
/**
|
|
652
|
+
* 有没有手机正在**看这条会话**。
|
|
653
|
+
*
|
|
654
|
+
* 判据是「这台设备开着会话流」:`stream-registry` 按 deviceId 只记「有没有流开着」,
|
|
655
|
+
* 会话级的观众表在审批桥里且是模块私有的。所以这里退一步用「这台会话绑的手机上有没有
|
|
656
|
+
* 流开着」—— 手机开着会话页就一定有一条流,关掉就一条都不剩。
|
|
657
|
+
*
|
|
658
|
+
* 判错的代价不对称:判成「有」只是多发一张没人答的卡(有硬期限兜底,最坏三分钟后回
|
|
659
|
+
* unavailable),判成「没有」会把一次本该发生的确认降级成 unavailable 而根本不问。
|
|
660
|
+
* 所以宁可宽一点。
|
|
661
|
+
*/
|
|
662
|
+
const someoneWatching = (sessionId) => {
|
|
663
|
+
let deviceId = '';
|
|
664
|
+
try {
|
|
665
|
+
const { boundDeviceOf } = require('./binding.js');
|
|
666
|
+
deviceId = boundDeviceOf(sessionId) || '';
|
|
667
|
+
} catch {
|
|
668
|
+
return true;
|
|
669
|
+
}
|
|
670
|
+
// 不知道绑哪台(新会话还没绑定):当作「可能有人看」,让审批桥自己去推。
|
|
671
|
+
if (!deviceId) return true;
|
|
672
|
+
try {
|
|
673
|
+
const registry = require('../stream-registry.js');
|
|
674
|
+
if (typeof registry.has === 'function') return registry.has(deviceId);
|
|
675
|
+
// 登记表没给按设备查的口子时只能看总数:有流开着就试一次。
|
|
676
|
+
if (typeof registry.size === 'function') return registry.size() > 0;
|
|
677
|
+
} catch { /* 没有登记表就不凭空白挡 */ }
|
|
678
|
+
return true;
|
|
679
|
+
};
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* 付款 / 下单那一步的关卡:先问宿主审批通道,拿到 `allowed-once` 才放行。
|
|
683
|
+
*
|
|
684
|
+
* 返回 null 表示可以执行;返回一句话表示不执行,那句话直接进工具结果交给模型。
|
|
685
|
+
* 只有 `payment` 走这里(`confirm.needsConfirmation` 的判据),发送 / 提交 / 删除照旧只按
|
|
686
|
+
* 节奏拉间隔 —— 那几种是任务本身要做的动作,每一步都拦会让人没法用。
|
|
687
|
+
*/
|
|
688
|
+
const requirePaymentConfirmation = async (sessionId, exec, what) => {
|
|
689
|
+
const callId = exec && typeof exec.callId === 'string' ? exec.callId : '';
|
|
690
|
+
const outcome = await confirm.requestConfirmation(ctx, {
|
|
691
|
+
sessionId,
|
|
692
|
+
summary: what,
|
|
693
|
+
agent: exec && exec.agent,
|
|
694
|
+
// 卡片上那个「工具 X 请求越权执行」的 X。写真实的工具名,用户在手机上看得出来
|
|
695
|
+
// 这一下是哪条工具要动手。
|
|
696
|
+
toolName: 'phone_tap',
|
|
697
|
+
// 有值才塞:宿主与守卫都不认值为 undefined 的字段。
|
|
698
|
+
...(callId ? { callId } : {}),
|
|
699
|
+
signal: exec && exec.signal,
|
|
700
|
+
watching: someoneWatching(sessionId),
|
|
701
|
+
});
|
|
702
|
+
if (outcome === 'allowed-once') return null;
|
|
703
|
+
return confirm.blockedMessage(outcome, what);
|
|
704
|
+
};
|
|
705
|
+
|
|
706
|
+
/**
|
|
707
|
+
* 这一屏里还认不认得那个 ref;认不得就按「上一次那个 ref 的文字 + 相对位置」在**新读的一屏**
|
|
708
|
+
* 里把同一个元素找回来。
|
|
709
|
+
*
|
|
710
|
+
* 手机侧自己也做了一次同样的重定位(`PhoneExecutor.relocate`),但那条路只覆盖「手机那一侧
|
|
711
|
+
* 拿到的旧屏还在」。插件这一层再兜一次,是因为模型拿到的 ref 来自我们上一次读屏的结果 ——
|
|
712
|
+
* 那一屏可能比手机自己记的还旧。多花一次读屏换模型少一次来回,值得。
|
|
713
|
+
*
|
|
714
|
+
* 只做一次:再认不出就是界面真的换了,那时报错比循环更有用(与协议 7.4「只重试一次」一致)。
|
|
715
|
+
*
|
|
716
|
+
* @returns {Promise<{ref: string, note: string}|null>} 找到就返回新 ref 与一句说明
|
|
717
|
+
*/
|
|
718
|
+
const relocateStaleRef = async (service, deviceId, sessionId, ref, signal) => {
|
|
719
|
+
const wanted = String(ref || '').trim();
|
|
720
|
+
if (!wanted) return null;
|
|
721
|
+
const previous = guardStateFor(ctx).lastElements.get(sessionId);
|
|
722
|
+
const staleEl = elementByRef(previous, wanted);
|
|
723
|
+
let data;
|
|
724
|
+
try {
|
|
725
|
+
data = await run(service, deviceId, sessionId, 'screen.read', { includeText: true }, signal);
|
|
726
|
+
} catch {
|
|
727
|
+
return null;
|
|
728
|
+
}
|
|
729
|
+
const elements = Array.isArray(data && data.elements) ? data.elements : [];
|
|
730
|
+
if (data && Array.isArray(data.elements)) guardStateFor(ctx).lastElements.set(sessionId, elements);
|
|
731
|
+
// 先按「旧文字 + 旧位置」认:这是协议里定的口径,比按序号猜准得多。
|
|
732
|
+
const hit = staleEl
|
|
733
|
+
? relocateIn(elements, { text: elementText(staleEl), center: centerOf(staleEl) })
|
|
734
|
+
: null;
|
|
735
|
+
if (hit && hit.ref) {
|
|
736
|
+
return {
|
|
737
|
+
ref: String(hit.ref),
|
|
738
|
+
note: `元素编号 ${wanted} 已经不在界面上了,按「${elementText(staleEl).slice(0, 24)}」重新定位到 ${hit.ref},重试过一次。`,
|
|
739
|
+
};
|
|
740
|
+
}
|
|
741
|
+
return null;
|
|
742
|
+
};
|
|
743
|
+
|
|
744
|
+
/**
|
|
745
|
+
* 带一次 stale-ref 重试的执行。
|
|
746
|
+
*
|
|
747
|
+
* `tap` 与 `type` 的 ref 都是「上一次读屏的编号」,界面一动就过期。手机侧会回 `stale-ref`,
|
|
748
|
+
* 这里收到之后**自动重新读一屏、找回同一个元素、用新编号再试一次**,并且把重试这件事
|
|
749
|
+
* 如实写进结果 —— 模型要知道「看到的编号不一定还作数」,但不必为一次正常的界面变化
|
|
750
|
+
* 再跑一整轮读屏 + 决策。
|
|
751
|
+
*/
|
|
752
|
+
const runWithStaleRetry = async (service, deviceId, sessionId, action, args, signal) => {
|
|
753
|
+
try {
|
|
754
|
+
return { data: await run(service, deviceId, sessionId, action, args, signal), note: '' };
|
|
755
|
+
} catch (error) {
|
|
756
|
+
if (!error || error.code !== 'stale-ref') throw error;
|
|
757
|
+
const ref = typeof args.ref === 'string' ? args.ref.trim() : '';
|
|
758
|
+
const found = await relocateStaleRef(service, deviceId, sessionId, ref, signal);
|
|
759
|
+
if (!found) throw error;
|
|
760
|
+
const retryArgs = { ...args, ref: found.ref };
|
|
761
|
+
return { data: await run(service, deviceId, sessionId, action, retryArgs, signal), note: found.note };
|
|
762
|
+
}
|
|
763
|
+
};
|
|
764
|
+
|
|
139
765
|
/**
|
|
140
766
|
* 拼出一句「这一轮来自谁、绑着谁」的提醒。
|
|
141
767
|
*
|
|
@@ -157,6 +783,10 @@ function installPhoneTools(ctx, deps) {
|
|
|
157
783
|
type: 'boolean',
|
|
158
784
|
description: 'Include the readable text of each element. Default true; turn off for very long lists.',
|
|
159
785
|
},
|
|
786
|
+
compact: {
|
|
787
|
+
type: 'boolean',
|
|
788
|
+
description: 'Only return elements that changed since the previous read, plus a count of unchanged ones. Default false; use it in a loop to keep results small. Ignored on phones that cannot report changes.',
|
|
789
|
+
},
|
|
160
790
|
},
|
|
161
791
|
output: {
|
|
162
792
|
schema: {
|
|
@@ -172,18 +802,44 @@ function installPhoneTools(ctx, deps) {
|
|
|
172
802
|
// 无障碍没开时这一格为真:说清楚「是我们看不见」而不是「界面是空的」。
|
|
173
803
|
a11yDisabled: { type: 'boolean' },
|
|
174
804
|
treeUnreadable: { type: 'boolean' },
|
|
805
|
+
// 读屏读不出应用窗口时,第一次读会自动补一张截图,和文字一起回来。
|
|
806
|
+
image: { type: 'object', additionalProperties: true },
|
|
175
807
|
note: { type: 'string' },
|
|
176
808
|
},
|
|
177
809
|
},
|
|
178
|
-
render: (_args, value) =>
|
|
810
|
+
render: (_args, value) => {
|
|
811
|
+
const blocks = [{ type: 'text', text: summarizeScreen(value) }];
|
|
812
|
+
if (value.image) blocks.push(value.image);
|
|
813
|
+
return blocks;
|
|
814
|
+
},
|
|
179
815
|
},
|
|
180
816
|
execute: async (args, exec) => {
|
|
181
817
|
const sessionId = sessionOf(exec);
|
|
182
818
|
const { deviceId, service } = targetOf(ctx, deps, sessionId);
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
819
|
+
const wantCompact = args.compact === true;
|
|
820
|
+
const canCompact = supports(service, deviceId, 'screen.compact');
|
|
821
|
+
const readArgs = { includeText: args.includeText !== false };
|
|
822
|
+
// 参数只有在手机确实报了这项能力时才透传;否则如实说一句,而不是假装成功。
|
|
823
|
+
const gsRev = guardStateFor(ctx).revisions;
|
|
824
|
+
if (wantCompact && canCompact) {
|
|
825
|
+
readArgs.compact = true;
|
|
826
|
+
// 带上上次的读屏编号,手机才知道「从哪一版开始算变化」;对不上它会自动回全量。
|
|
827
|
+
if (gsRev.has(sessionId)) readArgs.since = gsRev.get(sessionId);
|
|
828
|
+
}
|
|
829
|
+
const data = await run(service, deviceId, sessionId, 'screen.read', readArgs, exec.signal);
|
|
830
|
+
if (data && Number.isFinite(data.revision)) gsRev.set(sessionId, data.revision);
|
|
831
|
+
|
|
832
|
+
const notes = [mentionMismatch(sessionId, deviceId)];
|
|
833
|
+
if (wantCompact && !canCompact) notes.push(unsupportedNote('紧凑读屏(compact)'));
|
|
834
|
+
|
|
835
|
+
// App 用法知识:同一个对话、同一个 App 只附一次。知识库没就绪就当作没有。
|
|
836
|
+
if (phoneKnowledge && typeof phoneKnowledge.noteFor === 'function') {
|
|
837
|
+
try {
|
|
838
|
+
const knowledge = phoneKnowledge.noteFor(sessionId, data && data.packageName);
|
|
839
|
+
if (knowledge) notes.push(knowledge);
|
|
840
|
+
} catch { /* 知识是锦上添花,读坏了不影响读屏 */ }
|
|
841
|
+
}
|
|
842
|
+
|
|
187
843
|
// 这两个开关**只在这两件事真的成立时才有键**:宿主不允许工具结果里出现值为 undefined
|
|
188
844
|
// 的字段(整条结果会被判成不是合法 JSON,模型看到的是「读屏坏了」),所以按「有值才塞」
|
|
189
845
|
// 的写法构造,而不是摆一个 undefined 占位。
|
|
@@ -192,10 +848,28 @@ function installPhoneTools(ctx, deps) {
|
|
|
192
848
|
packageName: textOf(data && data.packageName, MAX_TEXT),
|
|
193
849
|
activity: textOf(data && data.activity, MAX_TEXT),
|
|
194
850
|
elements: Array.isArray(data && data.elements) ? data.elements.slice(0, 200) : [],
|
|
195
|
-
note,
|
|
196
851
|
};
|
|
197
852
|
if (data && data.a11yDisabled === true) value.a11yDisabled = true;
|
|
198
853
|
if (data && data.treeUnreadable === true) value.treeUnreadable = true;
|
|
854
|
+
|
|
855
|
+
// 读屏读不出应用窗口(自绘界面)时,第一次自动补一张截图:同一个结果里文字和图像一起给,
|
|
856
|
+
// 省掉模型先读一次、再决定要不要截的来回。屏恢复可读后清掉标记,下次再遇到还能自动补。
|
|
857
|
+
const gs = guardStateFor(ctx);
|
|
858
|
+
if (data && data.treeUnreadable === true) {
|
|
859
|
+
if (supports(service, deviceId, 'screen.screenshot') && !gs.autoShot.has(sessionId)) {
|
|
860
|
+
gs.autoShot.set(sessionId, true);
|
|
861
|
+
try {
|
|
862
|
+
const shot = await captureImage(service, deviceId, sessionId, 'phone-screen', exec.signal);
|
|
863
|
+
value.image = shot.image;
|
|
864
|
+
notes.push('读屏读不出应用窗口,已自动附一张截图。');
|
|
865
|
+
} catch { /* 截图失败不影响已经拿到的文字 */ }
|
|
866
|
+
}
|
|
867
|
+
} else if (gs.autoShot.has(sessionId)) {
|
|
868
|
+
gs.autoShot.delete(sessionId);
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
const note = notes.filter((n) => typeof n === 'string' && n).join(' ');
|
|
872
|
+
if (note) value.note = note;
|
|
199
873
|
return value;
|
|
200
874
|
},
|
|
201
875
|
});
|
|
@@ -239,42 +913,15 @@ function installPhoneTools(ctx, deps) {
|
|
|
239
913
|
execute: async (args, exec) => {
|
|
240
914
|
const sessionId = sessionOf(exec);
|
|
241
915
|
const { deviceId, service } = targetOf(ctx, deps, sessionId);
|
|
242
|
-
const
|
|
243
|
-
const blob = data && typeof data.blob === 'string' ? data.blob : '';
|
|
244
|
-
if (!blob) throw new Error('没截到画面:手机那边没有回图片。');
|
|
245
|
-
const store = attachmentsOf();
|
|
246
|
-
if (!store) throw new Error('宿主没有附件库,这张图存不进去。');
|
|
247
|
-
let ref;
|
|
248
|
-
try {
|
|
249
|
-
ref = await store.saveImage({
|
|
250
|
-
data: Buffer.from(blob, 'base64'),
|
|
251
|
-
mediaType: data && data.format === 'jpeg' ? 'image/jpeg' : 'image/png',
|
|
252
|
-
name: textOf(args.target, MAX_TEXT) || 'phone-screen',
|
|
253
|
-
});
|
|
254
|
-
} catch (error) {
|
|
255
|
-
throw new Error(`截到的画面存不下来:${String((error && error.message) || error).slice(0, 200)}`);
|
|
256
|
-
}
|
|
916
|
+
const shot = await captureImage(service, deviceId, sessionId, args.target, exec.signal);
|
|
257
917
|
// 有值才塞键:宿主不接受值为 undefined 的字段(见 A3-79 的坑)。
|
|
258
|
-
|
|
918
|
+
return {
|
|
259
919
|
deviceId,
|
|
260
|
-
width:
|
|
261
|
-
height:
|
|
920
|
+
width: shot.width,
|
|
921
|
+
height: shot.height,
|
|
262
922
|
note: mentionMismatch(sessionId, deviceId),
|
|
263
|
-
|
|
264
|
-
// 这一块会被原样留在历史里,下一轮请求按它取图。
|
|
265
|
-
image: {
|
|
266
|
-
type: 'image',
|
|
267
|
-
attachment: {
|
|
268
|
-
attachmentId: ref.attachmentId,
|
|
269
|
-
mediaType: ref.mediaType,
|
|
270
|
-
bytes: ref.bytes,
|
|
271
|
-
width: ref.width,
|
|
272
|
-
height: ref.height,
|
|
273
|
-
...(ref.name ? { name: ref.name } : {}),
|
|
274
|
-
},
|
|
275
|
-
},
|
|
923
|
+
image: shot.image,
|
|
276
924
|
};
|
|
277
|
-
return value;
|
|
278
925
|
},
|
|
279
926
|
});
|
|
280
927
|
|
|
@@ -297,10 +944,12 @@ function installPhoneTools(ctx, deps) {
|
|
|
297
944
|
// 元素是**任意形状的界面节点**(不同 App 给的字段不一样),所以 items 必须显式允许
|
|
298
945
|
// 附加字段 —— dsh-tools 的 schema 编译器不接受留空的 additionalProperties。
|
|
299
946
|
elements: { type: 'array', items: { type: 'object', additionalProperties: true } },
|
|
947
|
+
// 「没执行」的原因码:付款那一步没拿到用户确认时就只有它。
|
|
948
|
+
blocked: { type: 'string' },
|
|
300
949
|
note: { type: 'string' },
|
|
301
950
|
},
|
|
302
951
|
},
|
|
303
|
-
render: () => [{ type: 'text', text: '
|
|
952
|
+
render: (_args, value) => [{ type: 'text', text: value.ok === false ? `这一步没有执行:${value.note || '见说明'}。` : `已点击。\n${summarizeScreen(value)}` }],
|
|
304
953
|
},
|
|
305
954
|
execute: async (args, exec) => {
|
|
306
955
|
const sessionId = sessionOf(exec);
|
|
@@ -310,18 +959,45 @@ function installPhoneTools(ctx, deps) {
|
|
|
310
959
|
if (!hasRef && !hasPoint) throw new Error('要么给 ref(phone_screen 里的元素编号),要么给 x / y 百分比。');
|
|
311
960
|
if (hasRef && hasPoint) throw new Error('ref 和坐标只能给一样:有 ref 就按元素点。');
|
|
312
961
|
if (!hasRef && !textOf(args.target, MAX_TEXT)) throw new Error('按坐标点的时候要说明点的是什么(target)。');
|
|
313
|
-
const
|
|
314
|
-
|
|
962
|
+
const target = textOf(args.target, MAX_TEXT);
|
|
963
|
+
const ref = hasRef ? args.ref.trim() : '';
|
|
964
|
+
const tapArgs = {
|
|
965
|
+
ref,
|
|
315
966
|
x: hasPoint ? args.x : null,
|
|
316
967
|
y: hasPoint ? args.y : null,
|
|
317
|
-
target
|
|
318
|
-
}
|
|
319
|
-
|
|
968
|
+
target,
|
|
969
|
+
};
|
|
970
|
+
const notes = [mentionMismatch(sessionId, deviceId)];
|
|
971
|
+
|
|
972
|
+
// 付款 / 下单:先拿到人点的「同意」才按下去。判定用模型给的 ref + target,
|
|
973
|
+
// 而 target 是这一下点的是什么 —— 模型按 phone_screen 里的文字填,词表在
|
|
974
|
+
// irreversible.js,这里只认 kind === 'payment' 那一类。
|
|
975
|
+
const judged = judgeIrreversible('tap', { ref, x: args.x, y: args.y, target }, '');
|
|
976
|
+
if (judged && confirm.needsConfirmation(judged)) {
|
|
977
|
+
const blocked = await requirePaymentConfirmation(sessionId, exec, `在手机上点「${target || ref}」`);
|
|
978
|
+
if (blocked) {
|
|
979
|
+
// 不执行、也不抛:抛会把「问过用户、用户没同意」做成一次工具失败,模型会当成
|
|
980
|
+
// 手机坏了去重试。如实回一句「没做,为什么」才是它该看到的东西。
|
|
981
|
+
return { ok: false, blocked: 'payment-confirmation', note: blocked };
|
|
982
|
+
}
|
|
983
|
+
notes.push('这一步是付款 / 下单,用户已经确认过一次,就这一下有效。');
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
const { data, note: retried } = await runWithStaleRetry(service, deviceId, sessionId, 'tap', tapArgs, exec.signal);
|
|
987
|
+
if (retried) notes.push(retried);
|
|
988
|
+
// 单步本来就是做不可逆动作的正确姿势(批里会被熔断)。这里给一句确认,让模型知道
|
|
989
|
+
// 这一下为什么该单独做、做之前最好再核一眼。
|
|
990
|
+
if (judged && judged.irreversible && !confirm.needsConfirmation(judged)) {
|
|
991
|
+
notes.push(`这一步看起来不可逆(${judged.reason}),已经作为单独一步执行;接着看结果再决定下一步。`);
|
|
992
|
+
}
|
|
993
|
+
const value = {
|
|
320
994
|
ok: true,
|
|
321
995
|
packageName: textOf(data && data.packageName, MAX_TEXT),
|
|
322
996
|
elements: Array.isArray(data && data.elements) ? data.elements.slice(0, 120) : [],
|
|
323
|
-
note: mentionMismatch(sessionId, deviceId),
|
|
324
997
|
};
|
|
998
|
+
const note = notes.filter(Boolean).join(' ');
|
|
999
|
+
if (note) value.note = note;
|
|
1000
|
+
return value;
|
|
325
1001
|
},
|
|
326
1002
|
});
|
|
327
1003
|
|
|
@@ -346,24 +1022,28 @@ function installPhoneTools(ctx, deps) {
|
|
|
346
1022
|
note: { type: 'string' },
|
|
347
1023
|
},
|
|
348
1024
|
},
|
|
349
|
-
render: () => [{ type: 'text', text:
|
|
1025
|
+
render: (_args, value) => [{ type: 'text', text: `已输入。\n${summarizeScreen(value)}` }],
|
|
350
1026
|
},
|
|
351
1027
|
execute: async (args, exec) => {
|
|
352
1028
|
const sessionId = sessionOf(exec);
|
|
353
1029
|
const { deviceId, service } = targetOf(ctx, deps, sessionId);
|
|
354
1030
|
const value = typeof args.text === 'string' ? args.text : '';
|
|
355
1031
|
if (!value) throw new Error('要输入的文字是空的。');
|
|
356
|
-
const data = await
|
|
1032
|
+
const { data, note: retried } = await runWithStaleRetry(service, deviceId, sessionId, 'type', {
|
|
357
1033
|
text: value,
|
|
358
1034
|
ref: typeof args.ref === 'string' ? args.ref.trim() : '',
|
|
359
1035
|
submit: args.submit === true,
|
|
360
1036
|
}, exec.signal);
|
|
361
|
-
|
|
1037
|
+
const notes = [mentionMismatch(sessionId, deviceId)];
|
|
1038
|
+
if (retried) notes.push(retried);
|
|
1039
|
+
const result = {
|
|
362
1040
|
ok: true,
|
|
363
1041
|
packageName: textOf(data && data.packageName, MAX_TEXT),
|
|
364
1042
|
elements: Array.isArray(data && data.elements) ? data.elements.slice(0, 120) : [],
|
|
365
|
-
note: mentionMismatch(sessionId, deviceId),
|
|
366
1043
|
};
|
|
1044
|
+
const note = notes.filter(Boolean).join(' ');
|
|
1045
|
+
if (note) result.note = note;
|
|
1046
|
+
return result;
|
|
367
1047
|
},
|
|
368
1048
|
});
|
|
369
1049
|
|
|
@@ -387,7 +1067,7 @@ function installPhoneTools(ctx, deps) {
|
|
|
387
1067
|
note: { type: 'string' },
|
|
388
1068
|
},
|
|
389
1069
|
},
|
|
390
|
-
render: () => [{ type: 'text', text:
|
|
1070
|
+
render: (_args, value) => [{ type: 'text', text: `已滚动。\n${summarizeScreen(value)}` }],
|
|
391
1071
|
},
|
|
392
1072
|
execute: async (args, exec) => {
|
|
393
1073
|
const sessionId = sessionOf(exec);
|
|
@@ -407,9 +1087,9 @@ function installPhoneTools(ctx, deps) {
|
|
|
407
1087
|
|
|
408
1088
|
register({
|
|
409
1089
|
name: 'phone_key',
|
|
410
|
-
description: 'Press one system key on the phone: back, home, enter, or
|
|
1090
|
+
description: 'Press one system key on the phone: back, home, enter, recents, tab (next field), paste, selectall, or delete. Nothing else — the power and volume buttons are the user\'s own and are never pressed.',
|
|
411
1091
|
parameters: {
|
|
412
|
-
key: { type: 'string', required: true, enum: ['back', 'home', 'enter', 'recents'], description: 'Which key to press.' },
|
|
1092
|
+
key: { type: 'string', required: true, enum: ['back', 'home', 'enter', 'recents', 'tab', 'paste', 'selectall', 'delete'], description: 'Which key to press.' },
|
|
413
1093
|
},
|
|
414
1094
|
output: {
|
|
415
1095
|
schema: {
|
|
@@ -424,13 +1104,14 @@ function installPhoneTools(ctx, deps) {
|
|
|
424
1104
|
note: { type: 'string' },
|
|
425
1105
|
},
|
|
426
1106
|
},
|
|
427
|
-
render: () => [{ type: 'text', text:
|
|
1107
|
+
render: (_args, value) => [{ type: 'text', text: `已按键。\n${summarizeScreen(value)}` }],
|
|
428
1108
|
},
|
|
429
1109
|
execute: async (args, exec) => {
|
|
430
1110
|
const sessionId = sessionOf(exec);
|
|
431
1111
|
const { deviceId, service } = targetOf(ctx, deps, sessionId);
|
|
432
|
-
const
|
|
433
|
-
|
|
1112
|
+
const KEYS = ['back', 'home', 'enter', 'recents', 'tab', 'paste', 'selectall', 'delete'];
|
|
1113
|
+
const key = KEYS.includes(args.key) ? args.key : '';
|
|
1114
|
+
if (!key) throw new Error(`key 只能是 ${KEYS.join(' / ')}。`);
|
|
434
1115
|
const data = await run(service, deviceId, sessionId, 'key', { key }, exec.signal);
|
|
435
1116
|
return {
|
|
436
1117
|
ok: true,
|
|
@@ -514,6 +1195,451 @@ function installPhoneTools(ctx, deps) {
|
|
|
514
1195
|
},
|
|
515
1196
|
});
|
|
516
1197
|
|
|
1198
|
+
register({
|
|
1199
|
+
name: 'phone_steps',
|
|
1200
|
+
description: 'Run several phone actions in one call, executed in order on the phone. Use it to save round-trips when you already know the next few steps (open an app, tap a search box, type, tap search). Give 1-20 steps; each step has action + args and an optional until to wait for the next screen. Do NOT put a payment step (pay, order, buy, checkout) in a batch — the batch will stop right before it and you must do that one step on its own with phone_tap, because it needs the user to confirm. Sending, submitting and deleting are fine to batch. If a step fails the batch stops and tells you where.',
|
|
1201
|
+
parameters: {
|
|
1202
|
+
steps: {
|
|
1203
|
+
type: 'array',
|
|
1204
|
+
description: '1-20 steps run in order on the phone.',
|
|
1205
|
+
items: {
|
|
1206
|
+
type: 'object',
|
|
1207
|
+
additionalProperties: true,
|
|
1208
|
+
properties: {
|
|
1209
|
+
action: { type: 'string', description: 'One single action: tap, type, swipe, key, app.open, app.close, screen.read, wait, or gesture. Never batch.' },
|
|
1210
|
+
args: { type: 'object', additionalProperties: true, description: 'Arguments for that action, same fields as the matching single tool.' },
|
|
1211
|
+
until: {
|
|
1212
|
+
type: 'object',
|
|
1213
|
+
additionalProperties: true,
|
|
1214
|
+
properties: {
|
|
1215
|
+
appears: { type: 'string', description: 'Wait until an element containing this text appears (case-insensitive).' },
|
|
1216
|
+
gone: { type: 'string', description: 'Wait until this text is no longer on screen.' },
|
|
1217
|
+
app: { type: 'string', description: 'Wait until this package is in front.' },
|
|
1218
|
+
stable: { type: 'boolean', description: 'Wait until the screen stops changing.' },
|
|
1219
|
+
timeoutMs: { type: 'number', description: 'Max wait for this step, default 8000, max 20000.' },
|
|
1220
|
+
},
|
|
1221
|
+
},
|
|
1222
|
+
},
|
|
1223
|
+
},
|
|
1224
|
+
},
|
|
1225
|
+
quiet: { type: 'boolean', description: 'Only return the final screen (default true). Set false to also get the screen after each step.' },
|
|
1226
|
+
},
|
|
1227
|
+
output: {
|
|
1228
|
+
schema: {
|
|
1229
|
+
type: 'object',
|
|
1230
|
+
additionalProperties: false,
|
|
1231
|
+
properties: {
|
|
1232
|
+
ok: { type: 'boolean' },
|
|
1233
|
+
steps: { type: 'array', items: { type: 'object', additionalProperties: true } },
|
|
1234
|
+
stoppedAt: { type: 'number' },
|
|
1235
|
+
stopReason: { type: 'string' },
|
|
1236
|
+
packageName: { type: 'string' },
|
|
1237
|
+
elements: { type: 'array', items: { type: 'object', additionalProperties: true } },
|
|
1238
|
+
revision: { type: 'number' },
|
|
1239
|
+
note: { type: 'string' },
|
|
1240
|
+
},
|
|
1241
|
+
},
|
|
1242
|
+
render: (_args, value) => {
|
|
1243
|
+
const ran = Array.isArray(value.steps) ? value.steps.length : 0;
|
|
1244
|
+
const bad = value.stopReason ? `,在第 ${Number.isInteger(value.stoppedAt) ? value.stoppedAt + 1 : '?'} 步停下(${value.stopReason})` : '';
|
|
1245
|
+
return [{ type: 'text', text: `做了 ${ran} 步${bad}。` }];
|
|
1246
|
+
},
|
|
1247
|
+
},
|
|
1248
|
+
execute: async (args, exec) => {
|
|
1249
|
+
const sessionId = sessionOf(exec);
|
|
1250
|
+
const { deviceId, service } = targetOf(ctx, deps, sessionId);
|
|
1251
|
+
const raw = Array.isArray(args.steps) ? args.steps : [];
|
|
1252
|
+
if (raw.length < 1 || raw.length > 20) throw new Error('steps 要给 1 到 20 步。');
|
|
1253
|
+
|
|
1254
|
+
const steps = raw.map((step, i) => {
|
|
1255
|
+
if (!step || typeof step !== 'object') throw new Error(`第 ${i + 1} 步不是对象。`);
|
|
1256
|
+
const action = typeof step.action === 'string' ? step.action.trim() : '';
|
|
1257
|
+
if (!STEP_ACTIONS.has(action)) {
|
|
1258
|
+
throw new Error(`第 ${i + 1} 步的动作 ${action || '(空)'} 不能放进批量:只能是 ${[...STEP_ACTIONS].join(' / ')}。`);
|
|
1259
|
+
}
|
|
1260
|
+
const stepArgs = step.args && typeof step.args === 'object' ? step.args : {};
|
|
1261
|
+
const out = { action, args: stepArgs };
|
|
1262
|
+
if (step.until && typeof step.until === 'object') out.until = step.until;
|
|
1263
|
+
return out;
|
|
1264
|
+
});
|
|
1265
|
+
|
|
1266
|
+
// 批里只熔断**付款 / 下单**这一类:批内没有确认的位置,而这一步要用户亲自点一次同意。
|
|
1267
|
+
// 发送 / 提交 / 删除不再熔断 —— 它们是任务本身要做的事,每一步都停下来问会让「发条消息」
|
|
1268
|
+
// 变成一串确认框;那几种的代价由节奏控制拉间隔(协议 7.3)。
|
|
1269
|
+
// 找到第一处就把停点之前的部分交给手机,停点与之后都留下。
|
|
1270
|
+
let blockedAt = -1;
|
|
1271
|
+
let blockedReason = '';
|
|
1272
|
+
for (let i = 0; i < steps.length; i += 1) {
|
|
1273
|
+
const s = steps[i];
|
|
1274
|
+
const text = typeof s.args.target === 'string' ? s.args.target : (typeof s.args.text === 'string' ? s.args.text : '');
|
|
1275
|
+
const judged = judgeIrreversible(s.action, s.args, text);
|
|
1276
|
+
if (judged && confirm.needsConfirmation(judged)) { blockedAt = i; blockedReason = judged.reason || ''; break; }
|
|
1277
|
+
}
|
|
1278
|
+
const runnable = blockedAt >= 0 ? steps.slice(0, blockedAt) : steps;
|
|
1279
|
+
|
|
1280
|
+
const notes = [mentionMismatch(sessionId, deviceId)];
|
|
1281
|
+
if (blockedAt === 0) {
|
|
1282
|
+
// 第一步就要确认:这一批没有能跑的部分。不抛错 —— 抛错会被读成「批量坏了」,
|
|
1283
|
+
// 而事实是「这一步得单独做」,如实把这一批的结果写成「停在第一步」更贴切。
|
|
1284
|
+
const only = {
|
|
1285
|
+
ok: true,
|
|
1286
|
+
steps: [],
|
|
1287
|
+
stoppedAt: 0,
|
|
1288
|
+
stopReason: 'blocked-irreversible',
|
|
1289
|
+
note: `第 1 步是付款 / 下单(${blockedReason}),批里不执行它。请把这一步单独做(phone_tap),它会走一次用户确认;其余步骤再另起一批。`,
|
|
1290
|
+
};
|
|
1291
|
+
return only;
|
|
1292
|
+
}
|
|
1293
|
+
if (blockedAt > 0) {
|
|
1294
|
+
notes.push(`第 ${blockedAt + 1} 步是付款 / 下单(${blockedReason}),没有放进这一批:它要在这一批之后单独做,单独走一次确认。`);
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
const canBatch = supports(service, deviceId, 'batch');
|
|
1298
|
+
const stepResults = [];
|
|
1299
|
+
let stoppedAt = null;
|
|
1300
|
+
let stopReason = '';
|
|
1301
|
+
let finalData = null;
|
|
1302
|
+
|
|
1303
|
+
if (canBatch) {
|
|
1304
|
+
// 只把**停点之前**那几步交给手机。也可以把付款那步连同标记一起发过去、让手机自己停
|
|
1305
|
+
// (它认 `irreversible`),但那条路要求手机版本跟得上;发不过去的旧版本会直接把这步
|
|
1306
|
+
// 执行掉 —— 一次没人确认的付款不能押在「对方版本够新」上。切片在插件这边做,任何
|
|
1307
|
+
// 版本的手机都只会收到它该做的那几步。
|
|
1308
|
+
const batchArgs = {
|
|
1309
|
+
steps: runnable.map((s) => ({ action: s.action, args: s.args, ...(s.until ? { until: s.until } : {}) })),
|
|
1310
|
+
quiet: args.quiet !== false,
|
|
1311
|
+
};
|
|
1312
|
+
const data = await run(service, deviceId, sessionId, 'batch', batchArgs, exec.signal);
|
|
1313
|
+
finalData = data;
|
|
1314
|
+
const list = Array.isArray(data && data.steps) ? data.steps : [];
|
|
1315
|
+
for (const st of list) {
|
|
1316
|
+
const row = { index: st && st.index, action: st && st.action, ok: Boolean(st && st.ok === true) };
|
|
1317
|
+
if (st && Number.isFinite(st.waitedMs)) row.waitedMs = st.waitedMs;
|
|
1318
|
+
if (st && st.wait && typeof st.wait === 'object') row.wait = { met: st.wait.met === true, stopped: st.wait.stopped == null ? null : st.wait.stopped };
|
|
1319
|
+
if (st && typeof st.effect === 'string') row.effect = st.effect;
|
|
1320
|
+
if (st && st.error && typeof st.error === 'object') {
|
|
1321
|
+
row.error = {
|
|
1322
|
+
code: typeof st.error.code === 'string' ? st.error.code : 'failed',
|
|
1323
|
+
message: typeof st.error.message === 'string' ? st.error.message : '',
|
|
1324
|
+
};
|
|
1325
|
+
}
|
|
1326
|
+
stepResults.push(row);
|
|
1327
|
+
}
|
|
1328
|
+
if (data && data.stoppedAt != null) stoppedAt = data.stoppedAt;
|
|
1329
|
+
if (data && typeof data.stopReason === 'string' && data.stopReason) stopReason = data.stopReason;
|
|
1330
|
+
// 手机那边跑完了前面那几步、没别的原因停下:停点就是付款那一步(编号按原始 steps 算,
|
|
1331
|
+
// 因为切片是前缀,两边编号一致)。
|
|
1332
|
+
if (blockedAt >= 0 && stopReason === '') {
|
|
1333
|
+
stoppedAt = blockedAt;
|
|
1334
|
+
stopReason = 'blocked-irreversible';
|
|
1335
|
+
}
|
|
1336
|
+
} else {
|
|
1337
|
+
// 手机没报 batch 能力:退回逐步执行。慢一点,但任务能完成;并如实说一句。
|
|
1338
|
+
notes.push(`${unsupportedNote('一次多步(批量)')} 已改为逐步执行,会慢一些。`);
|
|
1339
|
+
const canNativeWait = supports(service, deviceId, 'screen.wait');
|
|
1340
|
+
for (let i = 0; i < runnable.length; i += 1) {
|
|
1341
|
+
if (i > 0) await sleep(betweenDelay());
|
|
1342
|
+
const s = runnable[i];
|
|
1343
|
+
try {
|
|
1344
|
+
await run(service, deviceId, sessionId, s.action, s.args, exec.signal);
|
|
1345
|
+
} catch (error) {
|
|
1346
|
+
const message = String((error && error.message) || error);
|
|
1347
|
+
const code = /不支持/.test(message) ? 'unsupported' : (/断开/.test(message) ? 'offline' : 'step-failed');
|
|
1348
|
+
stepResults.push({ index: i, action: s.action, ok: false, error: { code, message } });
|
|
1349
|
+
stoppedAt = i;
|
|
1350
|
+
stopReason = 'step-failed';
|
|
1351
|
+
break;
|
|
1352
|
+
}
|
|
1353
|
+
const row = { index: i, action: s.action, ok: true };
|
|
1354
|
+
if (s.until) {
|
|
1355
|
+
const w = await waitCondition(service, deviceId, sessionId, s.until, exec.signal, canNativeWait);
|
|
1356
|
+
row.waitedMs = w.waitedMs;
|
|
1357
|
+
row.wait = { met: w.met, stopped: w.stopped == null ? null : w.stopped };
|
|
1358
|
+
}
|
|
1359
|
+
stepResults.push(row);
|
|
1360
|
+
}
|
|
1361
|
+
try {
|
|
1362
|
+
finalData = await run(service, deviceId, sessionId, 'screen.read', { includeText: true }, exec.signal);
|
|
1363
|
+
} catch { /* 拿不到最终屏就给空,别把整批判成失败 */ }
|
|
1364
|
+
// 逐步执行这条路上停点由插件自己定:跑完了能跑的那几步、停在付款那一步之前。
|
|
1365
|
+
if (blockedAt >= 0 && stopReason === '') {
|
|
1366
|
+
stoppedAt = runnable.length;
|
|
1367
|
+
stopReason = 'blocked-irreversible';
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
const value = { ok: stopReason !== 'step-failed', steps: stepResults };
|
|
1372
|
+
if (stoppedAt !== null) value.stoppedAt = stoppedAt;
|
|
1373
|
+
if (stopReason) value.stopReason = stopReason;
|
|
1374
|
+
if (finalData && typeof finalData.packageName === 'string') value.packageName = textOf(finalData.packageName, MAX_TEXT);
|
|
1375
|
+
if (finalData && Array.isArray(finalData.elements)) value.elements = finalData.elements.slice(0, 200);
|
|
1376
|
+
if (finalData && Number.isFinite(finalData.revision)) value.revision = finalData.revision;
|
|
1377
|
+
// 批量结果同样附一次 App 用法知识:`noteFor` 的去重账与 phone_screen 是同一份,
|
|
1378
|
+
// 同一对话同一 App 不管先从哪条路看到,都只附一遍(协议 7.5)。
|
|
1379
|
+
if (phoneKnowledge && typeof phoneKnowledge.noteFor === 'function') {
|
|
1380
|
+
try {
|
|
1381
|
+
const knowledge = phoneKnowledge.noteFor(sessionId, finalData && finalData.packageName);
|
|
1382
|
+
if (knowledge) notes.push(knowledge);
|
|
1383
|
+
} catch { /* 知识是锦上添花,读坏了不影响这一批的结果 */ }
|
|
1384
|
+
}
|
|
1385
|
+
const note = notes.filter((n) => typeof n === 'string' && n).join(' ');
|
|
1386
|
+
if (note) value.note = note;
|
|
1387
|
+
return value;
|
|
1388
|
+
},
|
|
1389
|
+
});
|
|
1390
|
+
|
|
1391
|
+
register({
|
|
1392
|
+
name: 'phone_wait',
|
|
1393
|
+
description: 'Wait until the screen reaches a condition, using the phone\'s accessibility events instead of polling. Use it after an action that loads something (a search, a page open) instead of taking repeated screenshots. Give at least one of appears / gone / app / stable. Returns met=true or met=false on timeout.',
|
|
1394
|
+
parameters: {
|
|
1395
|
+
appears: { type: 'string', description: 'Wait until an element containing this text appears (case-insensitive).' },
|
|
1396
|
+
gone: { type: 'string', description: 'Wait until this text is no longer on screen.' },
|
|
1397
|
+
app: { type: 'string', description: 'Wait until this package is in front.' },
|
|
1398
|
+
stable: { type: 'boolean', description: 'Wait until the screen stops changing.' },
|
|
1399
|
+
timeoutMs: { type: 'number', description: 'Max wait in ms, default 8000, max 20000.' },
|
|
1400
|
+
},
|
|
1401
|
+
output: {
|
|
1402
|
+
schema: {
|
|
1403
|
+
type: 'object',
|
|
1404
|
+
additionalProperties: false,
|
|
1405
|
+
properties: {
|
|
1406
|
+
met: { type: 'boolean' },
|
|
1407
|
+
stopped: { type: 'string' },
|
|
1408
|
+
waitedMs: { type: 'number' },
|
|
1409
|
+
note: { type: 'string' },
|
|
1410
|
+
},
|
|
1411
|
+
},
|
|
1412
|
+
render: (_args, value) => [{ type: 'text', text: value.met ? `条件成立(等了 ${value.waitedMs || 0}ms)。` : '等超时了,条件没成立。' }],
|
|
1413
|
+
},
|
|
1414
|
+
execute: async (args, exec) => {
|
|
1415
|
+
const sessionId = sessionOf(exec);
|
|
1416
|
+
const { deviceId, service } = targetOf(ctx, deps, sessionId);
|
|
1417
|
+
const until = {};
|
|
1418
|
+
if (typeof args.appears === 'string' && args.appears.trim()) until.appears = args.appears;
|
|
1419
|
+
if (typeof args.gone === 'string' && args.gone.trim()) until.gone = args.gone;
|
|
1420
|
+
if (typeof args.app === 'string' && args.app.trim()) until.app = args.app;
|
|
1421
|
+
if (args.stable === true) until.stable = true;
|
|
1422
|
+
until.timeoutMs = clampTimeoutMs(args.timeoutMs);
|
|
1423
|
+
if (until.appears === undefined && until.gone === undefined && until.app === undefined && until.stable !== true) {
|
|
1424
|
+
throw new Error('要等什么?给 appears / gone / app / stable 其中至少一个。');
|
|
1425
|
+
}
|
|
1426
|
+
const canNative = supports(service, deviceId, 'screen.wait');
|
|
1427
|
+
const w = await waitCondition(service, deviceId, sessionId, until, exec.signal, canNative);
|
|
1428
|
+
const value = { met: w.met };
|
|
1429
|
+
if (w.stopped != null) value.stopped = w.stopped;
|
|
1430
|
+
if (Number.isFinite(w.waitedMs)) value.waitedMs = w.waitedMs;
|
|
1431
|
+
const notes = [mentionMismatch(sessionId, deviceId)];
|
|
1432
|
+
if (!canNative) notes.push(`${unsupportedNote('等条件(screen.wait)')} 这次是轮询读屏等到的,会慢一些。`);
|
|
1433
|
+
const note = notes.filter(Boolean).join(' ');
|
|
1434
|
+
if (note) value.note = note;
|
|
1435
|
+
return value;
|
|
1436
|
+
},
|
|
1437
|
+
});
|
|
1438
|
+
|
|
1439
|
+
register({
|
|
1440
|
+
name: 'phone_gesture',
|
|
1441
|
+
description: 'Send a finger gesture the simple tools cannot express: a long press, a drag along a path, or a two-finger pinch. Coordinates are viewport percentages (x 0-100 left-to-right, y 0-100 top-to-bottom). Give 1-2 strokes; each stroke is a list of points; a long press is one point with holdMs 500-800; two strokes at once is a pinch.',
|
|
1442
|
+
parameters: {
|
|
1443
|
+
strokes: {
|
|
1444
|
+
type: 'array',
|
|
1445
|
+
description: '1-2 finger strokes, done at the same time.',
|
|
1446
|
+
items: {
|
|
1447
|
+
type: 'object',
|
|
1448
|
+
additionalProperties: true,
|
|
1449
|
+
properties: {
|
|
1450
|
+
points: {
|
|
1451
|
+
type: 'array',
|
|
1452
|
+
description: '2-32 points the finger moves through, each { x, y } in 0-100 percent.',
|
|
1453
|
+
items: {
|
|
1454
|
+
type: 'object',
|
|
1455
|
+
additionalProperties: true,
|
|
1456
|
+
properties: {
|
|
1457
|
+
x: { type: 'number', description: 'Horizontal position, 0-100 percent of screen width.' },
|
|
1458
|
+
y: { type: 'number', description: 'Vertical position, 0-100 percent of screen height.' },
|
|
1459
|
+
},
|
|
1460
|
+
},
|
|
1461
|
+
},
|
|
1462
|
+
holdMs: { type: 'number', description: 'How long to hold at the first point before moving, in ms. Use 500-800 for a long press.' },
|
|
1463
|
+
startMs: { type: 'number', description: 'Delay before this stroke starts, in ms (to stagger strokes).' },
|
|
1464
|
+
},
|
|
1465
|
+
},
|
|
1466
|
+
},
|
|
1467
|
+
},
|
|
1468
|
+
output: {
|
|
1469
|
+
schema: {
|
|
1470
|
+
type: 'object',
|
|
1471
|
+
additionalProperties: false,
|
|
1472
|
+
properties: {
|
|
1473
|
+
ok: { type: 'boolean' },
|
|
1474
|
+
packageName: { type: 'string' },
|
|
1475
|
+
elements: { type: 'array', items: { type: 'object', additionalProperties: true } },
|
|
1476
|
+
note: { type: 'string' },
|
|
1477
|
+
},
|
|
1478
|
+
},
|
|
1479
|
+
render: () => [{ type: 'text', text: '已做手势。' }],
|
|
1480
|
+
},
|
|
1481
|
+
execute: async (args, exec) => {
|
|
1482
|
+
const sessionId = sessionOf(exec);
|
|
1483
|
+
const { deviceId, service } = targetOf(ctx, deps, sessionId);
|
|
1484
|
+
const raw = Array.isArray(args.strokes) ? args.strokes : [];
|
|
1485
|
+
if (raw.length < 1 || raw.length > 2) throw new Error('strokes 要给 1 到 2 段。');
|
|
1486
|
+
const strokes = raw.map((stroke, i) => {
|
|
1487
|
+
if (!stroke || typeof stroke !== 'object') throw new Error(`第 ${i + 1} 段手势格式不对,要是 { points: [...] }。`);
|
|
1488
|
+
const pts = Array.isArray(stroke.points) ? stroke.points : [];
|
|
1489
|
+
if (pts.length < 2 || pts.length > 32) throw new Error(`第 ${i + 1} 段手势要有 2 到 32 个点。`);
|
|
1490
|
+
const points = pts.map((p, j) => {
|
|
1491
|
+
const x = Number(p && p.x);
|
|
1492
|
+
const y = Number(p && p.y);
|
|
1493
|
+
if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || x > 100 || y < 0 || y > 100) {
|
|
1494
|
+
throw new Error(`第 ${i + 1} 段第 ${j + 1} 个点要是 0 到 100 的百分比坐标。`);
|
|
1495
|
+
}
|
|
1496
|
+
return { x, y };
|
|
1497
|
+
});
|
|
1498
|
+
const out = { points };
|
|
1499
|
+
if (Number.isFinite(stroke.holdMs)) out.holdMs = Math.max(0, Math.min(5000, Math.round(stroke.holdMs)));
|
|
1500
|
+
if (Number.isFinite(stroke.startMs)) out.startMs = Math.max(0, Math.min(3000, Math.round(stroke.startMs)));
|
|
1501
|
+
return out;
|
|
1502
|
+
});
|
|
1503
|
+
const data = await run(service, deviceId, sessionId, 'gesture', { strokes }, exec.signal);
|
|
1504
|
+
const notes = [mentionMismatch(sessionId, deviceId)];
|
|
1505
|
+
if (!supports(service, deviceId, 'gesture.multi')) notes.push(`${unsupportedNote('多指手势')} 单段手势仍会尝试,失败就以提示为准。`);
|
|
1506
|
+
const value = {
|
|
1507
|
+
ok: true,
|
|
1508
|
+
packageName: textOf(data && data.packageName, MAX_TEXT),
|
|
1509
|
+
elements: Array.isArray(data && data.elements) ? data.elements.slice(0, 120) : [],
|
|
1510
|
+
};
|
|
1511
|
+
const note = notes.filter(Boolean).join(' ');
|
|
1512
|
+
if (note) value.note = note;
|
|
1513
|
+
return value;
|
|
1514
|
+
},
|
|
1515
|
+
});
|
|
1516
|
+
|
|
1517
|
+
register({
|
|
1518
|
+
name: 'phone_open_link',
|
|
1519
|
+
description: 'Open a link or a system page by kind. kind=uri opens a web or app link (only http/https and common app schemes); kind=settings opens one standard Android settings page from a fixed list; kind=package opens an app\'s details page in system settings. Use it for system settings and app-info pages instead of hunting through menus. It cannot open arbitrary intents.',
|
|
1520
|
+
parameters: {
|
|
1521
|
+
kind: { type: 'string', required: true, enum: ['uri', 'settings', 'package'], description: 'What to open.' },
|
|
1522
|
+
uri: { type: 'string', description: 'For kind=uri: the link, e.g. https://example.com or a common app scheme.' },
|
|
1523
|
+
action: { type: 'string', description: 'For kind=settings: the Android settings action name, e.g. android.settings.WIFI_SETTINGS.' },
|
|
1524
|
+
packageName: { type: 'string', description: 'For kind=package (and app-details settings): the Android package name.' },
|
|
1525
|
+
},
|
|
1526
|
+
output: {
|
|
1527
|
+
schema: {
|
|
1528
|
+
type: 'object',
|
|
1529
|
+
additionalProperties: false,
|
|
1530
|
+
properties: {
|
|
1531
|
+
ok: { type: 'boolean' },
|
|
1532
|
+
packageName: { type: 'string' },
|
|
1533
|
+
elements: { type: 'array', items: { type: 'object', additionalProperties: true } },
|
|
1534
|
+
note: { type: 'string' },
|
|
1535
|
+
},
|
|
1536
|
+
},
|
|
1537
|
+
render: () => [{ type: 'text', text: '已打开链接/页面。' }],
|
|
1538
|
+
},
|
|
1539
|
+
execute: async (args, exec) => {
|
|
1540
|
+
const sessionId = sessionOf(exec);
|
|
1541
|
+
const { deviceId, service } = targetOf(ctx, deps, sessionId);
|
|
1542
|
+
const kind = ['uri', 'settings', 'package'].includes(args.kind) ? args.kind : '';
|
|
1543
|
+
if (!kind) throw new Error('kind 只能是 uri / settings / package。');
|
|
1544
|
+
const linkArgs = { kind };
|
|
1545
|
+
if (kind === 'uri') {
|
|
1546
|
+
const uri = textOf(args.uri, 2000);
|
|
1547
|
+
if (!uri) throw new Error('kind=uri 时要给 uri。');
|
|
1548
|
+
const scheme = uri.includes(':') ? uri.split(':')[0].toLowerCase() : '';
|
|
1549
|
+
if (!URI_SCHEMES.has(scheme)) {
|
|
1550
|
+
throw new Error(`不打开这个 scheme(${scheme || '无'})。只放行 http / https / tel / mailto / sms / geo / market 这类常见入口。`);
|
|
1551
|
+
}
|
|
1552
|
+
linkArgs.uri = uri;
|
|
1553
|
+
} else if (kind === 'settings') {
|
|
1554
|
+
const action = textOf(args.action, 200);
|
|
1555
|
+
if (!SETTINGS_ACTIONS.has(action)) throw new Error(`这个设置入口不在白名单里:${action || '(空)'}。`);
|
|
1556
|
+
linkArgs.action = action;
|
|
1557
|
+
if (action === 'android.settings.APPLICATION_DETAILS_SETTINGS') {
|
|
1558
|
+
const pkg = textOf(args.packageName, 200);
|
|
1559
|
+
if (!/^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z0-9_]+)+$/.test(pkg)) throw new Error('打开某个 App 的详情页时要带合法的 packageName。');
|
|
1560
|
+
linkArgs.packageName = pkg;
|
|
1561
|
+
}
|
|
1562
|
+
} else {
|
|
1563
|
+
const pkg = textOf(args.packageName, 200);
|
|
1564
|
+
if (!/^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z0-9_]+)+$/.test(pkg)) throw new Error('packageName 看起来不像安卓包名。');
|
|
1565
|
+
linkArgs.packageName = pkg;
|
|
1566
|
+
}
|
|
1567
|
+
const data = await run(service, deviceId, sessionId, 'app.link', linkArgs, exec.signal);
|
|
1568
|
+
const notes = [mentionMismatch(sessionId, deviceId)];
|
|
1569
|
+
if (!supports(service, deviceId, 'app.link')) notes.push(unsupportedNote('按类别打开链接(app.link)'));
|
|
1570
|
+
const value = {
|
|
1571
|
+
ok: true,
|
|
1572
|
+
packageName: textOf(data && data.packageName, MAX_TEXT),
|
|
1573
|
+
elements: Array.isArray(data && data.elements) ? data.elements.slice(0, 120) : [],
|
|
1574
|
+
};
|
|
1575
|
+
const note = notes.filter(Boolean).join(' ');
|
|
1576
|
+
if (note) value.note = note;
|
|
1577
|
+
return value;
|
|
1578
|
+
},
|
|
1579
|
+
});
|
|
1580
|
+
|
|
1581
|
+
register({
|
|
1582
|
+
name: 'phone_scroll_to',
|
|
1583
|
+
description: 'Scroll the screen until an element containing the given text is visible, then return its ref so you can tap it. It reads and swipes at most 8 times. Use it for a long list or search results when the target is not on the current screen; do not use it to read a whole page (use phone_screen). If it returns found=false, the text was not reached — say so rather than guessing.',
|
|
1584
|
+
parameters: {
|
|
1585
|
+
text: { type: 'string', required: true, description: 'Text on the element to scroll to (case-insensitive substring).' },
|
|
1586
|
+
direction: { type: 'string', enum: ['up', 'down'], description: 'Where the target is: down means scroll down to see lower content (default down).' },
|
|
1587
|
+
maxScrolls: { type: 'integer', description: 'How many swipes to try at most, 1-8 (default 8).' },
|
|
1588
|
+
},
|
|
1589
|
+
output: {
|
|
1590
|
+
schema: {
|
|
1591
|
+
type: 'object',
|
|
1592
|
+
additionalProperties: false,
|
|
1593
|
+
properties: {
|
|
1594
|
+
found: { type: 'boolean' },
|
|
1595
|
+
ref: { type: 'string' },
|
|
1596
|
+
text: { type: 'string' },
|
|
1597
|
+
packageName: { type: 'string' },
|
|
1598
|
+
scrolls: { type: 'number' },
|
|
1599
|
+
elements: { type: 'array', items: { type: 'object', additionalProperties: true } },
|
|
1600
|
+
note: { type: 'string' },
|
|
1601
|
+
},
|
|
1602
|
+
},
|
|
1603
|
+
render: (_args, value) => [{ type: 'text', text: value.found ? `找到了(${value.text || value.ref})。` : '滚到头也没找到。' }],
|
|
1604
|
+
},
|
|
1605
|
+
execute: async (args, exec) => {
|
|
1606
|
+
const sessionId = sessionOf(exec);
|
|
1607
|
+
const { deviceId, service } = targetOf(ctx, deps, sessionId);
|
|
1608
|
+
const query = textOf(args.text, 200);
|
|
1609
|
+
if (!query) throw new Error('要滚到哪个元素?给 text(元素上的文字)。');
|
|
1610
|
+
const direction = args.direction === 'up' ? 'up' : 'down';
|
|
1611
|
+
const maxScrolls = Number.isInteger(args.maxScrolls) ? Math.min(8, Math.max(1, args.maxScrolls)) : 8;
|
|
1612
|
+
const needle = query.toLowerCase();
|
|
1613
|
+
let found = null;
|
|
1614
|
+
let scrolls = 0;
|
|
1615
|
+
let packageName = '';
|
|
1616
|
+
let lastSig = '';
|
|
1617
|
+
let unchanged = 0;
|
|
1618
|
+
let elements = [];
|
|
1619
|
+
for (let i = 0; ; i += 1) {
|
|
1620
|
+
const data = await run(service, deviceId, sessionId, 'screen.read', { includeText: true }, exec.signal);
|
|
1621
|
+
packageName = textOf(data && data.packageName, MAX_TEXT) || packageName;
|
|
1622
|
+
elements = Array.isArray(data && data.elements) ? data.elements : [];
|
|
1623
|
+
found = elements.find((el) => elementText(el).toLowerCase().includes(needle)) || null;
|
|
1624
|
+
if (found) break;
|
|
1625
|
+
const sig = screenSignature({ packageName: data && data.packageName, elements });
|
|
1626
|
+
unchanged = sig === lastSig ? unchanged + 1 : 0;
|
|
1627
|
+
lastSig = sig;
|
|
1628
|
+
if (unchanged >= 2 || scrolls >= maxScrolls) break;
|
|
1629
|
+
await run(service, deviceId, sessionId, 'swipe', { direction, amount: 0.7 }, exec.signal);
|
|
1630
|
+
scrolls += 1;
|
|
1631
|
+
}
|
|
1632
|
+
const notes = [mentionMismatch(sessionId, deviceId)];
|
|
1633
|
+
if (!found) notes.push(`滚了 ${scrolls} 次也没找到「${query}」,如实说没找到,别猜它在哪。`);
|
|
1634
|
+
const value = { found: Boolean(found), scrolls, packageName, elements: elements.slice(0, 120) };
|
|
1635
|
+
if (found && found.ref) value.ref = String(found.ref);
|
|
1636
|
+
if (found) value.text = elementText(found).slice(0, MAX_TEXT);
|
|
1637
|
+
const note = notes.filter(Boolean).join(' ');
|
|
1638
|
+
if (note) value.note = note;
|
|
1639
|
+
return value;
|
|
1640
|
+
},
|
|
1641
|
+
});
|
|
1642
|
+
|
|
517
1643
|
register({
|
|
518
1644
|
name: 'phone_target',
|
|
519
1645
|
description: 'Show or change which phone this conversation operates. Without arguments it reports the binding and the phones currently connected. With a deviceId it rebinds — only do that when the user asked to switch phones.',
|