@mobileaidev/ai-app-bridge 0.2.15 → 0.3.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/LICENSE +201 -0
  2. package/NOTICE +7 -0
  3. package/README.md +306 -53
  4. package/bin/ai-app-bridge.js +51 -4504
  5. package/bin/android-permissions.js +152 -0
  6. package/bin/android-uia-xml.js +74 -0
  7. package/bin/artifact-paths.js +127 -1
  8. package/bin/bridge-forward.js +56 -0
  9. package/bin/command-discovery.js +86 -0
  10. package/bin/command-errors.js +60 -0
  11. package/bin/command-registry.js +504 -0
  12. package/bin/command-request.js +14 -0
  13. package/bin/command-router.js +80 -0
  14. package/bin/connection-cache.js +119 -0
  15. package/bin/device-provider.js +2934 -0
  16. package/bin/execution-host.js +596 -0
  17. package/bin/execution-runtime.js +101 -0
  18. package/bin/fact-codec.js +321 -0
  19. package/bin/fact-recorder.js +691 -0
  20. package/bin/fact-store.js +226 -0
  21. package/bin/feedback-probe.js +285 -0
  22. package/bin/intent/install-intent.js +267 -0
  23. package/bin/intent/intent-action-executor.js +129 -0
  24. package/bin/intent/intent-autonomous-adapter.js +46 -0
  25. package/bin/intent/intent-capture-port.js +64 -0
  26. package/bin/intent/intent-entry.js +226 -0
  27. package/bin/intent/intent-errors.js +28 -0
  28. package/bin/intent/intent-evidence-store.js +121 -0
  29. package/bin/intent/intent-lifetime.js +62 -0
  30. package/bin/intent/intent-observation-target.js +33 -0
  31. package/bin/intent/intent-observer.js +194 -0
  32. package/bin/intent/intent-production-adapter.js +334 -0
  33. package/bin/intent/intent-provider.js +27 -0
  34. package/bin/intent/intent-runtime.js +63 -0
  35. package/bin/intent/intent-worker.js +432 -0
  36. package/bin/intent/ios-intent-adapter.js +90 -0
  37. package/bin/intent/permission-intent.js +239 -0
  38. package/bin/intent/web-intent-adapter.js +31 -0
  39. package/bin/ios-device-outcome.js +47 -0
  40. package/bin/ios-execution.js +108 -0
  41. package/bin/ios-provider.js +497 -583
  42. package/bin/ios-runtime-binding.js +54 -0
  43. package/bin/ios-wda-execution.js +70 -0
  44. package/bin/ios-wda-port.js +98 -0
  45. package/bin/ios-wda-project.js +79 -0
  46. package/bin/mcp-server.js +83 -1040
  47. package/bin/mmap-scan-index.js +329 -0
  48. package/bin/observation-collector.js +862 -0
  49. package/bin/runtime-client.js +154 -0
  50. package/bin/runtime-directory.js +107 -0
  51. package/bin/runtime-protocol.js +36 -0
  52. package/bin/script/bounded-script-registry.js +103 -0
  53. package/bin/script/node-runtime-adapter.js +233 -0
  54. package/bin/script/progress-projector.js +63 -0
  55. package/bin/script/python-runtime-adapter.js +111 -0
  56. package/bin/script/rolling-summary.js +134 -0
  57. package/bin/script/script-agent-port.js +40 -0
  58. package/bin/script/script-assert.js +95 -0
  59. package/bin/script/script-capture-port.js +76 -0
  60. package/bin/script/script-catalog.js +85 -0
  61. package/bin/script/script-durable-restore.js +195 -0
  62. package/bin/script/script-entry-code.js +26 -0
  63. package/bin/script/script-entry-route.js +38 -0
  64. package/bin/script/script-entry.js +3 -0
  65. package/bin/script/script-errors.js +29 -0
  66. package/bin/script/script-evidence-store.js +22 -0
  67. package/bin/script/script-format-removed.js +26 -0
  68. package/bin/script/script-host-port.js +397 -0
  69. package/bin/script/script-ledger.js +64 -0
  70. package/bin/script/script-result.js +57 -0
  71. package/bin/script/script-sdk.js +152 -0
  72. package/bin/script/script-sdk.py +153 -0
  73. package/bin/script/script-session-channel.js +127 -0
  74. package/bin/script/script-spec.js +86 -0
  75. package/bin/script/script-supervisor.js +919 -0
  76. package/bin/script/templates/checkpoint-reentry.js +13 -0
  77. package/bin/segment-index.js +481 -0
  78. package/bin/segmented-fact-store.js +1571 -0
  79. package/bin/shared-kernel/android-h5-target.js +10 -0
  80. package/bin/shared-kernel/android-install-execution.js +176 -0
  81. package/bin/shared-kernel/android-sdk-endpoint.js +42 -0
  82. package/bin/shared-kernel/android-shell-execution.js +195 -0
  83. package/bin/shared-kernel/argument-schema.js +117 -0
  84. package/bin/shared-kernel/canonical-path.js +17 -0
  85. package/bin/shared-kernel/device-acknowledgements.js +53 -0
  86. package/bin/shared-kernel/device-completion-history.js +52 -0
  87. package/bin/shared-kernel/device-mutation-lease.js +219 -0
  88. package/bin/shared-kernel/device-ownership-recovery.js +95 -0
  89. package/bin/shared-kernel/device-ownership-store.js +94 -0
  90. package/bin/shared-kernel/evidence-adapters.js +251 -0
  91. package/bin/shared-kernel/evidence-archive.js +329 -0
  92. package/bin/shared-kernel/evidence-recording.js +131 -0
  93. package/bin/shared-kernel/evidence-schema.js +194 -0
  94. package/bin/shared-kernel/evidence-store.js +190 -0
  95. package/bin/shared-kernel/execution-admission.js +22 -0
  96. package/bin/shared-kernel/execution-contracts.js +171 -0
  97. package/bin/shared-kernel/execution-io.js +106 -0
  98. package/bin/shared-kernel/execution-ledger.js +125 -0
  99. package/bin/shared-kernel/execution-scope.js +87 -0
  100. package/bin/shared-kernel/execution-target.js +115 -0
  101. package/bin/shared-kernel/flutter-execution.js +13 -0
  102. package/bin/shared-kernel/flutter-h5-port.js +60 -0
  103. package/bin/shared-kernel/flutter-h5-target.js +9 -0
  104. package/bin/shared-kernel/flutter-target.js +75 -0
  105. package/bin/shared-kernel/h5-execution.js +11 -0
  106. package/bin/shared-kernel/h5-target.js +31 -0
  107. package/bin/shared-kernel/host-fact-store.js +49 -0
  108. package/bin/shared-kernel/ios-h5-target.js +9 -0
  109. package/bin/shared-kernel/ios-native-target.js +71 -0
  110. package/bin/shared-kernel/live-capture-query.js +115 -0
  111. package/bin/shared-kernel/managed-sdk-execution.js +78 -0
  112. package/bin/shared-kernel/native-execution.js +13 -0
  113. package/bin/shared-kernel/native-target.js +156 -0
  114. package/bin/shared-kernel/provider-command-contracts.js +55 -0
  115. package/bin/shared-kernel/recorded-payload-archive.js +195 -0
  116. package/bin/shared-kernel/request-context.js +35 -0
  117. package/bin/shared-kernel/semantic-node.js +55 -0
  118. package/bin/shared-kernel/summary-transformer.js +352 -0
  119. package/bin/shared-kernel/target-lease-protocol.js +47 -0
  120. package/bin/shared-kernel/text-wait.js +111 -0
  121. package/bin/shared-kernel/uia-execution.js +96 -0
  122. package/bin/shared-kernel/uia-protocol.js +214 -0
  123. package/bin/shared-kernel/uia-runtime-port.js +377 -0
  124. package/bin/shared-kernel/uia-target.js +39 -0
  125. package/bin/shared-kernel/web-dom-target.js +44 -0
  126. package/bin/shared-kernel/xml-attributes.js +25 -0
  127. package/bin/target-execution.js +275 -0
  128. package/bin/web/command-schema.js +60 -0
  129. package/bin/web/session-store.js +157 -0
  130. package/bin/web-provider.js +334 -553
  131. package/docs/COMMAND_CONTRACT.md +1563 -0
  132. package/docs/EVIDENCE_ARCHIVE.md +214 -0
  133. package/docs/INTENT_FOREGROUND.md +71 -0
  134. package/docs/INTENT_NATIVE_EDITING.md +79 -0
  135. package/docs/RELEASE.md +59 -0
  136. package/docs/SCRIPT_AUTHORING.md +489 -0
  137. package/node_modules/@mobileaidev/segmented-fact-store-native/LICENSE +201 -0
  138. package/node_modules/@mobileaidev/segmented-fact-store-native/NOTICE +7 -0
  139. package/node_modules/@mobileaidev/segmented-fact-store-native/binding.gyp +36 -0
  140. package/node_modules/@mobileaidev/segmented-fact-store-native/bindings/node/sfs_node.c +597 -0
  141. package/node_modules/@mobileaidev/segmented-fact-store-native/include/sfs.h +178 -0
  142. package/node_modules/@mobileaidev/segmented-fact-store-native/index.js +5 -0
  143. package/node_modules/@mobileaidev/segmented-fact-store-native/package.json +24 -0
  144. package/node_modules/@mobileaidev/segmented-fact-store-native/src/sfs.c +2349 -0
  145. package/package.json +60 -5
  146. package/runtime/ios-wda/AABWDABinding.h +19 -0
  147. package/runtime/ios-wda/AABWDABinding.m +97 -0
  148. package/runtime/ios-wda/AABWDAExecution.h +26 -0
  149. package/runtime/ios-wda/AABWDAExecution.m +172 -0
  150. package/runtime/ios-wda/AABWDAIntegration.h +71 -0
  151. package/runtime/ios-wda/AABWDAManagedRoutes.h +392 -0
  152. package/runtime/ios-wda/AABWDAReceiptStore.h +10 -0
  153. package/runtime/ios-wda/AABWDAReceiptStore.m +116 -0
  154. package/runtime/uia/ai-app-bridge-uia.jar +0 -0
  155. package/runtime/uia/manifest.json +22 -0
  156. package/skills/ai-app-bridge-use/SKILL.md +23 -360
@@ -0,0 +1,352 @@
1
+ 'use strict';
2
+
3
+ const { foregroundNativeWindow } = require('./native-target');
4
+
5
+ const { semanticNode } = require('./semantic-node');
6
+ const { parseXmlAttributes } = require('./xml-attributes');
7
+ const iosNative = require('./ios-native-target');
8
+ const uia = require('./uia-protocol');
9
+
10
+ const DEFAULT_MAX_BYTES = 64 * 1024;
11
+ const HARD_LIMIT_MS = 50;
12
+
13
+ function summarizeTree({
14
+ provider,
15
+ rawTree,
16
+ rawTreeId,
17
+ screenshotId = null,
18
+ now = Date.now,
19
+ maxBytes = DEFAULT_MAX_BYTES,
20
+ hardLimitMs = HARD_LIMIT_MS,
21
+ } = {}) {
22
+ if (!rawTreeId) {
23
+ return { ok: false, error: 'rawTreeId_required' };
24
+ }
25
+ if (!['native', 'uia', 'flutter', 'h5'].includes(provider)) {
26
+ return { ok: false, error: 'invalid_provider' };
27
+ }
28
+ const startedAtMs = now();
29
+ const collected = [];
30
+ let visited = 0;
31
+ let truncated = false;
32
+ let reason = null;
33
+ const page = provider === 'h5' && ([require('./ios-h5-target').schema, require('./android-h5-target').schema].includes(rawTree?.h5TargetSchema)
34
+ || rawTree?.webTargetSchema === require('./web-dom-target').schema)
35
+ ? { ...rawTree.pageRef, title: rawTree.dom.title.slice(0,500), readyState: rawTree.dom.readyState,
36
+ ...(rawTree.dom.viewport === undefined ? {} : { viewport: rawTree.dom.viewport }),
37
+ bodyText: rawTree.dom.bodyText.slice(0, Math.min(12000, Math.floor(maxBytes / 12))),
38
+ bodyTextTruncated: rawTree.dom.bodyTextTruncated || rawTree.dom.bodyText.length > Math.min(12000, Math.floor(maxBytes / 12)) } : undefined;
39
+ const activity = provider === 'native' && typeof rawTree?.activity === 'string' ? rawTree.activity : undefined;
40
+ const uiaSnapshot = provider === 'uia' ? uia.snapshotIdentity(rawTree) : null;
41
+
42
+ walkProvider(provider, rawTree, (source, sourceIndex) => {
43
+ visited += 1;
44
+ if (now() - startedAtMs > hardLimitMs) {
45
+ truncated = true;
46
+ reason = 'hard_limit_ms';
47
+ return false;
48
+ }
49
+ const node = toSemanticNode(provider, source, sourceIndex, rawTreeId, screenshotId, uiaSnapshot);
50
+ if (node && keepNode(node)) collected.push(node);
51
+ return true;
52
+ });
53
+
54
+ const nodes = fitNodes(collected, {
55
+ provider,
56
+ rawTreeId,
57
+ screenshotId,
58
+ visited,
59
+ truncated,
60
+ reason,
61
+ maxBytes,
62
+ activity,
63
+ page,
64
+ prioritizeVisibleControls: provider === 'native' && rawTree?.nativeTargetSchema === iosNative.schema,
65
+ });
66
+ truncated = truncated || nodes.truncated;
67
+ reason = reason || nodes.reason;
68
+
69
+ return {
70
+ ok: true,
71
+ provider,
72
+ rawTreeId,
73
+ screenshotId,
74
+ visited,
75
+ truncated,
76
+ reason,
77
+ ...(activity === undefined ? {} : { activity }),
78
+ ...(page === undefined ? {} : { page }),
79
+ nodes: nodes.nodes,
80
+ };
81
+ }
82
+
83
+ function walkProvider(provider, rawTree, visit) {
84
+ if (provider === 'native') walkNative(rawTree, visit);
85
+ else if (provider === 'uia') walkUia(rawTree, visit);
86
+ else if (provider === 'flutter') walkFlutter(rawTree, visit);
87
+ else walkH5(rawTree, visit);
88
+ }
89
+
90
+ function walkNative(tree, visit) {
91
+ if (tree?.nativeTargetSchema === iosNative.schema) {
92
+ let sourceIndex = 0;
93
+ for (const node of iosNative.nodesOf(tree)) {
94
+ const rect = node.rect;
95
+ const source = { id: node.elementId, elementId: node.elementId, className: node.type, elementType: node.type,
96
+ text: node.value, label: node.label, accessibilityId: node.rawIdentifier,
97
+ enabled: node.isEnabled === '1', visible: node.isVisible === '1',
98
+ editable: ['TextField', 'SecureTextField', 'TextView', 'SearchField'].includes(node.type),
99
+ bounds: rect ? { left: rect.x, top: rect.y, right: rect.x + rect.width, bottom: rect.y + rect.height } : null };
100
+ if (visit(source, sourceIndex++) === false) return;
101
+ }
102
+ return;
103
+ }
104
+ const index = { i: 0 };
105
+ if (Array.isArray(tree?.windows) && tree.windows.length) {
106
+ // Match native action selection: the last non-hidden root owns the foreground.
107
+ // An unknown/disabled root still blocks background controls. Do not spend the
108
+ // summary budget on a long background page before exposing its modal dialog.
109
+ const selected = foregroundNativeWindow(tree);
110
+ if (!selected) return;
111
+ const foreground = selected.index;
112
+ // Preserve the sourceIndex ordinal in the original windows' preorder.
113
+ for (let i = 0; i < foreground; i++) walkChildren(tree.windows[i]?.root, () => true, index);
114
+ walkChildren(tree.windows[foreground]?.root, visit, index);
115
+ return;
116
+ }
117
+ if (tree?.root) walkChildren(tree.root, visit, index);
118
+ else if (tree && !tree.windows && !tree.root) walkChildren(tree, visit, index);
119
+ }
120
+
121
+ function walkUia(tree, visit) {
122
+ if (typeof tree === 'string' || typeof tree?.xml === 'string' || typeof tree?.hierarchy === 'string') {
123
+ walkUiaXml(typeof tree === 'string' ? tree : (tree.xml || tree.hierarchy), visit);
124
+ return;
125
+ }
126
+ if (Array.isArray(tree?.nodes)) {
127
+ const index = { i: 0 };
128
+ for (const node of tree.nodes) {
129
+ if (visit(node, index.i++) === false) return;
130
+ }
131
+ return;
132
+ }
133
+ walkNative(tree, visit);
134
+ }
135
+
136
+ function walkUiaXml(xml, visit) {
137
+ const tokenRegex = /<\/node>|<node\b[^>]*\/?>/g;
138
+ let sourceIndex = 0;
139
+ let token;
140
+ while ((token = tokenRegex.exec(String(xml || ''))) !== null) {
141
+ const tag = token[0];
142
+ if (tag.startsWith('</node') || !tag.startsWith('<node')) continue;
143
+ if (visit(parseXmlAttributes(tag), sourceIndex++) === false) return;
144
+ }
145
+ }
146
+
147
+ function walkFlutter(tree, visit) {
148
+ const operable = tree?.operable || tree?.layout?.operable;
149
+ if (Array.isArray(operable?.nodes)) {
150
+ let sourceIndex = 0;
151
+ for (const node of operable.nodes) {
152
+ if (visit(node, sourceIndex++) === false) return;
153
+ }
154
+ return;
155
+ }
156
+ if (Array.isArray(tree?.nodes)) {
157
+ let sourceIndex = 0;
158
+ for (const node of tree.nodes) {
159
+ if (visit(node, sourceIndex++) === false) return;
160
+ }
161
+ return;
162
+ }
163
+ walkChildren(tree?.root || tree, visit, { i: 0 });
164
+ }
165
+
166
+ function walkH5(tree, visit) {
167
+ if ([require('./ios-h5-target').schema, require('./android-h5-target').schema].includes(tree?.h5TargetSchema) || tree?.webTargetSchema === require('./web-dom-target').schema) {
168
+ for (const [index, node] of tree.dom.controls.entries()) {
169
+ if (visit({ ...node, elementType: node.tag, label: node.ariaLabel, enabled: !node.disabled,
170
+ clickable: ['a', 'button'].includes(node.tag)
171
+ || node.tag === 'input' && ['checkbox', 'radio'].includes(node.type)
172
+ || ['checkbox', 'menuitemcheckbox', 'radio', 'menuitemradio', 'switch'].includes(node.role) }, index) === false) return;
173
+ }
174
+ return;
175
+ }
176
+ if (Array.isArray(tree?.nodes)) {
177
+ let sourceIndex = 0;
178
+ for (const node of tree.nodes) {
179
+ if (visit(node, sourceIndex++) === false) return;
180
+ }
181
+ return;
182
+ }
183
+ walkChildren(tree?.documentElement || tree?.root || tree, visit, { i: 0 });
184
+ }
185
+
186
+ function walkChildren(node, visit, index) {
187
+ if (!node || typeof node !== 'object') return true;
188
+ if (visit(node, index.i++) === false) return false;
189
+ const children = Array.isArray(node.children) ? node.children : [];
190
+ for (const child of children) {
191
+ if (walkChildren(child, visit, index) === false) return false;
192
+ }
193
+ return true;
194
+ }
195
+
196
+ function toSemanticNode(provider, source, sourceIndex, rawTreeId, screenshotId, uiaSnapshot) {
197
+ if (!source || typeof source !== 'object') return null;
198
+ const className = String(source.className || source.class || source.widgetType || source.tag || source.tagName || '');
199
+ const text = firstString(source.text, source.value);
200
+ const label = firstString(
201
+ source.contentDescription,
202
+ source['content-desc'],
203
+ source.label,
204
+ source.accessibilityLabel,
205
+ source['aria-label'],
206
+ source.alt,
207
+ );
208
+ const role = classifyRole(source, className);
209
+ return semanticNode({
210
+ nodeId: nodeIdOf(provider, source, sourceIndex),
211
+ sourceIndex,
212
+ rawTreeId,
213
+ screenshotId,
214
+ targetRef: uiaSnapshot ? uia.nodeTargetRef(uiaSnapshot, source) : undefined,
215
+ role,
216
+ text,
217
+ label,
218
+ ...(provider === 'flutter' ? { hint: source.hint, errorText: source.errorText } : {}),
219
+ bounds: boundsOf(source),
220
+ enabled: source.enabled !== false && source.enabled !== 'false',
221
+ checked: provider === 'uia' && boolOrNull(source.checkable) !== true
222
+ ? null : source.checked === 'mixed' ? 'mixed' : boolOrNull(source.checked),
223
+ selected: boolOrNull(source.selected),
224
+ clickable: Boolean(
225
+ source.clickable === true
226
+ || source.clickable === 'true'
227
+ || (Array.isArray(source.actions) && source.actions.includes('tap')),
228
+ ),
229
+ ...(['native', 'h5'].includes(provider) ? {
230
+ resourceName: source.resourceName,
231
+ editable: source.editable,
232
+ interaction: source.interaction,
233
+ visible: source.visible,
234
+ effectiveVisible: source.effectiveVisible,
235
+ accessibilityId: source.accessibilityId,
236
+ elementId: source.elementId,
237
+ elementType: source.elementType,
238
+ } : {}),
239
+ });
240
+ }
241
+
242
+ function keepNode(node) {
243
+ return Boolean(
244
+ node.text
245
+ || node.label
246
+ || node.accessibilityId
247
+ || node.clickable
248
+ || node.checked !== null
249
+ || node.selected === true
250
+ || node.role === 'button'
251
+ || node.role === 'input'
252
+ || node.role === 'image',
253
+ );
254
+ }
255
+
256
+ function classifyRole(source, className) {
257
+ const lower = className.toLowerCase();
258
+ const role = String(source.role || '').toLowerCase();
259
+ if (['checkbox', 'menuitemcheckbox', 'radio', 'menuitemradio', 'switch'].includes(role)) return role;
260
+ if (lower === 'input' && ['checkbox', 'radio'].includes(source.type)) return source.type;
261
+ if (role === 'textbox' || source.editable === true || /edit|input|textfield|textarea|search/.test(lower)) return 'input';
262
+ if (role === 'button' || /button|btn/.test(lower)) return 'button';
263
+ if (role === 'image' || source.src || /image|img|imageview/.test(lower)) return 'image';
264
+ if (source.clickable === true || source.clickable === 'true' || (Array.isArray(source.actions) && source.actions.includes('tap'))) {
265
+ return 'clickable';
266
+ }
267
+ return 'text';
268
+ }
269
+
270
+ function nodeIdOf(provider, source, sourceIndex) {
271
+ if (source.id != null && source.id !== '') return source.id;
272
+ if (source.nodeId != null && source.nodeId !== '') return source.nodeId;
273
+ if (source.resourceName) return source.resourceName;
274
+ if (source['resource-id']) return source['resource-id'];
275
+ if (source.resourceId) return source.resourceId;
276
+ return `${provider}:${sourceIndex}`;
277
+ }
278
+
279
+ function boundsOf(source) {
280
+ if (source.bounds && typeof source.bounds === 'object') return source.bounds;
281
+ if (typeof source.bounds === 'string') return parseUiaBounds(source.bounds);
282
+ return null;
283
+ }
284
+
285
+ function parseUiaBounds(value) {
286
+ const match = /\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/.exec(String(value || ''));
287
+ if (!match) return null;
288
+ return {
289
+ left: Number(match[1]),
290
+ top: Number(match[2]),
291
+ right: Number(match[3]),
292
+ bottom: Number(match[4]),
293
+ };
294
+ }
295
+
296
+ function firstString(...values) {
297
+ for (const value of values) {
298
+ if (value == null) continue;
299
+ const text = String(value);
300
+ if (text !== '') return text;
301
+ }
302
+ return null;
303
+ }
304
+
305
+ function boolOrNull(value) {
306
+ if (value === true || value === 'true') return true;
307
+ if (value === false || value === 'false') return false;
308
+ return null;
309
+ }
310
+
311
+ function fitNodes(nodes, { provider, rawTreeId, screenshotId, visited, truncated, reason, maxBytes, activity, page, prioritizeVisibleControls }) {
312
+ const meta = { ok: true, provider, rawTreeId, screenshotId, visited, truncated, reason, ...(activity === undefined ? {} : { activity }), ...(page === undefined ? {} : { page }) };
313
+ if (byteSize(nodes, meta) <= maxBytes) {
314
+ return { nodes, truncated, reason };
315
+ }
316
+ // WDA places native toolbars and sheets after potentially long WKWebView text.
317
+ // Reserve their visible controls before spending the budget on article content;
318
+ // retain the original identities, visibility and preorder in the final result.
319
+ const candidates = prioritizeVisibleControls
320
+ ? [...nodes].sort((a, b) => visibleControlPriority(a) - visibleControlPriority(b))
321
+ : nodes;
322
+ let low = 0;
323
+ let high = nodes.length;
324
+ while (low < high) {
325
+ const mid = Math.ceil((low + high) / 2);
326
+ const candidate = {
327
+ ...meta,
328
+ truncated: true,
329
+ reason: reason || 'max_bytes',
330
+ };
331
+ if (byteSize(candidates.slice(0, mid), candidate) <= maxBytes) low = mid;
332
+ else high = mid - 1;
333
+ }
334
+ const selected = candidates.slice(0, low);
335
+ if (prioritizeVisibleControls) selected.sort((a, b) => a.sourceIndex - b.sourceIndex);
336
+ return { nodes: selected, truncated: true, reason: reason || 'max_bytes' };
337
+ }
338
+
339
+ function visibleControlPriority(node) {
340
+ if (node.visible !== true) return 2;
341
+ return node.role === 'button' || node.role === 'input' || node.clickable ? 0 : 1;
342
+ }
343
+
344
+ function byteSize(nodes, meta) {
345
+ return Buffer.byteLength(JSON.stringify({ ...meta, nodes }), 'utf8');
346
+ }
347
+
348
+ module.exports = {
349
+ DEFAULT_MAX_BYTES,
350
+ HARD_LIMIT_MS,
351
+ summarizeTree,
352
+ };
@@ -0,0 +1,47 @@
1
+ 'use strict';
2
+
3
+ const { targetIdentity } = require('./execution-target');
4
+
5
+ function androidAppTargetKey(serial, packageName) {
6
+ const serialValue = String(serial || '');
7
+ if (!serialValue) return '';
8
+ return targetIdentity({ platform: 'android', serial: serialValue, packageName: String(packageName || '') });
9
+ }
10
+
11
+ function createTargetLease({ maxActive = 1 } = {}) {
12
+ const held = new Map();
13
+
14
+ function acquire(serial) {
15
+ const key = String(serial || '');
16
+ if (!key) return { ok: false, error: 'serial_required' };
17
+ const active = held.get(key) || 0;
18
+ if (active >= maxActive) {
19
+ return { ok: false, error: 'target_busy', serial: key, active };
20
+ }
21
+ held.set(key, active + 1);
22
+ return {
23
+ ok: true,
24
+ serial: key,
25
+ release() {
26
+ const current = held.get(key) || 0;
27
+ if (current <= 1) held.delete(key);
28
+ else held.set(key, current - 1);
29
+ },
30
+ };
31
+ }
32
+
33
+ function status(serial) {
34
+ const key = String(serial || '');
35
+ return { serial: key, active: held.get(key) || 0, maxActive };
36
+ }
37
+
38
+ return { acquire, status };
39
+ }
40
+
41
+ const processTargetLease = createTargetLease();
42
+
43
+ function getProcessTargetLease() {
44
+ return processTargetLease;
45
+ }
46
+
47
+ module.exports = { androidAppTargetKey, createTargetLease, getProcessTargetLease };
@@ -0,0 +1,111 @@
1
+ 'use strict';
2
+
3
+ const { CommandError } = require('../command-errors');
4
+ const { foregroundNativeWindow, explicitlyHidden } = require('./native-target');
5
+ const { parseXmlAttributes } = require('./xml-attributes');
6
+ const { looksLikeAndroidUiHierarchyXml, visitAndroidUiHierarchyTags } = require('../android-uia-xml');
7
+ const { runExecution, checkExecution, executionSleep } = require('./execution-scope');
8
+
9
+ function validateTextConditions({ targetText, requireText = [], absentText = [], requireActivity, provider = 'auto' }) {
10
+ const invalid = message => { throw new CommandError('invalid_argument', message, { field: 'conditions' }); };
11
+ if (!['auto', 'native', 'flutter', 'uia'].includes(provider)) invalid('provider must be auto, native, flutter, or uia.');
12
+ if (targetText !== undefined && (typeof targetText !== 'string' || !targetText)) invalid('targetText must be a non-empty exact label.');
13
+ if (requireActivity !== undefined && (typeof requireActivity !== 'string' || !requireActivity)) invalid('requireActivity must be a full Activity class name.');
14
+ for (const values of [requireText, absentText]) if (!Array.isArray(values) || values.some(value => typeof value !== 'string' || !value)) invalid('requireText and absentText must be arrays of non-empty exact labels.');
15
+ const present = [...new Set([...(targetText === undefined ? [] : [targetText]), ...requireText])];
16
+ if (!present.length && !absentText.length && !requireActivity) invalid('Supply targetText, requireText, absentText, or requireActivity.');
17
+ if (!present.length && provider === 'auto') invalid('Waiting for absence or Activity alone requires an explicit provider.');
18
+ if (present.some(value => absentText.includes(value))) invalid('The same label cannot be both required and absent.');
19
+ return { present, absent: [...new Set(absentText)], requireActivity, provider };
20
+ }
21
+
22
+ function visibleLabels(provider, rawTree) {
23
+ const labels = new Set();
24
+ const add = value => { if (typeof value === 'string' && value) labels.add(value); };
25
+ const unavailable = () => { throw new CommandError('observation_unavailable', `A valid ${provider} foreground tree is required.`); };
26
+ if (rawTree?.ok === false) unavailable();
27
+ if (provider === 'native') {
28
+ const window = foregroundNativeWindow(rawTree);
29
+ if (!window?.root || explicitlyHidden(window.root) || (window.root.visible !== true && window.root.effectiveVisible !== true)) unavailable();
30
+ const visit = node => {
31
+ if (!node || explicitlyHidden(node) || (node.visible !== true && node.effectiveVisible !== true)) return;
32
+ add(node.text); add(node.contentDescription);
33
+ for (const child of node.children || []) visit(child);
34
+ };
35
+ visit(window.root);
36
+ } else if (provider === 'flutter') {
37
+ if (!Array.isArray(rawTree?.nodes)) unavailable();
38
+ for (const node of rawTree.nodes) {
39
+ if (node.visible === false || node.offstage === true) continue;
40
+ add(node.text); add(node.value); add(node.label);
41
+ }
42
+ } else {
43
+ if (typeof rawTree !== 'string' || !looksLikeAndroidUiHierarchyXml(rawTree)) unavailable();
44
+ const hidden = [];
45
+ visitAndroidUiHierarchyTags(rawTree, ({ tag, closing, selfClosing }) => {
46
+ if (closing) { hidden.pop(); return; }
47
+ const attrs = parseXmlAttributes(tag);
48
+ const isHidden = hidden.at(-1) || attrs['visible-to-user'] === 'false' || attrs.displayed === 'false';
49
+ if (!isHidden) { add(attrs.text); add(attrs['content-desc']); }
50
+ if (!selfClosing) hidden.push(isHidden);
51
+ });
52
+ }
53
+ return labels;
54
+ }
55
+
56
+ function checkTextConditions(labels, activity, conditions) {
57
+ const missing = conditions.present.filter(value => !labels.has(value));
58
+ const unexpected = conditions.absent.filter(value => labels.has(value));
59
+ const activityMatches = !conditions.requireActivity || activity === conditions.requireActivity;
60
+ return { ok: !missing.length && !unexpected.length && activityMatches, missing, unexpected, activity, activityMatches };
61
+ }
62
+
63
+ async function waitForText({ options, readTree, readForeground }) {
64
+ const conditions = validateTextConditions(options);
65
+ const timeoutMs = options.timeoutMs ?? 10000;
66
+ const intervalMs = options.intervalMs ?? 500;
67
+ if (!Number.isSafeInteger(intervalMs) || intervalMs < 1) throw new CommandError('invalid_argument', 'intervalMs must be a positive integer.', { field: 'intervalMs' });
68
+ const deadline = Date.now() + timeoutMs;
69
+ let lastCheck = null;
70
+ try {
71
+ return await runExecution({ timeoutMs, mutation: false }, async () => {
72
+ for (;;) {
73
+ const foreground = await readForeground();
74
+ checkExecution();
75
+ if (!foreground?.ok) throw new CommandError('foreground_probe_failed', 'The foreground app could not be verified.');
76
+ if (options.packageName && foreground.packageName !== options.packageName) throw new CommandError('foreground_package_mismatch', 'The foreground app differs from packageName.');
77
+ let nativeTree;
78
+ let successfulReads = 0;
79
+ const failures = [];
80
+ for (const provider of conditions.provider === 'auto' ? ['native', 'flutter', 'uia'] : [conditions.provider]) {
81
+ try {
82
+ if (provider === 'flutter') {
83
+ nativeTree ??= await readTree('native');
84
+ if (foregroundNativeWindow(nativeTree)?.type !== 'activity') throw new CommandError('native_foreground_blocks_flutter', 'A native foreground window covers Flutter.');
85
+ }
86
+ const rawTree = provider === 'native' ? (nativeTree = await readTree(provider)) : await readTree(provider);
87
+ const labels = visibleLabels(provider, rawTree);
88
+ const after = await readForeground();
89
+ checkExecution();
90
+ if (!after?.ok || after.component !== foreground.component || after.packageName !== foreground.packageName) {
91
+ throw new CommandError('foreground_changed_during_observation', 'The foreground changed while reading text.');
92
+ }
93
+ successfulReads++;
94
+ lastCheck = { ...checkTextConditions(labels, after.activity, conditions), provider, observedAtMs: Date.now() };
95
+ if (lastCheck.ok) return { ok: true, conditions, matched: lastCheck };
96
+ } catch (error) {
97
+ checkExecution();
98
+ failures.push({ provider, error: error.code || 'observation_unavailable', message: String(error.message).split('\n')[0] });
99
+ }
100
+ }
101
+ if (!successfulReads) return { ok: false, error: 'observation_unavailable', dispatched: false, ambiguous: false, failures };
102
+ await executionSleep(Math.min(intervalMs, Math.max(1, deadline - Date.now())));
103
+ }
104
+ });
105
+ } catch (error) {
106
+ if (error.code !== 'deadline_exceeded') throw error;
107
+ return { ok: false, error: 'deadline_exceeded', dispatched: false, ambiguous: false, timeoutMs, conditions, lastCheck };
108
+ }
109
+ }
110
+
111
+ module.exports = { validateTextConditions, visibleLabels, checkTextConditions, waitForText };
@@ -0,0 +1,96 @@
1
+ 'use strict';
2
+
3
+ const { CommandError } = require('../command-errors');
4
+ const { createUiaRuntimePort } = require('./uia-runtime-port');
5
+ const { runDeviceEffect, isDeviceSettlementDurable, completeDeviceAcknowledgement } = require('./device-mutation-lease');
6
+ const { currentExecution, runExecution, checkExecution, withoutExecution, executionSleep } = require('./execution-scope');
7
+ const protocol = require('./uia-protocol');
8
+
9
+ function matchingPending(response, identity) {
10
+ return response?.schemaVersion === protocol.schema && response.settled === false && response.ok === false
11
+ && response.dispatched === null && response.ambiguous === true
12
+ && ['bootId', 'runtimeEpoch', 'actionId', 'requestSha256'].every(key => response[key] === identity[key])
13
+ && ['prepared', 'queued', 'admitted', 'unknown', 'terminal'].includes(response.phase)
14
+ && typeof response.error === 'string' && response.error.length > 0;
15
+ }
16
+
17
+ async function executeUiaAction({ adb, serial, binding, actionId, timeoutMs = 10000, clickPolicy = 'nearest_clickable_ancestor',
18
+ port = createUiaRuntimePort({ adb, serial, timeoutMs }) } = {}) {
19
+ if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2147483647)
20
+ throw new CommandError('invalid_argument', 'UIA timeoutMs must be a positive integer in milliseconds.', { dispatched: false, ambiguous: false });
21
+ if (!currentExecution()) return runExecution({ timeoutMs, mutation: true },
22
+ () => executeUiaAction({ adb, serial, binding, actionId, timeoutMs, clickPolicy, port }));
23
+ checkExecution();
24
+ const candidate = protocol.actionRequest(binding, { actionId, timeoutMs: Math.min(timeoutMs, 60000), clickPolicy });
25
+ let connection;
26
+ try {
27
+ connection = await port.ensure();
28
+ if (connection.peer.bootId !== binding?.bootId || connection.peer.runtimeEpoch !== binding?.runtimeEpoch) {
29
+ throw new CommandError('uia_stale_runtime', 'The observed UIA runtime has changed. Observe the target again.');
30
+ }
31
+ if (connection.status.count === connection.status.capacity) {
32
+ throw new CommandError('uia_action_capacity_exhausted', 'The UIA action journal is full. Reconcile pending actions and retain their completion receipts; a new observation rotates only a fully acknowledged session.');
33
+ }
34
+ } catch (error) {
35
+ return { ok: false, error: error.code || 'uia_runtime_unavailable', message: error.message, dispatched: false, ambiguous: false };
36
+ }
37
+ checkExecution();
38
+ const remaining = Math.floor(Math.min(timeoutMs, currentExecution().deadlineMs - Date.now(), 60000));
39
+ if (remaining < 1) throw new CommandError('deadline_exceeded', 'The UIA execution budget expired before preparation.', { dispatched: false, ambiguous: false });
40
+ const prepared = protocol.actionRequest(binding, { actionId: candidate.request.actionId, timeoutMs: remaining, clickPolicy });
41
+ const { request, requestJson, requestSha256 } = prepared;
42
+ const identity = { kind: 'uia-node', schemaVersion: protocol.schema, actionId: request.actionId,
43
+ bootId: request.bootId, runtimeEpoch: request.runtimeEpoch, requestJson, requestSha256,
44
+ target: { adb, serial, root: connection.peer.sessionPath.split('/sessions/')[0],
45
+ sessionPath: connection.peer.sessionPath, dexSha256: connection.peer.dexSha256 } };
46
+ if (!protocol.validIdentity(identity)) throw new CommandError('invalid_uia_execution_identity', 'The UIA runtime did not yield a valid original action identity.', { dispatched: false, ambiguous: false });
47
+
48
+ let terminal;
49
+ const result = await runDeviceEffect(identity, async () => {
50
+ let response, failure;
51
+ try {
52
+ response = await port.action(connection, 'prepare', identity);
53
+ if (!protocol.originalReceipt(response, identity)) {
54
+ if (!matchingPending(response, identity)) throw new CommandError('invalid_uia_execution_receipt', 'UIA preparation did not return the original action identity.');
55
+ if (response.error !== 'uia_action_pending') throw new CommandError(response.error, 'UIA could not prepare this action durably.');
56
+ if (response.phase === 'prepared') {
57
+ checkExecution();
58
+ response = await port.action(connection, 'start', identity);
59
+ }
60
+ }
61
+ for (;;) {
62
+ const receipt = protocol.originalReceipt(response, identity);
63
+ if (receipt) { terminal = response; return { ...receipt, request, executionReceipt: protocol.settlementProof(response, identity) }; }
64
+ if (!matchingPending(response, identity)) throw new CommandError('invalid_uia_execution_receipt', 'UIA did not return a matching original action state.');
65
+ if (response.error !== 'uia_action_pending') throw new CommandError(response.error, 'The original UIA action has not completed.');
66
+ checkExecution();
67
+ await executionSleep(25);
68
+ response = await port.action(connection, 'query', identity);
69
+ }
70
+ } catch (error) { failure = error; }
71
+
72
+ let recovery;
73
+ try {
74
+ recovery = await withoutExecution(() => runExecution({ timeoutMs: 5000, mutation: false }, () => port.recover(identity)));
75
+ } catch (error) { recovery = { ok: false, error: error.code || 'uia_completion_query_failed' }; }
76
+ const receipt = protocol.originalReceipt(recovery, identity);
77
+ if (receipt) terminal = recovery;
78
+ return { ok: false, error: failure.code || 'uia_action_response_lost', message: failure.message,
79
+ actionId: identity.actionId, bootId: identity.bootId, runtimeEpoch: identity.runtimeEpoch, request,
80
+ settled: Boolean(receipt), dispatched: receipt?.dispatched ?? null, ambiguous: receipt ? false : true,
81
+ recovery, executionReceipt: receipt ? protocol.settlementProof(recovery, identity) : null };
82
+ }, value => value?.executionReceipt || false);
83
+
84
+ // Shared ownership commits the proof before the phone may acknowledge it.
85
+ // A history/acknowledgement failure never changes the already known effect.
86
+ if (terminal && isDeviceSettlementDurable(result.executionReceipt)) {
87
+ try {
88
+ const acknowledgement = await completeDeviceAcknowledgement(result.executionReceipt, () => withoutExecution(() =>
89
+ runExecution({ timeoutMs: 5000, mutation: false }, () => port.acknowledge(identity, terminal))));
90
+ result.completionHistory = acknowledgement.history;
91
+ } catch (error) { result.cleanupError = error.code || 'uia_acknowledgement_failed'; }
92
+ }
93
+ return result;
94
+ }
95
+
96
+ module.exports = { executeUiaAction, matchingPending, ...protocol };