@mobileai/react-native 0.9.26 → 0.9.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -15
- package/android/build.gradle +17 -0
- package/android/src/main/java/com/mobileai/overlay/FloatingOverlayDialogRootViewGroup.kt +243 -0
- package/android/src/main/java/com/mobileai/overlay/FloatingOverlayView.kt +281 -87
- package/android/src/newarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +52 -17
- package/android/src/oldarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +49 -2
- package/bin/generate-map.cjs +556 -126
- package/ios/Podfile +63 -0
- package/ios/Podfile.lock +2290 -0
- package/ios/Podfile.properties.json +4 -0
- package/ios/mobileaireactnative/AppDelegate.swift +69 -0
- package/ios/mobileaireactnative/Images.xcassets/AppIcon.appiconset/Contents.json +13 -0
- package/ios/mobileaireactnative/Images.xcassets/Contents.json +6 -0
- package/ios/mobileaireactnative/Images.xcassets/SplashScreenLegacy.imageset/Contents.json +21 -0
- package/ios/mobileaireactnative/Images.xcassets/SplashScreenLegacy.imageset/SplashScreenLegacy.png +0 -0
- package/ios/mobileaireactnative/Info.plist +55 -0
- package/ios/mobileaireactnative/PrivacyInfo.xcprivacy +48 -0
- package/ios/mobileaireactnative/SplashScreen.storyboard +47 -0
- package/ios/mobileaireactnative/Supporting/Expo.plist +6 -0
- package/ios/mobileaireactnative/mobileaireactnative-Bridging-Header.h +3 -0
- package/ios/mobileaireactnative.xcodeproj/project.pbxproj +547 -0
- package/ios/mobileaireactnative.xcodeproj/xcshareddata/xcschemes/mobileaireactnative.xcscheme +88 -0
- package/ios/mobileaireactnative.xcworkspace/contents.xcworkspacedata +10 -0
- package/lib/module/components/AIAgent.js +407 -148
- package/lib/module/components/AgentChatBar.js +253 -62
- package/lib/module/components/FloatingOverlayWrapper.js +68 -32
- package/lib/module/config/endpoints.js +22 -1
- package/lib/module/core/AgentRuntime.js +192 -24
- package/lib/module/core/FiberTreeWalker.js +410 -34
- package/lib/module/core/OutcomeVerifier.js +149 -0
- package/lib/module/core/systemPrompt.js +126 -44
- package/lib/module/providers/GeminiProvider.js +9 -3
- package/lib/module/services/MobileAIKnowledgeRetriever.js +1 -1
- package/lib/module/services/telemetry/MobileAI.js +1 -1
- package/lib/module/services/telemetry/TelemetryService.js +21 -2
- package/lib/module/services/telemetry/TouchAutoCapture.js +45 -35
- package/lib/module/specs/FloatingOverlayNativeComponent.ts +7 -1
- package/lib/module/support/supportPrompt.js +22 -7
- package/lib/module/support/supportStyle.js +55 -0
- package/lib/module/support/types.js +2 -0
- package/lib/module/tools/tapTool.js +77 -6
- package/lib/module/tools/typeTool.js +20 -0
- package/lib/module/utils/humanizeScreenName.js +49 -0
- package/lib/typescript/src/components/AIAgent.d.ts +6 -2
- package/lib/typescript/src/components/AgentChatBar.d.ts +15 -1
- package/lib/typescript/src/components/FloatingOverlayWrapper.d.ts +22 -10
- package/lib/typescript/src/config/endpoints.d.ts +4 -0
- package/lib/typescript/src/core/AgentRuntime.d.ts +17 -1
- package/lib/typescript/src/core/FiberTreeWalker.d.ts +12 -1
- package/lib/typescript/src/core/OutcomeVerifier.d.ts +46 -0
- package/lib/typescript/src/core/systemPrompt.d.ts +3 -10
- package/lib/typescript/src/core/types.d.ts +37 -1
- package/lib/typescript/src/index.d.ts +1 -0
- package/lib/typescript/src/services/MobileAIKnowledgeRetriever.d.ts +1 -1
- package/lib/typescript/src/services/telemetry/TelemetryService.d.ts +7 -1
- package/lib/typescript/src/services/telemetry/types.d.ts +1 -1
- package/lib/typescript/src/specs/FloatingOverlayNativeComponent.d.ts +5 -0
- package/lib/typescript/src/support/index.d.ts +1 -0
- package/lib/typescript/src/support/supportStyle.d.ts +9 -0
- package/lib/typescript/src/support/types.d.ts +3 -0
- package/lib/typescript/src/tools/tapTool.d.ts +3 -2
- package/lib/typescript/src/utils/humanizeScreenName.d.ts +6 -0
- package/lib/typescript/test-tree.d.ts +2 -0
- package/package.json +5 -2
- package/src/specs/FloatingOverlayNativeComponent.ts +7 -1
- package/ios/MobileAIFloatingOverlayComponentView.mm +0 -73
- package/ios/MobileAIPilotIntents.swift +0 -51
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import { Dimensions } from 'react-native';
|
|
12
13
|
import { logger } from "../utils/logger.js";
|
|
13
14
|
import { getChild, getSibling, getParent, getProps, getStateNode, getType, getDisplayName } from "./FiberAdapter.js";
|
|
14
15
|
import { getActiveAlert } from "./NativeAlertInterceptor.js";
|
|
@@ -25,6 +26,7 @@ const SLIDER_TYPES = new Set(['Slider', 'RNCSlider', 'RCTSlider']);
|
|
|
25
26
|
const PICKER_TYPES = new Set(['Picker', 'RNCPicker', 'RNPickerSelect', 'DropDownPicker', 'SelectDropdown']);
|
|
26
27
|
const DATE_PICKER_TYPES = new Set(['DateTimePicker', 'RNDateTimePicker', 'DatePicker', 'RNDatePicker']);
|
|
27
28
|
const TEXT_TYPES = new Set(['Text', 'RCTText']);
|
|
29
|
+
const RADIO_TYPES = new Set(['Radio', 'RadioButton', 'RadioItem', 'RadioButtonItem', 'RadioGroupItem']);
|
|
28
30
|
|
|
29
31
|
// Media component types for Component-Context Media Inference
|
|
30
32
|
const IMAGE_TYPES = new Set(['Image', 'RCTImageView', 'ExpoImage', 'FastImage', 'CachedImage']);
|
|
@@ -33,11 +35,16 @@ const VIDEO_TYPES = new Set(['Video', 'ExpoVideo', 'RCTVideo', 'VideoPlayer', 'V
|
|
|
33
35
|
// Known RN internal component names to skip when walking up for context
|
|
34
36
|
const RN_INTERNAL_NAMES = new Set(['View', 'RCTView', 'Pressable', 'TouchableOpacity', 'TouchableHighlight', 'ScrollView', 'RCTScrollView', 'FlatList', 'SectionList', 'SafeAreaView', 'RNCSafeAreaView', 'KeyboardAvoidingView', 'Modal', 'StatusBar', 'Text', 'RCTText', 'AnimatedComponent', 'AnimatedComponentWrapper', 'Animated']);
|
|
35
37
|
const LOW_SIGNAL_RUNTIME_LABELS = new Set(['button', 'buttons', 'label', 'labels', 'title', 'titles', 'name', 'text', 'value', 'values', 'content', 'card', 'cards', 'row', 'rows', 'item', 'items', 'component', 'screen']);
|
|
36
|
-
|
|
38
|
+
const EXTERNALLY_LABELED_TYPES = new Set(['switch', 'radio', 'slider', 'picker', 'date-picker']);
|
|
37
39
|
// ─── State Extraction ──
|
|
38
40
|
|
|
39
41
|
/** Props to extract as state attributes — covers lazy devs who skip accessibility */
|
|
40
42
|
const STATE_PROPS = ['value', 'checked', 'selected', 'active', 'on', 'isOn', 'toggled', 'enabled'];
|
|
43
|
+
const RADIO_SELECTION_KEYS = ['checked', 'selected', 'isChecked', 'isSelected'];
|
|
44
|
+
const RADIO_TRUE_VALUES = new Set(['true', 'checked', 'selected', 'on']);
|
|
45
|
+
const RADIO_FALSE_VALUES = new Set(['false', 'unchecked', 'unselected', 'off']);
|
|
46
|
+
const RADIO_DECORATION_PATTERN = /(indicator|icon|label|provider|context)$/i;
|
|
47
|
+
const RADIO_GROUP_PATTERN = /(radiobuttongroup|radiogroup)/i;
|
|
41
48
|
|
|
42
49
|
/**
|
|
43
50
|
* Extract state attributes from a fiber node's props.
|
|
@@ -123,6 +130,240 @@ function getComponentName(fiber) {
|
|
|
123
130
|
if (type.render?.name) return type.render.name;
|
|
124
131
|
return null;
|
|
125
132
|
}
|
|
133
|
+
function hasSliderLikeSemantics(props) {
|
|
134
|
+
if (!props || typeof props !== 'object') return false;
|
|
135
|
+
if (typeof props.onSlidingComplete === 'function') return true;
|
|
136
|
+
const hasOnValueChange = typeof props.onValueChange === 'function';
|
|
137
|
+
if (!hasOnValueChange) return false;
|
|
138
|
+
const hasExplicitRange = props.minimumValue !== undefined || props.maximumValue !== undefined;
|
|
139
|
+
if (hasExplicitRange) return true;
|
|
140
|
+
const accessibilityValue = props.accessibilityValue;
|
|
141
|
+
const hasAccessibilityRange = !!accessibilityValue && typeof accessibilityValue === 'object' && (accessibilityValue.min !== undefined || accessibilityValue.max !== undefined);
|
|
142
|
+
if (hasAccessibilityRange) return true;
|
|
143
|
+
return typeof props.value === 'number';
|
|
144
|
+
}
|
|
145
|
+
function isScalarSelectionValue(value) {
|
|
146
|
+
return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean';
|
|
147
|
+
}
|
|
148
|
+
function coerceRadioBoolean(value) {
|
|
149
|
+
if (typeof value === 'boolean') return value;
|
|
150
|
+
if (typeof value === 'string') {
|
|
151
|
+
const normalized = value.trim().toLowerCase();
|
|
152
|
+
if (RADIO_TRUE_VALUES.has(normalized)) return true;
|
|
153
|
+
if (RADIO_FALSE_VALUES.has(normalized)) return false;
|
|
154
|
+
}
|
|
155
|
+
return undefined;
|
|
156
|
+
}
|
|
157
|
+
function getOwnRadioCheckedState(props) {
|
|
158
|
+
if (!props || typeof props !== 'object') return undefined;
|
|
159
|
+
if (props.accessibilityState && typeof props.accessibilityState === 'object') {
|
|
160
|
+
const accessibilityChecked = coerceRadioBoolean(props.accessibilityState.checked);
|
|
161
|
+
if (accessibilityChecked !== undefined) return accessibilityChecked;
|
|
162
|
+
}
|
|
163
|
+
for (const key of RADIO_SELECTION_KEYS) {
|
|
164
|
+
const checked = coerceRadioBoolean(props[key]);
|
|
165
|
+
if (checked !== undefined) return checked;
|
|
166
|
+
}
|
|
167
|
+
const statusChecked = coerceRadioBoolean(props.status);
|
|
168
|
+
if (statusChecked !== undefined) return statusChecked;
|
|
169
|
+
return undefined;
|
|
170
|
+
}
|
|
171
|
+
function getRadioSelectionValue(props) {
|
|
172
|
+
if (!props || typeof props !== 'object') return undefined;
|
|
173
|
+
return isScalarSelectionValue(props.value) ? props.value : undefined;
|
|
174
|
+
}
|
|
175
|
+
function isRadioGroupComponentName(name) {
|
|
176
|
+
if (!name) return false;
|
|
177
|
+
return RADIO_GROUP_PATTERN.test(name) && !/(item|option)/i.test(name);
|
|
178
|
+
}
|
|
179
|
+
function isRadioLikeComponentName(name) {
|
|
180
|
+
if (!name || !/radio/i.test(name)) return false;
|
|
181
|
+
if (RADIO_TYPES.has(name)) return true;
|
|
182
|
+
if (RADIO_DECORATION_PATTERN.test(name)) return false;
|
|
183
|
+
if (isRadioGroupComponentName(name)) return false;
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
function getRadioSelectionHandler(props) {
|
|
187
|
+
if (!props || typeof props !== 'object') return null;
|
|
188
|
+
if (typeof props.onValueChange === 'function') {
|
|
189
|
+
return {
|
|
190
|
+
channel: 'onValueChange',
|
|
191
|
+
handler: props.onValueChange
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
if (typeof props.onCheckedChange === 'function') {
|
|
195
|
+
return {
|
|
196
|
+
channel: 'onCheckedChange',
|
|
197
|
+
handler: props.onCheckedChange
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
if (typeof props.onChange === 'function') {
|
|
201
|
+
return {
|
|
202
|
+
channel: 'onChange',
|
|
203
|
+
handler: props.onChange
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
if (typeof props.onSelect === 'function') {
|
|
207
|
+
return {
|
|
208
|
+
channel: 'onSelect',
|
|
209
|
+
handler: props.onSelect
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
function findAncestorRadioSelectionController(fiber, maxDepth = 8) {
|
|
215
|
+
let current = getParent(fiber);
|
|
216
|
+
let depth = 0;
|
|
217
|
+
while (current && depth < maxDepth) {
|
|
218
|
+
const name = getComponentName(current);
|
|
219
|
+
const props = getProps(current);
|
|
220
|
+
const role = props.accessibilityRole || props.role;
|
|
221
|
+
const handler = getRadioSelectionHandler(props);
|
|
222
|
+
const selectedValue = isScalarSelectionValue(props.selectedValue) ? props.selectedValue : isScalarSelectionValue(props.value) ? props.value : undefined;
|
|
223
|
+
if (isRadioGroupComponentName(name) || role === 'radiogroup') {
|
|
224
|
+
return {
|
|
225
|
+
channel: handler?.channel,
|
|
226
|
+
handler: handler?.handler,
|
|
227
|
+
selectedValue
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
current = getParent(current);
|
|
231
|
+
depth++;
|
|
232
|
+
}
|
|
233
|
+
return null;
|
|
234
|
+
}
|
|
235
|
+
function inferRadioCheckedState(fiber, props) {
|
|
236
|
+
const ownState = getOwnRadioCheckedState(props);
|
|
237
|
+
if (ownState !== undefined) return ownState;
|
|
238
|
+
const itemValue = getRadioSelectionValue(props);
|
|
239
|
+
if (itemValue === undefined) return undefined;
|
|
240
|
+
const controller = findAncestorRadioSelectionController(fiber);
|
|
241
|
+
if (controller?.selectedValue !== undefined) {
|
|
242
|
+
return controller.selectedValue === itemValue;
|
|
243
|
+
}
|
|
244
|
+
return undefined;
|
|
245
|
+
}
|
|
246
|
+
function hasRadioLikeSemantics(fiber, name, props) {
|
|
247
|
+
if (!props || typeof props !== 'object') return false;
|
|
248
|
+
const role = props.accessibilityRole || props.role;
|
|
249
|
+
if (role === 'radio') return true;
|
|
250
|
+
if (!isRadioLikeComponentName(name)) return false;
|
|
251
|
+
if (typeof props.onPress === 'function' || typeof props.onLongPress === 'function') return true;
|
|
252
|
+
if (getRadioSelectionHandler(props)) return true;
|
|
253
|
+
if (getOwnRadioCheckedState(props) !== undefined) return true;
|
|
254
|
+
const itemValue = getRadioSelectionValue(props);
|
|
255
|
+
if (itemValue !== undefined && findAncestorRadioSelectionController(fiber)) return true;
|
|
256
|
+
return false;
|
|
257
|
+
}
|
|
258
|
+
function buildDerivedElementProps(fiber, elementType, props) {
|
|
259
|
+
const derivedProps = {
|
|
260
|
+
...props
|
|
261
|
+
};
|
|
262
|
+
if (elementType === 'radio') {
|
|
263
|
+
const checked = inferRadioCheckedState(fiber, props);
|
|
264
|
+
if (checked !== undefined && derivedProps.checked === undefined) {
|
|
265
|
+
derivedProps.checked = checked;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return derivedProps;
|
|
269
|
+
}
|
|
270
|
+
function getInteractionSignature(elementType, props) {
|
|
271
|
+
if (!elementType || !props || typeof props !== 'object') return null;
|
|
272
|
+
if (elementType === 'pressable') {
|
|
273
|
+
if (typeof props.onPress === 'function') {
|
|
274
|
+
return {
|
|
275
|
+
type: elementType,
|
|
276
|
+
channel: 'onPress',
|
|
277
|
+
handler: props.onPress
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
if (typeof props.onLongPress === 'function') {
|
|
281
|
+
return {
|
|
282
|
+
type: elementType,
|
|
283
|
+
channel: 'onLongPress',
|
|
284
|
+
handler: props.onLongPress
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
return null;
|
|
288
|
+
}
|
|
289
|
+
if (elementType === 'text-input' && typeof props.onChangeText === 'function') {
|
|
290
|
+
return {
|
|
291
|
+
type: elementType,
|
|
292
|
+
channel: 'onChangeText',
|
|
293
|
+
handler: props.onChangeText
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
if (elementType === 'switch' && typeof props.onValueChange === 'function') {
|
|
297
|
+
return {
|
|
298
|
+
type: elementType,
|
|
299
|
+
channel: 'onValueChange',
|
|
300
|
+
handler: props.onValueChange
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
if (elementType === 'radio') {
|
|
304
|
+
if (typeof props.onPress === 'function') {
|
|
305
|
+
return {
|
|
306
|
+
type: elementType,
|
|
307
|
+
channel: 'onPress',
|
|
308
|
+
handler: props.onPress
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
const selectionHandler = getRadioSelectionHandler(props);
|
|
312
|
+
if (selectionHandler) {
|
|
313
|
+
return {
|
|
314
|
+
type: elementType,
|
|
315
|
+
channel: selectionHandler.channel,
|
|
316
|
+
handler: selectionHandler.handler
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
return null;
|
|
320
|
+
}
|
|
321
|
+
if (elementType === 'slider') {
|
|
322
|
+
if (typeof props.onSlidingComplete === 'function') {
|
|
323
|
+
return {
|
|
324
|
+
type: elementType,
|
|
325
|
+
channel: 'onSlidingComplete',
|
|
326
|
+
handler: props.onSlidingComplete
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
if (typeof props.onValueChange === 'function') {
|
|
330
|
+
return {
|
|
331
|
+
type: elementType,
|
|
332
|
+
channel: 'onValueChange',
|
|
333
|
+
handler: props.onValueChange
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
return null;
|
|
337
|
+
}
|
|
338
|
+
if (elementType === 'picker' && typeof props.onValueChange === 'function') {
|
|
339
|
+
return {
|
|
340
|
+
type: elementType,
|
|
341
|
+
channel: 'onValueChange',
|
|
342
|
+
handler: props.onValueChange
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
if (elementType === 'date-picker') {
|
|
346
|
+
if (typeof props.onChange === 'function') {
|
|
347
|
+
return {
|
|
348
|
+
type: elementType,
|
|
349
|
+
channel: 'onChange',
|
|
350
|
+
handler: props.onChange
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
if (typeof props.onDateChange === 'function') {
|
|
354
|
+
return {
|
|
355
|
+
type: elementType,
|
|
356
|
+
channel: 'onDateChange',
|
|
357
|
+
handler: props.onDateChange
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
return null;
|
|
362
|
+
}
|
|
363
|
+
function interactionSignaturesMatch(a, b) {
|
|
364
|
+
if (!a || !b) return false;
|
|
365
|
+
return a.channel === b.channel && a.handler === b.handler;
|
|
366
|
+
}
|
|
126
367
|
|
|
127
368
|
/**
|
|
128
369
|
* Check if a fiber node represents an interactive element.
|
|
@@ -130,20 +371,24 @@ function getComponentName(fiber) {
|
|
|
130
371
|
function getElementType(fiber) {
|
|
131
372
|
const name = getComponentName(fiber);
|
|
132
373
|
const props = getProps(fiber);
|
|
374
|
+
if (isRadioGroupComponentName(name)) return null;
|
|
133
375
|
|
|
134
376
|
// Check by component name (known React Native types)
|
|
135
377
|
if (name && PRESSABLE_TYPES.has(name)) return 'pressable';
|
|
136
378
|
if (name && TEXT_INPUT_TYPES.has(name)) return 'text-input';
|
|
137
379
|
if (name && SWITCH_TYPES.has(name)) return 'switch';
|
|
380
|
+
if (hasRadioLikeSemantics(fiber, name, props)) return 'radio';
|
|
138
381
|
if (name && SLIDER_TYPES.has(name)) return 'slider';
|
|
139
382
|
if (name && PICKER_TYPES.has(name)) return 'picker';
|
|
140
383
|
if (name && DATE_PICKER_TYPES.has(name)) return 'date-picker';
|
|
141
384
|
|
|
142
385
|
// Check by accessibilityRole (covers custom components with proper ARIA)
|
|
143
386
|
const role = props.accessibilityRole || props.role;
|
|
387
|
+
if (role === 'radiogroup') return null;
|
|
388
|
+
if (role === 'radio') return 'radio';
|
|
144
389
|
if (role === 'switch') return 'switch';
|
|
145
|
-
if (role === 'adjustable') return 'slider';
|
|
146
|
-
if (role === 'button' || role === 'link' || role === 'checkbox'
|
|
390
|
+
if (role === 'adjustable' && hasSliderLikeSemantics(props)) return 'slider';
|
|
391
|
+
if (role === 'button' || role === 'link' || role === 'checkbox') {
|
|
147
392
|
return 'pressable';
|
|
148
393
|
}
|
|
149
394
|
|
|
@@ -153,9 +398,8 @@ function getElementType(fiber) {
|
|
|
153
398
|
// TextInput detection by props
|
|
154
399
|
if (props.onChangeText && typeof props.onChangeText === 'function') return 'text-input';
|
|
155
400
|
|
|
156
|
-
// Slider detection by props
|
|
157
|
-
if (props
|
|
158
|
-
if (props.onValueChange && typeof props.onValueChange === 'function' && (props.minimumValue !== undefined || props.maximumValue !== undefined)) return 'slider';
|
|
401
|
+
// Slider detection by props
|
|
402
|
+
if (hasSliderLikeSemantics(props)) return 'slider';
|
|
159
403
|
|
|
160
404
|
// DatePicker detection by props
|
|
161
405
|
if (props.onChange && typeof props.onChange === 'function' && (props.mode === 'date' || props.mode === 'time' || props.mode === 'datetime')) return 'date-picker';
|
|
@@ -233,6 +477,7 @@ function scoreRuntimeLabel(text, source) {
|
|
|
233
477
|
const words = normalized.split(/\s+/).filter(Boolean);
|
|
234
478
|
const lowered = normalized.toLowerCase();
|
|
235
479
|
if (source === 'deep-text') score += 70;
|
|
480
|
+
if (source === 'sibling-text') score += 58;
|
|
236
481
|
if (source === 'accessibility') score += 80;
|
|
237
482
|
if (source === 'placeholder') score += 25;
|
|
238
483
|
if (source === 'context') score += 10;
|
|
@@ -254,6 +499,25 @@ function chooseBestRuntimeLabel(candidates) {
|
|
|
254
499
|
})).filter(candidate => candidate.text).sort((a, b) => b.score - a.score)[0];
|
|
255
500
|
return best?.text || null;
|
|
256
501
|
}
|
|
502
|
+
function extractSiblingTextLabel(fiber) {
|
|
503
|
+
const parent = getParent(fiber);
|
|
504
|
+
if (!parent) return '';
|
|
505
|
+
const candidates = [];
|
|
506
|
+
let sibling = getChild(parent);
|
|
507
|
+
while (sibling) {
|
|
508
|
+
if (sibling !== fiber) {
|
|
509
|
+
const text = extractDeepTextContent(sibling, 4);
|
|
510
|
+
if (text) {
|
|
511
|
+
candidates.push({
|
|
512
|
+
text,
|
|
513
|
+
source: 'sibling-text'
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
sibling = getSibling(sibling);
|
|
518
|
+
}
|
|
519
|
+
return chooseBestRuntimeLabel(candidates) || '';
|
|
520
|
+
}
|
|
257
521
|
|
|
258
522
|
/**
|
|
259
523
|
* Extract raw text from React children prop.
|
|
@@ -457,13 +721,14 @@ export function walkFiberTree(rootRef, config) {
|
|
|
457
721
|
};
|
|
458
722
|
}
|
|
459
723
|
|
|
460
|
-
// Always walk from the root Fiber —
|
|
461
|
-
//
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
724
|
+
// Always walk from the absolute root Fiber (HostRoot) — this ensures we
|
|
725
|
+
// capture portals, overlays, and bottom sheets mounted outside the AIAgent.
|
|
726
|
+
let startNode = fiber;
|
|
727
|
+
while (getParent(startNode)) {
|
|
728
|
+
startNode = getParent(startNode);
|
|
729
|
+
}
|
|
465
730
|
if (config?.screenName) {
|
|
466
|
-
logger.debug('FiberTreeWalker', `Walk active for screen "${config.screenName}" (inactive screens pruned via display
|
|
731
|
+
logger.debug('FiberTreeWalker', `Walk active for screen "${config.screenName}" (inactive screens pruned via display checks)`);
|
|
467
732
|
}
|
|
468
733
|
|
|
469
734
|
// Overlay detection is superseded by root-level walk — all visible nodes
|
|
@@ -472,7 +737,7 @@ export function walkFiberTree(rootRef, config) {
|
|
|
472
737
|
const interactives = [];
|
|
473
738
|
let currentIndex = 0;
|
|
474
739
|
const hasWhitelist = config?.interactiveWhitelist && (config.interactiveWhitelist.length ?? 0) > 0;
|
|
475
|
-
function processNode(node, depth = 0, isInsideInteractive = false,
|
|
740
|
+
function processNode(node, depth = 0, isInsideInteractive = false, ancestorInteractionSignature = null, currentZoneId = undefined) {
|
|
476
741
|
if (!node) return '';
|
|
477
742
|
const props = getProps(node);
|
|
478
743
|
|
|
@@ -492,23 +757,36 @@ export function walkFiberTree(rootRef, config) {
|
|
|
492
757
|
if (props.activityState === 0) return '';
|
|
493
758
|
if (props.pointerEvents === 'none') return '';
|
|
494
759
|
|
|
760
|
+
// Fast heuristic for inline hidden styles (avoids expensive StyleSheet.flatten)
|
|
761
|
+
// Ensures Modals/Portals that are technically mounted but visually hidden are skipped.
|
|
762
|
+
if (props.style) {
|
|
763
|
+
if (props.style.display === 'none' || props.style.opacity === 0) return '';
|
|
764
|
+
if (Array.isArray(props.style)) {
|
|
765
|
+
for (let i = props.style.length - 1; i >= 0; i--) {
|
|
766
|
+
const s = props.style[i];
|
|
767
|
+
if (s && (s.display === 'none' || s.opacity === 0)) return '';
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
495
772
|
// ── Security Constraints ──
|
|
496
773
|
if (props.aiIgnore === true) return '';
|
|
497
774
|
if (matchesRefList(node, config?.interactiveBlacklist)) {
|
|
498
775
|
let childText = '';
|
|
499
776
|
let currentChild = getChild(node);
|
|
500
777
|
while (currentChild) {
|
|
501
|
-
childText += processNode(currentChild, depth, isInsideInteractive);
|
|
778
|
+
childText += processNode(currentChild, depth, isInsideInteractive, ancestorInteractionSignature, currentZoneId);
|
|
502
779
|
currentChild = getSibling(currentChild);
|
|
503
780
|
}
|
|
504
781
|
return childText;
|
|
505
782
|
}
|
|
506
783
|
|
|
507
|
-
// Interactive check — nested interactives
|
|
508
|
-
//
|
|
509
|
-
//
|
|
784
|
+
// Interactive check — nested interactives should only be suppressed when
|
|
785
|
+
// they reuse the same actionable handler as their interactive ancestor.
|
|
786
|
+
// This keeps real child controls like switches, pickers, and text inputs.
|
|
510
787
|
const isWhitelisted = matchesRefList(node, config?.interactiveWhitelist);
|
|
511
788
|
const elementType = getElementType(node);
|
|
789
|
+
const ownInteractionSignature = getInteractionSignature(elementType, props);
|
|
512
790
|
let shouldInclude = false;
|
|
513
791
|
if (hasWhitelist) {
|
|
514
792
|
shouldInclude = isWhitelisted;
|
|
@@ -516,14 +794,12 @@ export function walkFiberTree(rootRef, config) {
|
|
|
516
794
|
if (!isInsideInteractive) {
|
|
517
795
|
shouldInclude = true;
|
|
518
796
|
} else {
|
|
519
|
-
|
|
520
|
-
const ownOnPress = props.onPress;
|
|
521
|
-
shouldInclude = !!ownOnPress && ownOnPress !== ancestorOnPress;
|
|
797
|
+
shouldInclude = !!ownInteractionSignature && !interactionSignaturesMatch(ownInteractionSignature, ancestorInteractionSignature);
|
|
522
798
|
}
|
|
523
799
|
}
|
|
524
800
|
|
|
525
|
-
// Track the
|
|
526
|
-
const
|
|
801
|
+
// Track the actionable signature for descendant dedup.
|
|
802
|
+
const nextAncestorInteractionSignature = shouldInclude ? ownInteractionSignature || ancestorInteractionSignature : ancestorInteractionSignature;
|
|
527
803
|
|
|
528
804
|
// Determine if this node produces visible output (affects depth for children)
|
|
529
805
|
// Only visible nodes (interactives, text, images, videos) should increment
|
|
@@ -559,7 +835,7 @@ export function walkFiberTree(rootRef, config) {
|
|
|
559
835
|
let childrenText = '';
|
|
560
836
|
let currentChild = getChild(node);
|
|
561
837
|
while (currentChild) {
|
|
562
|
-
childrenText += processNode(currentChild, childDepth, isInsideInteractive || !!shouldInclude,
|
|
838
|
+
childrenText += processNode(currentChild, childDepth, isInsideInteractive || !!shouldInclude, nextAncestorInteractionSignature, nextZoneId);
|
|
563
839
|
currentChild = getSibling(currentChild);
|
|
564
840
|
}
|
|
565
841
|
|
|
@@ -569,21 +845,26 @@ export function walkFiberTree(rootRef, config) {
|
|
|
569
845
|
}
|
|
570
846
|
if (shouldInclude) {
|
|
571
847
|
const resolvedType = elementType || 'pressable';
|
|
848
|
+
const derivedProps = buildDerivedElementProps(node, resolvedType, props);
|
|
572
849
|
const parentContext = getNearestCustomComponentName(node);
|
|
850
|
+
const siblingTextLabel = EXTERNALLY_LABELED_TYPES.has(resolvedType) ? extractSiblingTextLabel(node) : null;
|
|
573
851
|
const label = chooseBestRuntimeLabel([{
|
|
574
|
-
text:
|
|
852
|
+
text: derivedProps.accessibilityLabel,
|
|
575
853
|
source: 'accessibility'
|
|
576
854
|
}, {
|
|
577
855
|
text: extractDeepTextContent(node),
|
|
578
856
|
source: 'deep-text'
|
|
579
857
|
}, {
|
|
580
|
-
text:
|
|
858
|
+
text: siblingTextLabel,
|
|
859
|
+
source: 'sibling-text'
|
|
860
|
+
}, {
|
|
861
|
+
text: resolvedType === 'text-input' ? derivedProps.placeholder : null,
|
|
581
862
|
source: 'placeholder'
|
|
582
863
|
}, {
|
|
583
864
|
text: extractIconName(node),
|
|
584
865
|
source: 'icon'
|
|
585
866
|
}, {
|
|
586
|
-
text:
|
|
867
|
+
text: derivedProps.testID || derivedProps.nativeID,
|
|
587
868
|
source: 'test-id'
|
|
588
869
|
}, {
|
|
589
870
|
text: parentContext && !new Set(['ScrollViewContext', 'VirtualizedListContext', 'ViewabilityHelper', 'ScrollResponder', 'AnimatedComponent', 'TouchableOpacity']).has(parentContext) ? parentContext : null,
|
|
@@ -596,20 +877,18 @@ export function walkFiberTree(rootRef, config) {
|
|
|
596
877
|
aiPriority: props.aiPriority,
|
|
597
878
|
zoneId: currentZoneId,
|
|
598
879
|
fiberNode: node,
|
|
599
|
-
props:
|
|
600
|
-
|
|
601
|
-
},
|
|
602
|
-
requiresConfirmation: props.aiConfirm === true
|
|
880
|
+
props: derivedProps,
|
|
881
|
+
requiresConfirmation: derivedProps.aiConfirm === true
|
|
603
882
|
});
|
|
604
883
|
|
|
605
884
|
// Build output tag with state attributes
|
|
606
|
-
const stateAttrs = extractStateAttributes(
|
|
885
|
+
const stateAttrs = extractStateAttributes(derivedProps);
|
|
607
886
|
let attrStr = stateAttrs ? ` ${stateAttrs}` : '';
|
|
608
|
-
if (
|
|
609
|
-
attrStr += ` aiPriority="${
|
|
887
|
+
if (derivedProps.aiPriority) {
|
|
888
|
+
attrStr += ` aiPriority="${derivedProps.aiPriority}"`;
|
|
610
889
|
if (currentZoneId) attrStr += ` zoneId="${currentZoneId}"`;
|
|
611
890
|
}
|
|
612
|
-
if (
|
|
891
|
+
if (derivedProps.aiConfirm === true) {
|
|
613
892
|
attrStr += ' aiConfirm';
|
|
614
893
|
}
|
|
615
894
|
const textContent = label || '';
|
|
@@ -893,4 +1172,101 @@ function resolveNativeScrollRef(fiberNode) {
|
|
|
893
1172
|
logger.debug('FiberTreeWalker', 'Could not resolve native scroll ref — returning stateNode as fallback');
|
|
894
1173
|
return stateNode;
|
|
895
1174
|
}
|
|
1175
|
+
|
|
1176
|
+
// ─── Wireframe Capture ─────────────────────────────────────────
|
|
1177
|
+
|
|
1178
|
+
/** Max elements to measure — keeps bridge work bounded */
|
|
1179
|
+
const WIREFRAME_MAX_ELEMENTS = 50;
|
|
1180
|
+
/** Measure this many elements per frame, then yield */
|
|
1181
|
+
const WIREFRAME_BATCH_SIZE = 10;
|
|
1182
|
+
|
|
1183
|
+
/**
|
|
1184
|
+
* Measure a single element on the native bridge.
|
|
1185
|
+
* Returns null if the element is off-screen or unmeasurable.
|
|
1186
|
+
*/
|
|
1187
|
+
function measureElement(el) {
|
|
1188
|
+
return new Promise(resolve => {
|
|
1189
|
+
try {
|
|
1190
|
+
const stateNode = getStateNode(el.fiberNode);
|
|
1191
|
+
if (!stateNode || typeof stateNode.measure !== 'function') {
|
|
1192
|
+
resolve(null);
|
|
1193
|
+
return;
|
|
1194
|
+
}
|
|
1195
|
+
stateNode.measure((_x, _y, width, height, pageX, pageY) => {
|
|
1196
|
+
if (width > 0 && height > 0) {
|
|
1197
|
+
resolve({
|
|
1198
|
+
type: el.type,
|
|
1199
|
+
label: el.label || el.type,
|
|
1200
|
+
x: pageX,
|
|
1201
|
+
y: pageY,
|
|
1202
|
+
width,
|
|
1203
|
+
height
|
|
1204
|
+
});
|
|
1205
|
+
} else {
|
|
1206
|
+
resolve(null);
|
|
1207
|
+
}
|
|
1208
|
+
});
|
|
1209
|
+
} catch {
|
|
1210
|
+
resolve(null);
|
|
1211
|
+
}
|
|
1212
|
+
});
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
/**
|
|
1216
|
+
* Yield one frame so measure work doesn't block gestures/animations.
|
|
1217
|
+
* Uses requestAnimationFrame where available, falls back to setTimeout(16ms).
|
|
1218
|
+
*/
|
|
1219
|
+
function yieldFrame() {
|
|
1220
|
+
return new Promise(resolve => {
|
|
1221
|
+
if (typeof requestAnimationFrame === 'function') {
|
|
1222
|
+
requestAnimationFrame(() => resolve());
|
|
1223
|
+
} else {
|
|
1224
|
+
setTimeout(resolve, 16);
|
|
1225
|
+
}
|
|
1226
|
+
});
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
/**
|
|
1230
|
+
* Capture a privacy-safe wireframe of the current screen.
|
|
1231
|
+
*
|
|
1232
|
+
* Performance guarantees:
|
|
1233
|
+
* - Capped at WIREFRAME_MAX_ELEMENTS (50) — enough for wireframe context
|
|
1234
|
+
* - Measures in batches of WIREFRAME_BATCH_SIZE (10), yielding a frame
|
|
1235
|
+
* between batches so the bridge stays free for user interactions
|
|
1236
|
+
* - The caller (AIAgent) defers this via InteractionManager so it
|
|
1237
|
+
* never competes with screen transitions or gestures
|
|
1238
|
+
*/
|
|
1239
|
+
export async function captureWireframe(rootRef, config = {}) {
|
|
1240
|
+
const result = walkFiberTree(rootRef, config);
|
|
1241
|
+
const elements = result.interactives;
|
|
1242
|
+
if (elements.length === 0) return null;
|
|
1243
|
+
|
|
1244
|
+
// Cap the number of elements to keep bridge work bounded
|
|
1245
|
+
const capped = elements.slice(0, WIREFRAME_MAX_ELEMENTS);
|
|
1246
|
+
const components = [];
|
|
1247
|
+
for (let i = 0; i < capped.length; i += WIREFRAME_BATCH_SIZE) {
|
|
1248
|
+
const batch = capped.slice(i, i + WIREFRAME_BATCH_SIZE);
|
|
1249
|
+
const batchResults = await Promise.all(batch.map(measureElement));
|
|
1250
|
+
for (const r of batchResults) {
|
|
1251
|
+
if (r) components.push(r);
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
// Yield between batches — never monopolize the bridge
|
|
1255
|
+
if (i + WIREFRAME_BATCH_SIZE < capped.length) {
|
|
1256
|
+
await yieldFrame();
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
if (components.length === 0) return null;
|
|
1260
|
+
const {
|
|
1261
|
+
width: deviceWidth,
|
|
1262
|
+
height: deviceHeight
|
|
1263
|
+
} = Dimensions.get('window');
|
|
1264
|
+
return {
|
|
1265
|
+
screen: config.screenName || 'Unknown',
|
|
1266
|
+
components,
|
|
1267
|
+
deviceWidth,
|
|
1268
|
+
deviceHeight,
|
|
1269
|
+
capturedAt: new Date().toISOString()
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
896
1272
|
//# sourceMappingURL=FiberTreeWalker.js.map
|