@mobileai/react-native 0.9.16 → 0.9.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +5 -8
- package/lib/module/__cli_tmp__.js.map +0 -1
- package/lib/module/components/AIAgent.js.map +0 -1
- package/lib/module/components/AIZone.js.map +0 -1
- package/lib/module/components/AgentChatBar.js.map +0 -1
- package/lib/module/components/AgentErrorBoundary.js.map +0 -1
- package/lib/module/components/AgentOverlay.js.map +0 -1
- package/lib/module/components/DiscoveryTooltip.js.map +0 -1
- package/lib/module/components/HighlightOverlay.js.map +0 -1
- package/lib/module/components/Icons.js.map +0 -1
- package/lib/module/components/ProactiveHint.js.map +0 -1
- package/lib/module/components/cards/InfoCard.js.map +0 -1
- package/lib/module/components/cards/ReviewSummary.js.map +0 -1
- package/lib/module/config/endpoints.js.map +0 -1
- package/lib/module/core/ActionRegistry.js.map +0 -1
- package/lib/module/core/AgentRuntime.js.map +0 -1
- package/lib/module/core/FiberTreeWalker.js.map +0 -1
- package/lib/module/core/IdleDetector.js.map +0 -1
- package/lib/module/core/MCPBridge.js.map +0 -1
- package/lib/module/core/ScreenDehydrator.js.map +0 -1
- package/lib/module/core/ZoneRegistry.js.map +0 -1
- package/lib/module/core/systemPrompt.js.map +0 -1
- package/lib/module/core/types.js.map +0 -1
- package/lib/module/hooks/useAction.js.map +0 -1
- package/lib/module/index.js.map +0 -1
- package/lib/module/plugin/withAppIntents.js.map +0 -1
- package/lib/module/providers/GeminiProvider.js.map +0 -1
- package/lib/module/providers/OpenAIProvider.js.map +0 -1
- package/lib/module/providers/ProviderFactory.js.map +0 -1
- package/lib/module/services/AudioInputService.js.map +0 -1
- package/lib/module/services/AudioOutputService.js.map +0 -1
- package/lib/module/services/KnowledgeBaseService.js.map +0 -1
- package/lib/module/services/VoiceService.js.map +0 -1
- package/lib/module/services/flags/FlagService.js.map +0 -1
- package/lib/module/services/telemetry/MobileAI.js.map +0 -1
- package/lib/module/services/telemetry/PiiScrubber.js.map +0 -1
- package/lib/module/services/telemetry/TelemetryService.js.map +0 -1
- package/lib/module/services/telemetry/TouchAutoCapture.js.map +0 -1
- package/lib/module/services/telemetry/device.js.map +0 -1
- package/lib/module/services/telemetry/deviceMetadata.js.map +0 -1
- package/lib/module/services/telemetry/index.js.map +0 -1
- package/lib/module/services/telemetry/types.js.map +0 -1
- package/lib/module/support/CSATSurvey.js.map +0 -1
- package/lib/module/support/EscalationEventSource.js.map +0 -1
- package/lib/module/support/EscalationSocket.js.map +0 -1
- package/lib/module/support/SupportChatModal.js.map +0 -1
- package/lib/module/support/SupportGreeting.js.map +0 -1
- package/lib/module/support/TicketStore.js.map +0 -1
- package/lib/module/support/escalateTool.js.map +0 -1
- package/lib/module/support/index.js.map +0 -1
- package/lib/module/support/supportPrompt.js.map +0 -1
- package/lib/module/support/types.js.map +0 -1
- package/lib/module/tools/datePickerTool.js.map +0 -1
- package/lib/module/tools/guideTool.js.map +0 -1
- package/lib/module/tools/index.js.map +0 -1
- package/lib/module/tools/keyboardTool.js.map +0 -1
- package/lib/module/tools/longPressTool.js.map +0 -1
- package/lib/module/tools/pickerTool.js.map +0 -1
- package/lib/module/tools/restoreTool.js.map +0 -1
- package/lib/module/tools/scrollTool.js.map +0 -1
- package/lib/module/tools/simplifyTool.js.map +0 -1
- package/lib/module/tools/sliderTool.js.map +0 -1
- package/lib/module/tools/tapTool.js.map +0 -1
- package/lib/module/tools/typeTool.js.map +0 -1
- package/lib/module/tools/types.js.map +0 -1
- package/lib/module/types/jsx.d.js.map +0 -1
- package/lib/module/utils/audioUtils.js.map +0 -1
- package/lib/module/utils/logger.js.map +0 -1
- package/lib/typescript/babel.config.d.ts.map +0 -1
- package/lib/typescript/bin/generate-map.d.cts.map +0 -1
- package/lib/typescript/eslint.config.d.mts.map +0 -1
- package/lib/typescript/generate-map.d.ts.map +0 -1
- package/lib/typescript/src/__cli_tmp__.d.ts.map +0 -1
- package/lib/typescript/src/components/AIAgent.d.ts.map +0 -1
- package/lib/typescript/src/components/AIZone.d.ts.map +0 -1
- package/lib/typescript/src/components/AgentChatBar.d.ts.map +0 -1
- package/lib/typescript/src/components/AgentErrorBoundary.d.ts.map +0 -1
- package/lib/typescript/src/components/AgentOverlay.d.ts.map +0 -1
- package/lib/typescript/src/components/DiscoveryTooltip.d.ts.map +0 -1
- package/lib/typescript/src/components/HighlightOverlay.d.ts.map +0 -1
- package/lib/typescript/src/components/Icons.d.ts.map +0 -1
- package/lib/typescript/src/components/ProactiveHint.d.ts.map +0 -1
- package/lib/typescript/src/components/cards/InfoCard.d.ts.map +0 -1
- package/lib/typescript/src/components/cards/ReviewSummary.d.ts.map +0 -1
- package/lib/typescript/src/config/endpoints.d.ts.map +0 -1
- package/lib/typescript/src/core/ActionRegistry.d.ts.map +0 -1
- package/lib/typescript/src/core/AgentRuntime.d.ts.map +0 -1
- package/lib/typescript/src/core/FiberTreeWalker.d.ts.map +0 -1
- package/lib/typescript/src/core/IdleDetector.d.ts.map +0 -1
- package/lib/typescript/src/core/MCPBridge.d.ts.map +0 -1
- package/lib/typescript/src/core/ScreenDehydrator.d.ts.map +0 -1
- package/lib/typescript/src/core/ZoneRegistry.d.ts.map +0 -1
- package/lib/typescript/src/core/systemPrompt.d.ts.map +0 -1
- package/lib/typescript/src/core/types.d.ts.map +0 -1
- package/lib/typescript/src/hooks/useAction.d.ts.map +0 -1
- package/lib/typescript/src/index.d.ts.map +0 -1
- package/lib/typescript/src/plugin/withAppIntents.d.ts.map +0 -1
- package/lib/typescript/src/providers/GeminiProvider.d.ts.map +0 -1
- package/lib/typescript/src/providers/OpenAIProvider.d.ts.map +0 -1
- package/lib/typescript/src/providers/ProviderFactory.d.ts.map +0 -1
- package/lib/typescript/src/services/AudioInputService.d.ts.map +0 -1
- package/lib/typescript/src/services/AudioOutputService.d.ts.map +0 -1
- package/lib/typescript/src/services/KnowledgeBaseService.d.ts.map +0 -1
- package/lib/typescript/src/services/VoiceService.d.ts.map +0 -1
- package/lib/typescript/src/services/flags/FlagService.d.ts.map +0 -1
- package/lib/typescript/src/services/telemetry/MobileAI.d.ts.map +0 -1
- package/lib/typescript/src/services/telemetry/PiiScrubber.d.ts.map +0 -1
- package/lib/typescript/src/services/telemetry/TelemetryService.d.ts.map +0 -1
- package/lib/typescript/src/services/telemetry/TouchAutoCapture.d.ts.map +0 -1
- package/lib/typescript/src/services/telemetry/device.d.ts.map +0 -1
- package/lib/typescript/src/services/telemetry/deviceMetadata.d.ts.map +0 -1
- package/lib/typescript/src/services/telemetry/index.d.ts.map +0 -1
- package/lib/typescript/src/services/telemetry/types.d.ts.map +0 -1
- package/lib/typescript/src/support/CSATSurvey.d.ts.map +0 -1
- package/lib/typescript/src/support/EscalationEventSource.d.ts.map +0 -1
- package/lib/typescript/src/support/EscalationSocket.d.ts.map +0 -1
- package/lib/typescript/src/support/SupportChatModal.d.ts.map +0 -1
- package/lib/typescript/src/support/SupportGreeting.d.ts.map +0 -1
- package/lib/typescript/src/support/TicketStore.d.ts.map +0 -1
- package/lib/typescript/src/support/escalateTool.d.ts.map +0 -1
- package/lib/typescript/src/support/index.d.ts.map +0 -1
- package/lib/typescript/src/support/supportPrompt.d.ts.map +0 -1
- package/lib/typescript/src/support/types.d.ts.map +0 -1
- package/lib/typescript/src/tools/datePickerTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/guideTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/index.d.ts.map +0 -1
- package/lib/typescript/src/tools/keyboardTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/longPressTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/pickerTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/restoreTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/scrollTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/simplifyTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/sliderTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/tapTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/typeTool.d.ts.map +0 -1
- package/lib/typescript/src/tools/types.d.ts.map +0 -1
- package/lib/typescript/src/utils/audioUtils.d.ts.map +0 -1
- package/lib/typescript/src/utils/logger.d.ts.map +0 -1
- package/src/__cli_tmp__.tsx +0 -9
- package/src/cli/analyzers/chain-analyzer.ts +0 -183
- package/src/cli/extractors/ai-extractor.ts +0 -6
- package/src/cli/extractors/ast-extractor.ts +0 -551
- package/src/cli/generate-intents.ts +0 -140
- package/src/cli/generate-map.ts +0 -121
- package/src/cli/generate-swift.ts +0 -116
- package/src/cli/scanners/expo-scanner.ts +0 -203
- package/src/cli/scanners/rn-scanner.ts +0 -445
- package/src/components/AIAgent.tsx +0 -1716
- package/src/components/AIZone.tsx +0 -147
- package/src/components/AgentChatBar.tsx +0 -1143
- package/src/components/AgentErrorBoundary.tsx +0 -78
- package/src/components/AgentOverlay.tsx +0 -73
- package/src/components/DiscoveryTooltip.tsx +0 -148
- package/src/components/HighlightOverlay.tsx +0 -136
- package/src/components/Icons.tsx +0 -253
- package/src/components/ProactiveHint.tsx +0 -145
- package/src/components/cards/InfoCard.tsx +0 -58
- package/src/components/cards/ReviewSummary.tsx +0 -76
- package/src/config/endpoints.ts +0 -22
- package/src/core/ActionRegistry.ts +0 -105
- package/src/core/AgentRuntime.ts +0 -1471
- package/src/core/FiberTreeWalker.ts +0 -930
- package/src/core/IdleDetector.ts +0 -72
- package/src/core/MCPBridge.ts +0 -163
- package/src/core/ScreenDehydrator.ts +0 -53
- package/src/core/ZoneRegistry.ts +0 -44
- package/src/core/systemPrompt.ts +0 -431
- package/src/core/types.ts +0 -521
- package/src/hooks/useAction.ts +0 -182
- package/src/index.ts +0 -83
- package/src/plugin/withAppIntents.ts +0 -98
- package/src/providers/GeminiProvider.ts +0 -357
- package/src/providers/OpenAIProvider.ts +0 -379
- package/src/providers/ProviderFactory.ts +0 -36
- package/src/services/AudioInputService.ts +0 -226
- package/src/services/AudioOutputService.ts +0 -236
- package/src/services/KnowledgeBaseService.ts +0 -156
- package/src/services/VoiceService.ts +0 -451
- package/src/services/flags/FlagService.ts +0 -137
- package/src/services/telemetry/MobileAI.ts +0 -66
- package/src/services/telemetry/PiiScrubber.ts +0 -17
- package/src/services/telemetry/TelemetryService.ts +0 -323
- package/src/services/telemetry/TouchAutoCapture.ts +0 -165
- package/src/services/telemetry/device.ts +0 -93
- package/src/services/telemetry/deviceMetadata.ts +0 -13
- package/src/services/telemetry/index.ts +0 -13
- package/src/services/telemetry/types.ts +0 -75
- package/src/support/CSATSurvey.tsx +0 -304
- package/src/support/EscalationEventSource.ts +0 -190
- package/src/support/EscalationSocket.ts +0 -152
- package/src/support/SupportChatModal.tsx +0 -563
- package/src/support/SupportGreeting.tsx +0 -161
- package/src/support/TicketStore.ts +0 -100
- package/src/support/escalateTool.ts +0 -174
- package/src/support/index.ts +0 -29
- package/src/support/supportPrompt.ts +0 -55
- package/src/support/types.ts +0 -155
- package/src/tools/datePickerTool.ts +0 -60
- package/src/tools/guideTool.ts +0 -76
- package/src/tools/index.ts +0 -20
- package/src/tools/keyboardTool.ts +0 -30
- package/src/tools/longPressTool.ts +0 -61
- package/src/tools/pickerTool.ts +0 -115
- package/src/tools/restoreTool.ts +0 -33
- package/src/tools/scrollTool.ts +0 -156
- package/src/tools/simplifyTool.ts +0 -33
- package/src/tools/sliderTool.ts +0 -65
- package/src/tools/tapTool.ts +0 -93
- package/src/tools/typeTool.ts +0 -113
- package/src/tools/types.ts +0 -58
- package/src/types/jsx.d.ts +0 -20
- package/src/utils/audioUtils.ts +0 -54
- package/src/utils/logger.ts +0 -38
|
@@ -1,930 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* FiberTreeWalker — Traverses React's Fiber tree to discover interactive elements.
|
|
3
|
-
*
|
|
4
|
-
* Walks the React Native fiber tree to extract a text representation of the UI.
|
|
5
|
-
* Instead of traversing HTML nodes, we traverse React Fiber nodes and detect
|
|
6
|
-
* interactive elements by their type and props (onPress, onChangeText, etc.).
|
|
7
|
-
*
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
import { logger } from '../utils/logger';
|
|
11
|
-
import type { InteractiveElement, ElementType } from './types';
|
|
12
|
-
|
|
13
|
-
// ─── Walk Configuration ─────────
|
|
14
|
-
|
|
15
|
-
export interface WalkConfig {
|
|
16
|
-
/** React refs of elements to exclude */
|
|
17
|
-
interactiveBlacklist?: React.RefObject<any>[];
|
|
18
|
-
/** If set, only these elements are interactive */
|
|
19
|
-
interactiveWhitelist?: React.RefObject<any>[];
|
|
20
|
-
/** Optional screen name to scope interactives to the active screen */
|
|
21
|
-
screenName?: string;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
// ─── Fiber Node Type Detection ─────────────────────────────────
|
|
25
|
-
|
|
26
|
-
/** React Native component names that are inherently interactive */
|
|
27
|
-
const PRESSABLE_TYPES = new Set([
|
|
28
|
-
'Pressable',
|
|
29
|
-
'TouchableOpacity',
|
|
30
|
-
'TouchableHighlight',
|
|
31
|
-
'TouchableWithoutFeedback',
|
|
32
|
-
'TouchableNativeFeedback',
|
|
33
|
-
'Button',
|
|
34
|
-
]);
|
|
35
|
-
|
|
36
|
-
const TEXT_INPUT_TYPES = new Set(['TextInput', 'RCTSinglelineTextInputView', 'RCTMultilineTextInputView']);
|
|
37
|
-
const SWITCH_TYPES = new Set(['Switch', 'RCTSwitch']);
|
|
38
|
-
const SLIDER_TYPES = new Set(['Slider', 'RNCSlider', 'RCTSlider']);
|
|
39
|
-
const PICKER_TYPES = new Set(['Picker', 'RNCPicker', 'RNPickerSelect', 'DropDownPicker', 'SelectDropdown']);
|
|
40
|
-
const DATE_PICKER_TYPES = new Set(['DateTimePicker', 'RNDateTimePicker', 'DatePicker', 'RNDatePicker']);
|
|
41
|
-
const TEXT_TYPES = new Set(['Text', 'RCTText']);
|
|
42
|
-
|
|
43
|
-
// Media component types for Component-Context Media Inference
|
|
44
|
-
const IMAGE_TYPES = new Set([
|
|
45
|
-
'Image', 'RCTImageView', 'ExpoImage', 'FastImage', 'CachedImage',
|
|
46
|
-
]);
|
|
47
|
-
const VIDEO_TYPES = new Set([
|
|
48
|
-
'Video', 'ExpoVideo', 'RCTVideo', 'VideoPlayer', 'VideoView',
|
|
49
|
-
]);
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
// Known RN internal component names to skip when walking up for context
|
|
54
|
-
const RN_INTERNAL_NAMES = new Set([
|
|
55
|
-
'View', 'RCTView', 'Pressable', 'TouchableOpacity', 'TouchableHighlight',
|
|
56
|
-
'ScrollView', 'RCTScrollView', 'FlatList', 'SectionList',
|
|
57
|
-
'SafeAreaView', 'RNCSafeAreaView', 'KeyboardAvoidingView',
|
|
58
|
-
'Modal', 'StatusBar', 'Text', 'RCTText', 'AnimatedComponent',
|
|
59
|
-
'AnimatedComponentWrapper', 'Animated',
|
|
60
|
-
]);
|
|
61
|
-
|
|
62
|
-
// ─── State Extraction ──
|
|
63
|
-
|
|
64
|
-
/** Props to extract as state attributes — covers lazy devs who skip accessibility */
|
|
65
|
-
const STATE_PROPS = ['value', 'checked', 'selected', 'active', 'on', 'isOn', 'toggled', 'enabled'];
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Extract state attributes from a fiber node's props.
|
|
69
|
-
* Extracts meaningful state from a fiber node.
|
|
70
|
-
* Priority: accessibilityState > accessibilityRole > direct scalar props.
|
|
71
|
-
*/
|
|
72
|
-
function extractStateAttributes(props: any): string {
|
|
73
|
-
const parts: string[] = [];
|
|
74
|
-
|
|
75
|
-
// Priority 1: accessibilityState (proper ARIA equivalent)
|
|
76
|
-
if (props.accessibilityState && typeof props.accessibilityState === 'object') {
|
|
77
|
-
for (const [k, v] of Object.entries(props.accessibilityState)) {
|
|
78
|
-
if (v !== undefined) parts.push(`${k}="${v}"`);
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
// Priority 2: accessibilityRole
|
|
83
|
-
if (props.accessibilityRole) {
|
|
84
|
-
parts.push(`role="${props.accessibilityRole}"`);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
// Priority 3: Direct scalar props fallback (lazy developer support)
|
|
88
|
-
for (const key of STATE_PROPS) {
|
|
89
|
-
if (props[key] !== undefined && typeof props[key] !== 'function' && typeof props[key] !== 'object') {
|
|
90
|
-
parts.push(`${key}="${props[key]}"`);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
return parts.join(' ');
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
/**
|
|
98
|
-
* Check if a node has ANY event handler prop (on* function).
|
|
99
|
-
* Mirrors RNTL's getEventHandlerFromProps pattern.
|
|
100
|
-
*/
|
|
101
|
-
export function hasAnyEventHandler(props: any): boolean {
|
|
102
|
-
if (!props || typeof props !== 'object') return false;
|
|
103
|
-
for (const key of Object.keys(props)) {
|
|
104
|
-
if (key.startsWith('on') && typeof props[key] === 'function') {
|
|
105
|
-
return true;
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
return false;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/**
|
|
112
|
-
* Walk UP the Fiber tree to find the nearest custom (user-defined) component name.
|
|
113
|
-
* Skips known React Native internal component names.
|
|
114
|
-
* This provides semantic context for media elements (e.g., an Image inside "ProfileHeader").
|
|
115
|
-
*/
|
|
116
|
-
function getNearestCustomComponentName(fiber: any, maxDepth: number = 8): string | null {
|
|
117
|
-
let current = fiber?.return;
|
|
118
|
-
let depth = 0;
|
|
119
|
-
while (current && depth < maxDepth) {
|
|
120
|
-
const name = getComponentName(current);
|
|
121
|
-
if (name && !RN_INTERNAL_NAMES.has(name) && !PRESSABLE_TYPES.has(name)) {
|
|
122
|
-
return name;
|
|
123
|
-
}
|
|
124
|
-
current = current.return;
|
|
125
|
-
depth++;
|
|
126
|
-
}
|
|
127
|
-
return null;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
// ─── Fiber Node Helpers ────────────────────────────────────────
|
|
131
|
-
|
|
132
|
-
/**
|
|
133
|
-
* Get the display name of a Fiber node's component type.
|
|
134
|
-
*/
|
|
135
|
-
function getComponentName(fiber: any): string | null {
|
|
136
|
-
if (!fiber || !fiber.type) return null;
|
|
137
|
-
|
|
138
|
-
// Host components (View, Text, etc.) — type is a string
|
|
139
|
-
if (typeof fiber.type === 'string') return fiber.type;
|
|
140
|
-
|
|
141
|
-
// Function/Class components — type has displayName or name
|
|
142
|
-
if (fiber.type.displayName) return fiber.type.displayName;
|
|
143
|
-
if (fiber.type.name) return fiber.type.name;
|
|
144
|
-
|
|
145
|
-
// ForwardRef components
|
|
146
|
-
if (fiber.type.render?.displayName) return fiber.type.render.displayName;
|
|
147
|
-
if (fiber.type.render?.name) return fiber.type.render.name;
|
|
148
|
-
|
|
149
|
-
return null;
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
/**
|
|
153
|
-
* Check if a fiber node represents an interactive element.
|
|
154
|
-
*/
|
|
155
|
-
function getElementType(fiber: any): ElementType | null {
|
|
156
|
-
const name = getComponentName(fiber);
|
|
157
|
-
const props = fiber.memoizedProps || {};
|
|
158
|
-
|
|
159
|
-
// Check by component name (known React Native types)
|
|
160
|
-
if (name && PRESSABLE_TYPES.has(name)) return 'pressable';
|
|
161
|
-
if (name && TEXT_INPUT_TYPES.has(name)) return 'text-input';
|
|
162
|
-
if (name && SWITCH_TYPES.has(name)) return 'switch';
|
|
163
|
-
if (name && SLIDER_TYPES.has(name)) return 'slider';
|
|
164
|
-
if (name && PICKER_TYPES.has(name)) return 'picker';
|
|
165
|
-
if (name && DATE_PICKER_TYPES.has(name)) return 'date-picker';
|
|
166
|
-
|
|
167
|
-
// Check by accessibilityRole (covers custom components with proper ARIA)
|
|
168
|
-
const role = props.accessibilityRole || props.role;
|
|
169
|
-
if (role === 'switch') return 'switch';
|
|
170
|
-
if (role === 'adjustable') return 'slider';
|
|
171
|
-
if (role === 'button' || role === 'link' || role === 'checkbox' || role === 'radio') {
|
|
172
|
-
return 'pressable';
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// Check by props — any component with onPress is interactive
|
|
176
|
-
if (props.onPress && typeof props.onPress === 'function') return 'pressable';
|
|
177
|
-
|
|
178
|
-
// TextInput detection by props
|
|
179
|
-
if (props.onChangeText && typeof props.onChangeText === 'function') return 'text-input';
|
|
180
|
-
|
|
181
|
-
// Slider detection by props (has both onValueChange AND min/max values)
|
|
182
|
-
if (props.onSlidingComplete && typeof props.onSlidingComplete === 'function') return 'slider';
|
|
183
|
-
if (props.onValueChange && typeof props.onValueChange === 'function' &&
|
|
184
|
-
(props.minimumValue !== undefined || props.maximumValue !== undefined)) return 'slider';
|
|
185
|
-
|
|
186
|
-
// DatePicker detection by props
|
|
187
|
-
if (props.onChange && typeof props.onChange === 'function' &&
|
|
188
|
-
(props.mode === 'date' || props.mode === 'time' || props.mode === 'datetime')) return 'date-picker';
|
|
189
|
-
|
|
190
|
-
// Switch detection by props (custom switches with onValueChange — no min/max)
|
|
191
|
-
if (props.onValueChange && typeof props.onValueChange === 'function') return 'switch';
|
|
192
|
-
|
|
193
|
-
return null;
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
/**
|
|
197
|
-
* Check if element is disabled.
|
|
198
|
-
*/
|
|
199
|
-
function isDisabled(fiber: any): boolean {
|
|
200
|
-
const props = fiber.memoizedProps || {};
|
|
201
|
-
return props.disabled === true || props.editable === false;
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
/**
|
|
205
|
-
* Recursively extract ALL text content from a fiber's children.
|
|
206
|
-
* Pierces through nested interactive elements — unlike typical tree walkers
|
|
207
|
-
* that stop at inner Pressable/TouchableOpacity boundaries.
|
|
208
|
-
*
|
|
209
|
-
* This is critical for wrapper components (e.g. ZButton → internal
|
|
210
|
-
* TouchableOpacity → Text) where stopping at nested interactives
|
|
211
|
-
* would lose the text label entirely.
|
|
212
|
-
*/
|
|
213
|
-
function extractDeepTextContent(fiber: any, maxDepth: number = 10): string {
|
|
214
|
-
if (!fiber || maxDepth <= 0) return '';
|
|
215
|
-
|
|
216
|
-
const parts: string[] = [];
|
|
217
|
-
|
|
218
|
-
let child = fiber.child;
|
|
219
|
-
while (child) {
|
|
220
|
-
const childName = getComponentName(child);
|
|
221
|
-
const childProps = child.memoizedProps || {};
|
|
222
|
-
|
|
223
|
-
// Text node — extract content
|
|
224
|
-
if (childName && TEXT_TYPES.has(childName)) {
|
|
225
|
-
const text = extractRawText(childProps.children);
|
|
226
|
-
// Filter out icon font glyphs (Private Use Area unicode chars U+E000–U+F8FF)
|
|
227
|
-
// Icon libraries (Ionicons, MaterialIcons, etc.) render as <Text> with
|
|
228
|
-
// single-char glyphs that look blank in output but block icon name fallback
|
|
229
|
-
if (text && !isIconGlyph(text)) {
|
|
230
|
-
parts.push(text);
|
|
231
|
-
}
|
|
232
|
-
} else {
|
|
233
|
-
// Recurse into ALL children, including nested interactives
|
|
234
|
-
const nestedText = extractDeepTextContent(child, maxDepth - 1);
|
|
235
|
-
if (nestedText) parts.push(nestedText);
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
child = child.sibling;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
return parts.join(' ').trim();
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
/**
|
|
245
|
-
* Check if a string is an icon font glyph character.
|
|
246
|
-
* Icon fonts use Unicode Private Use Area (PUA) characters:
|
|
247
|
-
* - Basic PUA: U+E000–U+F8FF
|
|
248
|
-
* - Supplementary PUA-A: U+F0000–U+FFFFD
|
|
249
|
-
* - Supplementary PUA-B: U+100000–U+10FFFD
|
|
250
|
-
*/
|
|
251
|
-
function isIconGlyph(text: string): boolean {
|
|
252
|
-
const trimmed = text.trim();
|
|
253
|
-
if (trimmed.length === 0 || trimmed.length > 2) return false; // Glyphs are 1-2 chars (surrogate pairs)
|
|
254
|
-
const code = trimmed.codePointAt(0) || 0;
|
|
255
|
-
return (code >= 0xE000 && code <= 0xF8FF) ||
|
|
256
|
-
(code >= 0xF0000 && code <= 0xFFFFF) ||
|
|
257
|
-
(code >= 0x100000 && code <= 0x10FFFF);
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
/**
|
|
261
|
-
* Extract raw text from React children prop.
|
|
262
|
-
* Handles strings, numbers, arrays, and nested structures.
|
|
263
|
-
*/
|
|
264
|
-
function extractRawText(children: any): string {
|
|
265
|
-
if (children == null) return '';
|
|
266
|
-
if (typeof children === 'string') return children;
|
|
267
|
-
if (typeof children === 'number') return String(children);
|
|
268
|
-
|
|
269
|
-
if (Array.isArray(children)) {
|
|
270
|
-
return children
|
|
271
|
-
.map(child => extractRawText(child))
|
|
272
|
-
.filter(Boolean)
|
|
273
|
-
.join(' ');
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
// React element — try to extract text from its props
|
|
277
|
-
if (children && typeof children === 'object' && children.props) {
|
|
278
|
-
return extractRawText(children.props.children);
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
return '';
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
/**
|
|
285
|
-
* Recursively search a fiber subtree for icon/symbol components and
|
|
286
|
-
* return their `name` prop as a semantic label.
|
|
287
|
-
*
|
|
288
|
-
* Works generically: any non-RN-internal child component with a string
|
|
289
|
-
* `name` prop is treated as an icon (covers Ionicons, MaterialIcons,
|
|
290
|
-
* FontAwesome, custom SVG wrappers, etc. — no hardcoded list needed).
|
|
291
|
-
*
|
|
292
|
-
* e.g. a TouchableOpacity wrapping <Ionicons name="add-circle" /> → "icon:add-circle"
|
|
293
|
-
*/
|
|
294
|
-
function extractIconName(fiber: any, maxDepth: number = 5): string {
|
|
295
|
-
if (!fiber || maxDepth <= 0) return '';
|
|
296
|
-
|
|
297
|
-
let child = fiber.child;
|
|
298
|
-
while (child) {
|
|
299
|
-
const componentName = getComponentName(child);
|
|
300
|
-
const childProps = child.memoizedProps || {};
|
|
301
|
-
|
|
302
|
-
// Generic icon detection: non-RN-internal component with a string `name` prop
|
|
303
|
-
if (
|
|
304
|
-
componentName &&
|
|
305
|
-
!RN_INTERNAL_NAMES.has(componentName) &&
|
|
306
|
-
!PRESSABLE_TYPES.has(componentName) &&
|
|
307
|
-
!TEXT_INPUT_TYPES.has(componentName) &&
|
|
308
|
-
typeof childProps.name === 'string' &&
|
|
309
|
-
childProps.name.length > 0
|
|
310
|
-
) {
|
|
311
|
-
return `icon:${childProps.name}`;
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
// Recurse into ALL children (pierce through nested interactives)
|
|
315
|
-
const found = extractIconName(child, maxDepth - 1);
|
|
316
|
-
if (found) return found;
|
|
317
|
-
|
|
318
|
-
child = child.sibling;
|
|
319
|
-
}
|
|
320
|
-
return '';
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
/**
|
|
324
|
-
* Get the Fiber root node via __REACT_DEVTOOLS_GLOBAL_HOOK__.
|
|
325
|
-
*
|
|
326
|
-
* This hook is injected by React in development builds so React DevTools
|
|
327
|
-
* can inspect the component tree. It is NOT available in release/production.
|
|
328
|
-
* Used as a last-resort fallback when ref-based strategies fail.
|
|
329
|
-
*/
|
|
330
|
-
function getFiberRootFromDevTools(): any | null {
|
|
331
|
-
try {
|
|
332
|
-
const hook = (globalThis as any).__REACT_DEVTOOLS_GLOBAL_HOOK__;
|
|
333
|
-
if (hook) {
|
|
334
|
-
const renderers = hook.renderers;
|
|
335
|
-
if (renderers && renderers.size > 0) {
|
|
336
|
-
for (const [rendererId] of renderers) {
|
|
337
|
-
const roots = hook.getFiberRoots(rendererId);
|
|
338
|
-
if (roots && roots.size > 0) {
|
|
339
|
-
const fiberRoot = roots.values().next().value;
|
|
340
|
-
if (fiberRoot && fiberRoot.current) {
|
|
341
|
-
logger.debug('FiberTreeWalker', 'Accessed Fiber tree via DevTools hook');
|
|
342
|
-
return fiberRoot.current;
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
} catch (e) {
|
|
349
|
-
logger.debug('FiberTreeWalker', 'DevTools hook not available:', e);
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
return null;
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
/**
|
|
356
|
-
* Resolve a Fiber node from a React ref.
|
|
357
|
-
*
|
|
358
|
-
* Strategy order prioritizes ref-based access (works in BOTH dev and release)
|
|
359
|
-
* over the DevTools hook (dev-only):
|
|
360
|
-
*
|
|
361
|
-
* 1. __reactFiber$ keys — React attaches these to native host nodes in both
|
|
362
|
-
* dev and release. This is how the reconciler maps native nodes back to
|
|
363
|
-
* their Fiber. Most reliable for <View ref={...}> refs.
|
|
364
|
-
* 2. _reactInternals — available on class component instances.
|
|
365
|
-
* 3. _reactInternalInstance — legacy React (pre-Fiber) pattern.
|
|
366
|
-
* 4. Direct fiber properties — ref IS already a Fiber node (e.g. in tests).
|
|
367
|
-
* 5. __REACT_DEVTOOLS_GLOBAL_HOOK__ — dev-only last resort. Finds the root
|
|
368
|
-
* Fiber regardless of what ref is, but stripped in production builds.
|
|
369
|
-
*/
|
|
370
|
-
function getFiberFromRef(ref: any): any | null {
|
|
371
|
-
if (!ref) return null;
|
|
372
|
-
|
|
373
|
-
// Strategy 1: __reactFiber$ keys (React DOM/RN style)
|
|
374
|
-
// Works in both dev AND release builds — primary strategy.
|
|
375
|
-
try {
|
|
376
|
-
const keys = Object.keys(ref);
|
|
377
|
-
const fiberKey = keys.find(
|
|
378
|
-
key => key.startsWith('__reactFiber$') || key.startsWith('__reactInternalInstance$'),
|
|
379
|
-
);
|
|
380
|
-
if (fiberKey) {
|
|
381
|
-
logger.debug('FiberTreeWalker', 'Accessed Fiber tree via __reactFiber$ key');
|
|
382
|
-
return (ref as any)[fiberKey];
|
|
383
|
-
}
|
|
384
|
-
} catch {
|
|
385
|
-
// Object.keys may fail on some native nodes
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
// Strategy 2: _reactInternals (class components)
|
|
389
|
-
if (ref._reactInternals) return ref._reactInternals;
|
|
390
|
-
|
|
391
|
-
// Strategy 3: _reactInternalInstance (older React)
|
|
392
|
-
if (ref._reactInternalInstance) return ref._reactInternalInstance;
|
|
393
|
-
|
|
394
|
-
// Strategy 4: Direct fiber node properties (ref IS a fiber — used in tests)
|
|
395
|
-
if (ref.child || ref.memoizedProps) return ref;
|
|
396
|
-
|
|
397
|
-
// Strategy 5: DevTools hook (dev-only last resort)
|
|
398
|
-
const rootFiber = getFiberRootFromDevTools();
|
|
399
|
-
if (rootFiber) return rootFiber;
|
|
400
|
-
|
|
401
|
-
console.warn(
|
|
402
|
-
'[AIAgent] Could not access React Fiber tree. ' +
|
|
403
|
-
'The AI agent will not be able to detect interactive elements. ' +
|
|
404
|
-
'Ensure the rootRef is attached to a rendered <View>.'
|
|
405
|
-
);
|
|
406
|
-
logger.warn('FiberTreeWalker', 'All Fiber access strategies failed');
|
|
407
|
-
return null;
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
// ─── Blacklist/Whitelist Matching ──────────────────────────────
|
|
411
|
-
|
|
412
|
-
/**
|
|
413
|
-
* Check if a Fiber node matches any ref in the given list.
|
|
414
|
-
* Checks if an element is excluded from AI interaction
|
|
415
|
-
* We compare the Fiber's stateNode (native instance) against ref.current.
|
|
416
|
-
*/
|
|
417
|
-
function matchesRefList(node: any, refs?: React.RefObject<any>[]): boolean {
|
|
418
|
-
if (!refs || refs.length === 0) return false;
|
|
419
|
-
const stateNode = node.stateNode;
|
|
420
|
-
if (!stateNode) return false;
|
|
421
|
-
|
|
422
|
-
for (const ref of refs) {
|
|
423
|
-
if (ref.current && ref.current === stateNode) return true;
|
|
424
|
-
}
|
|
425
|
-
return false;
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
export interface WalkResult {
|
|
429
|
-
elementsText: string;
|
|
430
|
-
interactives: InteractiveElement[];
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
// ─── Note on overlays ──────────────────────────────────────────
|
|
434
|
-
// Toasts, modals, and dialogs are now captured naturally because
|
|
435
|
-
// walkFiberTree starts from the root Fiber and prunes only
|
|
436
|
-
// display:none nodes (inactive screens). No separate overlay scan needed.
|
|
437
|
-
|
|
438
|
-
// ─── Main Tree Walker ──────────────────────────────────────────
|
|
439
|
-
|
|
440
|
-
/**
|
|
441
|
-
* Walk the React Fiber tree from a root and collect all interactive elements
|
|
442
|
-
* as well as a hierarchical layout representation for the LLM.
|
|
443
|
-
*/
|
|
444
|
-
export function walkFiberTree(rootRef: any, config?: WalkConfig): WalkResult {
|
|
445
|
-
const fiber = getFiberFromRef(rootRef);
|
|
446
|
-
if (!fiber) {
|
|
447
|
-
logger.warn('FiberTreeWalker', 'Could not access Fiber tree from ref');
|
|
448
|
-
return { elementsText: '', interactives: [] };
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
// Always walk from the root Fiber — inactive screens are pruned inside
|
|
452
|
-
// processNode by checking for display:none, which React Navigation applies
|
|
453
|
-
// to all inactive screens. This ensures navigation chrome (tab bar, header)
|
|
454
|
-
// is always included without hardcoding any component names.
|
|
455
|
-
const startNode = fiber;
|
|
456
|
-
if (config?.screenName) {
|
|
457
|
-
logger.debug('FiberTreeWalker', `Walk active for screen "${config.screenName}" (inactive screens pruned via display:none)`);
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
// Overlay detection is superseded by root-level walk — all visible nodes
|
|
461
|
-
// (toasts, modals, overlays) are included naturally since they aren't hidden.
|
|
462
|
-
const overlayNodes: any[] = [];
|
|
463
|
-
|
|
464
|
-
const interactives: InteractiveElement[] = [];
|
|
465
|
-
let currentIndex = 0;
|
|
466
|
-
const hasWhitelist = config?.interactiveWhitelist && (config.interactiveWhitelist.length ?? 0) > 0;
|
|
467
|
-
|
|
468
|
-
function processNode(
|
|
469
|
-
node: any,
|
|
470
|
-
depth: number = 0,
|
|
471
|
-
isInsideInteractive: boolean = false,
|
|
472
|
-
ancestorOnPress: any = null,
|
|
473
|
-
currentZoneId: string | undefined = undefined
|
|
474
|
-
): string {
|
|
475
|
-
if (!node) return '';
|
|
476
|
-
|
|
477
|
-
const props = node.memoizedProps || {};
|
|
478
|
-
|
|
479
|
-
// ── Prune inactive screens ──────────────────────────────────
|
|
480
|
-
// Two mechanisms cover all React Navigation setups:
|
|
481
|
-
//
|
|
482
|
-
// 1. react-native-screens (Expo default, recommended):
|
|
483
|
-
// React Navigation sets activityState=0 on inactive tab screens.
|
|
484
|
-
// Stack screens retain activityState=2 even in the background (NativeStack
|
|
485
|
-
// never decreases activityState — confirmed from react-native-screens source).
|
|
486
|
-
// Those are kept as useful navigation history context for the agent.
|
|
487
|
-
//
|
|
488
|
-
// 2. ResourceSavingScene (React Navigation fallback without react-native-screens):
|
|
489
|
-
// Wraps inactive screen content with pointerEvents="none". Prune any such wrapper.
|
|
490
|
-
//
|
|
491
|
-
// Both are prop-based — no component names, no CSS display property.
|
|
492
|
-
if (props.activityState === 0) return '';
|
|
493
|
-
if (props.pointerEvents === 'none') return '';
|
|
494
|
-
|
|
495
|
-
// ── Security Constraints ──
|
|
496
|
-
if (props.aiIgnore === true) return '';
|
|
497
|
-
if (matchesRefList(node, config?.interactiveBlacklist)) {
|
|
498
|
-
let childText = '';
|
|
499
|
-
let currentChild = node.child;
|
|
500
|
-
while (currentChild) {
|
|
501
|
-
childText += processNode(currentChild, depth, isInsideInteractive);
|
|
502
|
-
currentChild = currentChild.sibling;
|
|
503
|
-
}
|
|
504
|
-
return childText;
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
// Interactive check — nested interactives with a DIFFERENT onPress than
|
|
508
|
-
// their ancestor are separate actions (e.g. "+" button inside a dish card).
|
|
509
|
-
// Only suppress true wrapper duplicates (same onPress reference).
|
|
510
|
-
const isWhitelisted = matchesRefList(node, config?.interactiveWhitelist);
|
|
511
|
-
const elementType = getElementType(node);
|
|
512
|
-
let shouldInclude = false;
|
|
513
|
-
if (hasWhitelist) {
|
|
514
|
-
shouldInclude = isWhitelisted;
|
|
515
|
-
} else if (elementType && !isDisabled(node)) {
|
|
516
|
-
if (!isInsideInteractive) {
|
|
517
|
-
shouldInclude = true;
|
|
518
|
-
} else {
|
|
519
|
-
// Inside an ancestor interactive — only include if onPress is DIFFERENT
|
|
520
|
-
const ownOnPress = props.onPress;
|
|
521
|
-
shouldInclude = !!ownOnPress && ownOnPress !== ancestorOnPress;
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
// Track the onPress for descendant dedup
|
|
526
|
-
const nextAncestorOnPress = shouldInclude ? (props.onPress || ancestorOnPress) : ancestorOnPress;
|
|
527
|
-
|
|
528
|
-
// Determine if this node produces visible output (affects depth for children)
|
|
529
|
-
// Only visible nodes (interactives, text, images, videos) should increment
|
|
530
|
-
// depth. Structural View wrappers are transparent — they pass through the
|
|
531
|
-
// same depth so indentation stays flat and doesn't waste LLM tokens.
|
|
532
|
-
const typeStr = node.type && typeof node.type === 'string' ? node.type :
|
|
533
|
-
(node.elementType && typeof node.elementType === 'string' ? node.elementType : null);
|
|
534
|
-
const componentName = getComponentName(node);
|
|
535
|
-
const isTextNode = typeStr === 'RCTText' || typeStr === 'Text';
|
|
536
|
-
const isImageNode = !!(componentName && IMAGE_TYPES.has(componentName));
|
|
537
|
-
const isVideoNode = !!(componentName && VIDEO_TYPES.has(componentName));
|
|
538
|
-
const producesOutput = shouldInclude || isTextNode || isImageNode || isVideoNode;
|
|
539
|
-
const childDepth = producesOutput ? depth + 1 : depth;
|
|
540
|
-
|
|
541
|
-
const nextZoneId = componentName === 'AIZone' && props.id ? props.id : currentZoneId;
|
|
542
|
-
|
|
543
|
-
const indent = ' '.repeat(depth);
|
|
544
|
-
|
|
545
|
-
// ── Zone Header Emission ──────────────────────────────────
|
|
546
|
-
// When entering an AIZone boundary, emit a section header so the agent
|
|
547
|
-
// knows this zone exists, what its id is, and what it's allowed to do.
|
|
548
|
-
// This is what lets the agent call simplify_zone / guide_user proactively.
|
|
549
|
-
let zoneHeader = '';
|
|
550
|
-
if (componentName === 'AIZone' && props.id) {
|
|
551
|
-
const permissions: string[] = [];
|
|
552
|
-
if (props.allowSimplify) permissions.push('simplify');
|
|
553
|
-
if (props.allowHighlight) permissions.push('highlight');
|
|
554
|
-
if (props.allowInjectHint) permissions.push('hint');
|
|
555
|
-
if (props.allowInjectCard) permissions.push('card');
|
|
556
|
-
const permStr = permissions.length ? ` | permissions: ${permissions.join(', ')}` : '';
|
|
557
|
-
zoneHeader = `${indent}[zone: ${props.id}${permStr}]\n`;
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
// Process children
|
|
561
|
-
let childrenText = '';
|
|
562
|
-
let currentChild = node.child;
|
|
563
|
-
while (currentChild) {
|
|
564
|
-
childrenText += processNode(
|
|
565
|
-
currentChild,
|
|
566
|
-
childDepth,
|
|
567
|
-
isInsideInteractive || !!shouldInclude,
|
|
568
|
-
nextAncestorOnPress,
|
|
569
|
-
nextZoneId,
|
|
570
|
-
);
|
|
571
|
-
currentChild = currentChild.sibling;
|
|
572
|
-
}
|
|
573
|
-
|
|
574
|
-
// Prepend zone header before children if this is a zone root
|
|
575
|
-
if (zoneHeader) {
|
|
576
|
-
childrenText = zoneHeader + childrenText;
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
if (shouldInclude) {
|
|
580
|
-
const resolvedType = elementType || 'pressable';
|
|
581
|
-
// Primary: accessibilityLabel → deep text (pierces nested interactives)
|
|
582
|
-
let label = props.accessibilityLabel || extractDeepTextContent(node);
|
|
583
|
-
|
|
584
|
-
// Fallback: TextInput placeholder
|
|
585
|
-
if (!label && resolvedType === 'text-input' && props.placeholder) {
|
|
586
|
-
label = props.placeholder;
|
|
587
|
-
}
|
|
588
|
-
// Fallback: Icon/symbol name (any component with a `name` prop)
|
|
589
|
-
if (!label) {
|
|
590
|
-
label = extractIconName(node);
|
|
591
|
-
}
|
|
592
|
-
// Fallback: testID/nativeID
|
|
593
|
-
if (!label && (props.testID || props.nativeID)) {
|
|
594
|
-
label = props.testID || props.nativeID;
|
|
595
|
-
}
|
|
596
|
-
// Fallback: Parent component context (skip internal RN context names)
|
|
597
|
-
if (!label) {
|
|
598
|
-
const parentContext = getNearestCustomComponentName(node);
|
|
599
|
-
if (parentContext) {
|
|
600
|
-
// Skip React Native internal implementation names that leak as labels.
|
|
601
|
-
// These are never meaningful to the AI or developer.
|
|
602
|
-
const RN_INTERNAL_NAMES = new Set([
|
|
603
|
-
'ScrollViewContext', 'VirtualizedListContext', 'ViewabilityHelper',
|
|
604
|
-
'ScrollResponder', 'AnimatedComponent', 'TouchableOpacity',
|
|
605
|
-
]);
|
|
606
|
-
if (!RN_INTERNAL_NAMES.has(parentContext)) {
|
|
607
|
-
label = parentContext;
|
|
608
|
-
}
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
interactives.push({
|
|
613
|
-
index: currentIndex,
|
|
614
|
-
type: resolvedType,
|
|
615
|
-
label: label || `[${resolvedType}]`,
|
|
616
|
-
aiPriority: props.aiPriority,
|
|
617
|
-
zoneId: currentZoneId,
|
|
618
|
-
fiberNode: node,
|
|
619
|
-
props: { ...props },
|
|
620
|
-
requiresConfirmation: props.aiConfirm === true,
|
|
621
|
-
});
|
|
622
|
-
|
|
623
|
-
// Build output tag with state attributes
|
|
624
|
-
const stateAttrs = extractStateAttributes(props);
|
|
625
|
-
let attrStr = stateAttrs ? ` ${stateAttrs}` : '';
|
|
626
|
-
if (props.aiPriority) {
|
|
627
|
-
attrStr += ` aiPriority="${props.aiPriority}"`;
|
|
628
|
-
if (currentZoneId) attrStr += ` zoneId="${currentZoneId}"`;
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
if (props.aiConfirm === true) {
|
|
632
|
-
attrStr += ' aiConfirm';
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
const textContent = label || '';
|
|
636
|
-
const elementOutput = `${indent}[${currentIndex}]<${resolvedType}${attrStr}>${textContent} />${childrenText.trim() ? '\n' + childrenText : ''}\n`;
|
|
637
|
-
currentIndex++;
|
|
638
|
-
return elementOutput;
|
|
639
|
-
}
|
|
640
|
-
|
|
641
|
-
// Non-interactive structural nodes — collapse view chains to reduce noise
|
|
642
|
-
// Only emit text content; structural <view> wrappers are transparent
|
|
643
|
-
if (isTextNode) {
|
|
644
|
-
const textContent = extractRawText(props.children);
|
|
645
|
-
if (textContent && textContent.trim() !== '') {
|
|
646
|
-
const priorityTag = props.aiPriority ? ` (aiPriority="${props.aiPriority}"${currentZoneId ? ` zoneId="${currentZoneId}"` : ''})` : '';
|
|
647
|
-
return `${indent}${textContent.trim()}${priorityTag}\n`;
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
// ── Media Detection: Component-Context Media Inference ──
|
|
652
|
-
if (isImageNode) {
|
|
653
|
-
const context = getNearestCustomComponentName(node);
|
|
654
|
-
const alt = props.alt || props.accessibilityLabel || '';
|
|
655
|
-
// Emit the full URI so Gemini can use vision to analyze the image
|
|
656
|
-
const src = typeof props.source === 'object' && props.source?.uri
|
|
657
|
-
? props.source.uri
|
|
658
|
-
: '';
|
|
659
|
-
const attrs = [
|
|
660
|
-
context ? `in="${context}"` : '',
|
|
661
|
-
alt ? `alt="${alt}"` : '',
|
|
662
|
-
src ? `src="${src}"` : '',
|
|
663
|
-
].filter(Boolean).join(' ');
|
|
664
|
-
return `${indent}[image${attrs ? ' ' + attrs : ''}]\n`;
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
if (isVideoNode) {
|
|
668
|
-
const context = getNearestCustomComponentName(node);
|
|
669
|
-
const paused = props.paused !== undefined ? props.paused : props.shouldPlay !== undefined ? !props.shouldPlay : null;
|
|
670
|
-
// Capture video source URI and poster image
|
|
671
|
-
const src = typeof props.source === 'object' && props.source?.uri
|
|
672
|
-
? props.source.uri
|
|
673
|
-
: '';
|
|
674
|
-
const poster = props.posterSource?.uri || props.poster || '';
|
|
675
|
-
const attrs = [
|
|
676
|
-
context ? `in="${context}"` : '',
|
|
677
|
-
paused !== null ? `state="${paused ? 'paused' : 'playing'}"` : '',
|
|
678
|
-
src ? `src="${src}"` : '',
|
|
679
|
-
poster ? `poster="${poster}"` : '',
|
|
680
|
-
].filter(Boolean).join(' ');
|
|
681
|
-
return `${indent}[video${attrs ? ' ' + attrs : ''}]\n`;
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
// Structural views: pass children through without adding <view> wrapper
|
|
685
|
-
// This collapses the 50+ nesting levels into flat, readable output
|
|
686
|
-
return childrenText;
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
let elementsText = processNode(startNode, 0);
|
|
690
|
-
|
|
691
|
-
// Second pass: walk overlay nodes (toasts, dialogs, modals)
|
|
692
|
-
// These render outside the screen subtree — detected by findOverlayNodes()
|
|
693
|
-
if (overlayNodes.length > 0) {
|
|
694
|
-
let overlayText = '';
|
|
695
|
-
for (const overlay of overlayNodes) {
|
|
696
|
-
const name = getComponentName(overlay);
|
|
697
|
-
overlayText += `\n--- ${name || 'Overlay'} ---\n`;
|
|
698
|
-
overlayText += processNode(overlay, 0);
|
|
699
|
-
}
|
|
700
|
-
if (overlayText.trim()) {
|
|
701
|
-
elementsText += '\n' + overlayText;
|
|
702
|
-
logger.info('FiberTreeWalker', `Included ${overlayNodes.length} overlay(s) in screen context`);
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
// Clean up excessive blank lines
|
|
707
|
-
elementsText = elementsText.replace(/\n{3,}/g, '\n\n');
|
|
708
|
-
|
|
709
|
-
logger.info('FiberTreeWalker', `Found ${interactives.length} interactive elements`);
|
|
710
|
-
return { elementsText: elementsText.trim(), interactives };
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
// ─── Scrollable Container Detection ────────────────────────────
|
|
714
|
-
|
|
715
|
-
/** React Native component names that are scrollable containers */
|
|
716
|
-
const SCROLLABLE_TYPES = new Set([
|
|
717
|
-
'ScrollView', 'RCTScrollView',
|
|
718
|
-
'FlatList', 'SectionList', 'VirtualizedList',
|
|
719
|
-
]);
|
|
720
|
-
|
|
721
|
-
/**
|
|
722
|
-
* React Native component names that are pager/tab containers.
|
|
723
|
-
* These look scrollable but use page-based navigation internally.
|
|
724
|
-
* Scrolling them directly causes native assertion crashes (e.g. setPage(NSNull)).
|
|
725
|
-
* Pattern: Detox's ScrollToIndexAction.getConstraints() rejects these entirely.
|
|
726
|
-
*/
|
|
727
|
-
const PAGER_TYPES = new Set([
|
|
728
|
-
'RNCViewPager', 'PagerView', 'ViewPager',
|
|
729
|
-
'RNViewPager', 'TabView', 'MaterialTabView',
|
|
730
|
-
'ScrollableTabView', 'RNTabView',
|
|
731
|
-
]);
|
|
732
|
-
|
|
733
|
-
export interface ScrollableContainer {
|
|
734
|
-
/** Index for identification when multiple scrollables exist */
|
|
735
|
-
index: number;
|
|
736
|
-
/** Component name (e.g., 'FlatList', 'ScrollView') */
|
|
737
|
-
componentName: string;
|
|
738
|
-
/** Contextual label — nearest custom component name or text header */
|
|
739
|
-
label: string;
|
|
740
|
-
/** The Fiber node */
|
|
741
|
-
fiberNode: any;
|
|
742
|
-
/** The native stateNode (has scrollToOffset, scrollToEnd, etc.) */
|
|
743
|
-
stateNode: any;
|
|
744
|
-
/**
|
|
745
|
-
* True if this container is a PagerView/TabView.
|
|
746
|
-
* These must NOT be scrolled — use tap on tab labels instead.
|
|
747
|
-
* Pattern from Detox: ScrollToIndexAction rejects non-ScrollView types.
|
|
748
|
-
*/
|
|
749
|
-
isPagerLike: boolean;
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
/**
|
|
753
|
-
* Find the fiber node whose component name matches the given screen name.
|
|
754
|
-
* Matches by checking if the component name starts with or equals the screen name.
|
|
755
|
-
* e.g., screenName "Menu" matches component "MenuScreen" or "Menu".
|
|
756
|
-
*
|
|
757
|
-
* Returns the first (deepest active) match found via depth-first search.
|
|
758
|
-
*/
|
|
759
|
-
function findScreenFiberNode(rootFiber: any, screenName: string): any | null {
|
|
760
|
-
if (!rootFiber || !screenName) return null;
|
|
761
|
-
|
|
762
|
-
const lowerScreen = screenName.toLowerCase();
|
|
763
|
-
|
|
764
|
-
function search(node: any): any | null {
|
|
765
|
-
if (!node) return null;
|
|
766
|
-
|
|
767
|
-
const name = getComponentName(node);
|
|
768
|
-
if (name) {
|
|
769
|
-
const lowerName = name.toLowerCase();
|
|
770
|
-
// Match: "MenuScreen" starts with "menu", or "Menu" equals "menu"
|
|
771
|
-
if (lowerName.startsWith(lowerScreen) || lowerScreen.startsWith(lowerName)) {
|
|
772
|
-
return node;
|
|
773
|
-
}
|
|
774
|
-
}
|
|
775
|
-
|
|
776
|
-
// Depth-first: search children first, then siblings
|
|
777
|
-
let child = node.child;
|
|
778
|
-
while (child) {
|
|
779
|
-
const found = search(child);
|
|
780
|
-
if (found) return found;
|
|
781
|
-
child = child.sibling;
|
|
782
|
-
}
|
|
783
|
-
|
|
784
|
-
return null;
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
return search(rootFiber);
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
/**
|
|
791
|
-
* Walk the Fiber tree to discover scrollable containers.
|
|
792
|
-
* Returns native stateNodes that expose scrollToOffset(), scrollToEnd(), scrollTo().
|
|
793
|
-
*
|
|
794
|
-
* When `screenName` is provided, the search is scoped to the matching screen's
|
|
795
|
-
* subtree — this prevents finding containers from other mounted screens
|
|
796
|
-
* (React Navigation keeps all stack screens in the tree).
|
|
797
|
-
*
|
|
798
|
-
* For FlatList: the Fiber's stateNode is a VirtualizedList instance.
|
|
799
|
-
* Its underlying scroll view can be accessed via getNativeScrollRef() or
|
|
800
|
-
* getScrollRef(), which returns the native ScrollView with scrollTo/scrollToEnd.
|
|
801
|
-
*
|
|
802
|
-
* For ScrollView: the stateNode IS the native scroll view directly.
|
|
803
|
-
*/
|
|
804
|
-
export function findScrollableContainers(rootRef: any, screenName?: string): ScrollableContainer[] {
|
|
805
|
-
const fiber = getFiberFromRef(rootRef);
|
|
806
|
-
if (!fiber) {
|
|
807
|
-
logger.warn('FiberTreeWalker', 'Could not access Fiber tree for scroll detection');
|
|
808
|
-
return [];
|
|
809
|
-
}
|
|
810
|
-
|
|
811
|
-
// Scope to the active screen's subtree when screenName is provided
|
|
812
|
-
let startNode = fiber;
|
|
813
|
-
if (screenName) {
|
|
814
|
-
const screenFiber = findScreenFiberNode(fiber, screenName);
|
|
815
|
-
if (screenFiber) {
|
|
816
|
-
startNode = screenFiber;
|
|
817
|
-
logger.debug('FiberTreeWalker', `Scroll scoped to screen "${screenName}" (component: ${getComponentName(screenFiber)})`);
|
|
818
|
-
} else {
|
|
819
|
-
logger.debug('FiberTreeWalker', `Screen "${screenName}" not found in Fiber tree — searching entire tree`);
|
|
820
|
-
}
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
const containers: ScrollableContainer[] = [];
|
|
824
|
-
let currentIndex = 0;
|
|
825
|
-
|
|
826
|
-
function walk(node: any): void {
|
|
827
|
-
if (!node) return;
|
|
828
|
-
|
|
829
|
-
const name = getComponentName(node);
|
|
830
|
-
const isScrollable = name && SCROLLABLE_TYPES.has(name);
|
|
831
|
-
const isPagerLike = name ? PAGER_TYPES.has(name) : false;
|
|
832
|
-
|
|
833
|
-
if (isScrollable || isPagerLike) {
|
|
834
|
-
// Get context: nearest custom parent component name
|
|
835
|
-
const contextLabel = getNearestCustomComponentName(node) || name || 'Unknown';
|
|
836
|
-
|
|
837
|
-
// For scrollable containers, we need the native scroll ref.
|
|
838
|
-
// FlatList Fiber stateNode may be the component instance —
|
|
839
|
-
// we need to find the underlying native ScrollView.
|
|
840
|
-
let scrollRef = isPagerLike ? node.stateNode : resolveNativeScrollRef(node);
|
|
841
|
-
|
|
842
|
-
if (scrollRef) {
|
|
843
|
-
containers.push({
|
|
844
|
-
index: currentIndex++,
|
|
845
|
-
componentName: name || 'Unknown',
|
|
846
|
-
label: contextLabel,
|
|
847
|
-
fiberNode: node,
|
|
848
|
-
stateNode: scrollRef,
|
|
849
|
-
isPagerLike,
|
|
850
|
-
});
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
|
|
854
|
-
// Recurse into children and siblings
|
|
855
|
-
let child = node.child;
|
|
856
|
-
while (child) {
|
|
857
|
-
walk(child);
|
|
858
|
-
child = child.sibling;
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
walk(startNode);
|
|
863
|
-
logger.info('FiberTreeWalker', `Found ${containers.length} scrollable container(s)${screenName ? ` for screen "${screenName}"` : ''}`);
|
|
864
|
-
return containers;
|
|
865
|
-
}
|
|
866
|
-
|
|
867
|
-
/**
|
|
868
|
-
* Resolve the native scroll view reference from a Fiber node.
|
|
869
|
-
*
|
|
870
|
-
* Handles multiple React Native internals:
|
|
871
|
-
* - RCTScrollView: stateNode IS the native scroll view
|
|
872
|
-
* - FlatList/VirtualizedList: stateNode is a component instance,
|
|
873
|
-
* need to find the inner ScrollView via getNativeScrollRef() or
|
|
874
|
-
* by walking down the Fiber tree to find the RCTScrollView child
|
|
875
|
-
*/
|
|
876
|
-
function resolveNativeScrollRef(fiberNode: any): any {
|
|
877
|
-
const stateNode = fiberNode.stateNode;
|
|
878
|
-
|
|
879
|
-
// Case 1: stateNode has scrollTo (native ScrollView or RCTScrollView)
|
|
880
|
-
if (stateNode && typeof stateNode.scrollTo === 'function') {
|
|
881
|
-
return stateNode;
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
// Case 2: stateNode has getNativeScrollRef (FlatList / VirtualizedList)
|
|
885
|
-
if (stateNode && typeof stateNode.getNativeScrollRef === 'function') {
|
|
886
|
-
try {
|
|
887
|
-
const ref = stateNode.getNativeScrollRef();
|
|
888
|
-
if (ref && typeof ref.scrollTo === 'function') return ref;
|
|
889
|
-
} catch { /* fall through */ }
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
// Case 3: stateNode has getScrollRef (another VirtualizedList pattern)
|
|
893
|
-
if (stateNode && typeof stateNode.getScrollRef === 'function') {
|
|
894
|
-
try {
|
|
895
|
-
const ref = stateNode.getScrollRef();
|
|
896
|
-
if (ref && typeof ref.scrollTo === 'function') return ref;
|
|
897
|
-
// getScrollRef might return another wrapper — try getNativeScrollRef on it
|
|
898
|
-
if (ref && typeof ref.getNativeScrollRef === 'function') {
|
|
899
|
-
const nativeRef = ref.getNativeScrollRef();
|
|
900
|
-
if (nativeRef && typeof nativeRef.scrollTo === 'function') return nativeRef;
|
|
901
|
-
}
|
|
902
|
-
} catch { /* fall through */ }
|
|
903
|
-
}
|
|
904
|
-
|
|
905
|
-
// Case 4: stateNode has scrollToOffset directly (VirtualizedList instance)
|
|
906
|
-
if (stateNode && typeof stateNode.scrollToOffset === 'function') {
|
|
907
|
-
return stateNode;
|
|
908
|
-
}
|
|
909
|
-
|
|
910
|
-
// Case 5: Walk down Fiber tree to find an RCTScrollView child
|
|
911
|
-
let child = fiberNode.child;
|
|
912
|
-
while (child) {
|
|
913
|
-
const childName = getComponentName(child);
|
|
914
|
-
if (childName === 'RCTScrollView' && child.stateNode) {
|
|
915
|
-
return child.stateNode;
|
|
916
|
-
}
|
|
917
|
-
// Go one level deeper for wrapper patterns
|
|
918
|
-
if (child.child) {
|
|
919
|
-
const grandchildName = getComponentName(child.child);
|
|
920
|
-
if (grandchildName === 'RCTScrollView' && child.child.stateNode) {
|
|
921
|
-
return child.child.stateNode;
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
child = child.sibling;
|
|
925
|
-
}
|
|
926
|
-
|
|
927
|
-
logger.debug('FiberTreeWalker', 'Could not resolve native scroll ref — returning stateNode as fallback');
|
|
928
|
-
return stateNode;
|
|
929
|
-
}
|
|
930
|
-
|