execbro 1.10.12 → 1.10.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/__tests__/unit/screenState.test.js +204 -6
- package/build/__tests__/unit/screenState.test.js.map +1 -1
- package/build/__tests__/unit/tap.test.js +0 -2
- package/build/__tests__/unit/tap.test.js.map +1 -1
- package/build/core/screenState.d.ts +78 -10
- package/build/core/screenState.d.ts.map +1 -1
- package/build/core/screenState.js +269 -38
- package/build/core/screenState.js.map +1 -1
- package/build/pro/tap.d.ts +0 -4
- package/build/pro/tap.d.ts.map +1 -1
- package/build/pro/tap.js +0 -98
- package/build/pro/tap.js.map +1 -1
- package/build/tools/componentTools.d.ts.map +1 -1
- package/build/tools/componentTools.js +11 -10
- package/build/tools/componentTools.js.map +1 -1
- package/package.json +1 -1
|
@@ -25,10 +25,41 @@ export interface ScreenStatePressable {
|
|
|
25
25
|
/** What pressing triggers: "onPress=handleSubmit()", "onBack=goBack()", or "onPress→onBack" (prop route when the fn is anonymous). Absent when nothing meaningful survives Hermes. */
|
|
26
26
|
onPressHint?: string | null;
|
|
27
27
|
}
|
|
28
|
+
export interface ScreenStateText {
|
|
29
|
+
text: string;
|
|
30
|
+
center: {
|
|
31
|
+
x: number;
|
|
32
|
+
y: number;
|
|
33
|
+
};
|
|
34
|
+
bounds: {
|
|
35
|
+
x: number;
|
|
36
|
+
y: number;
|
|
37
|
+
width: number;
|
|
38
|
+
height: number;
|
|
39
|
+
};
|
|
40
|
+
blockedByOverlay?: boolean;
|
|
41
|
+
}
|
|
42
|
+
export interface ScreenStateImage {
|
|
43
|
+
src?: string | null;
|
|
44
|
+
alt?: string | null;
|
|
45
|
+
center: {
|
|
46
|
+
x: number;
|
|
47
|
+
y: number;
|
|
48
|
+
};
|
|
49
|
+
bounds: {
|
|
50
|
+
x: number;
|
|
51
|
+
y: number;
|
|
52
|
+
width: number;
|
|
53
|
+
height: number;
|
|
54
|
+
};
|
|
55
|
+
blockedByOverlay?: boolean;
|
|
56
|
+
}
|
|
28
57
|
export interface ScreenStateOverlay {
|
|
29
58
|
type: "BottomSheet" | "Modal" | "Alert" | "ActionSheet" | "Unknown";
|
|
30
59
|
title: string | null;
|
|
31
60
|
pressables: ScreenStatePressable[];
|
|
61
|
+
texts?: ScreenStateText[];
|
|
62
|
+
images?: ScreenStateImage[];
|
|
32
63
|
}
|
|
33
64
|
export interface ScreenStateRoute {
|
|
34
65
|
name: string;
|
|
@@ -39,6 +70,8 @@ export interface ScreenState {
|
|
|
39
70
|
route: ScreenStateRoute | null;
|
|
40
71
|
overlays: ScreenStateOverlay[];
|
|
41
72
|
pressables: ScreenStatePressable[];
|
|
73
|
+
texts: ScreenStateText[];
|
|
74
|
+
images: ScreenStateImage[];
|
|
42
75
|
}
|
|
43
76
|
export declare function markPressablesCoveredByOverlay(pressables: ScreenStatePressable[], overlayBounds: {
|
|
44
77
|
x: number;
|
|
@@ -51,14 +84,30 @@ export declare function markPressablesCoveredByOverlay(pressables: ScreenStatePr
|
|
|
51
84
|
* pressable's icon child name carries recognizable semantics:
|
|
52
85
|
* { label: "[FloatingHeader]", icon: "SvgChevronBackward" }
|
|
53
86
|
* → label "[SvgChevronBackward — possibly back button]"
|
|
87
|
+
* An icon button whose only text is a count badge (cart with "1") keeps that
|
|
88
|
+
* count as nearby context while the label is upgraded to the icon's meaning:
|
|
89
|
+
* { label: "1", icon: "SvgCartNew" } → label "[SvgCartNew — possibly cart button]", nearbyText "1"
|
|
54
90
|
* Labels from text/a11y are never touched (icon is null for those).
|
|
55
91
|
*/
|
|
56
92
|
export declare function applyIconHintToLabel(p: ScreenStatePressable): ScreenStatePressable;
|
|
57
93
|
/**
|
|
58
|
-
*
|
|
59
|
-
* screenshot
|
|
94
|
+
* Coordinate converter for any positioned item (pressable, text, image) — lets
|
|
95
|
+
* screenshot tools map points/dp into screenshot pixels (including conditional
|
|
96
|
+
* shifts like the iOS safe-area band). Reads only center/bounds, so it applies
|
|
97
|
+
* uniformly across item types.
|
|
60
98
|
*/
|
|
61
|
-
export type
|
|
99
|
+
export type ItemCoordConverter = (item: {
|
|
100
|
+
center: {
|
|
101
|
+
x: number;
|
|
102
|
+
y: number;
|
|
103
|
+
};
|
|
104
|
+
bounds: {
|
|
105
|
+
x: number;
|
|
106
|
+
y: number;
|
|
107
|
+
width: number;
|
|
108
|
+
height: number;
|
|
109
|
+
};
|
|
110
|
+
}) => {
|
|
62
111
|
center: {
|
|
63
112
|
x: number;
|
|
64
113
|
y: number;
|
|
@@ -70,21 +119,40 @@ export type PressableCoordConverter = (p: ScreenStatePressable) => {
|
|
|
70
119
|
height: number;
|
|
71
120
|
};
|
|
72
121
|
};
|
|
122
|
+
/** Alias kept for existing call sites; prefer ItemCoordConverter. */
|
|
123
|
+
export type PressableCoordConverter = ItemCoordConverter;
|
|
124
|
+
/** One merged-list line for a static text node — coordinates match pressable lines. */
|
|
125
|
+
export declare function formatTextEntry(t: ScreenStateText, convert?: ItemCoordConverter, opts?: {
|
|
126
|
+
fullText?: boolean;
|
|
127
|
+
}): string;
|
|
128
|
+
/** One merged-list line for an image node — coordinates match pressable lines. */
|
|
129
|
+
export declare function formatImageEntry(img: ScreenStateImage, convert?: ItemCoordConverter): string;
|
|
73
130
|
/**
|
|
74
|
-
* Render a ScreenState as the
|
|
75
|
-
*
|
|
76
|
-
*
|
|
131
|
+
* Render a ScreenState as the orientation summary used by get_screen_state and the
|
|
132
|
+
* screenshot tools. By default it merges pressables, static text (📝), and images
|
|
133
|
+
* (🖼) into one spatially-ordered list per reachability group — enough to read and
|
|
134
|
+
* navigate the screen without a screenshot:
|
|
135
|
+
* 📍 Detail [Detail]
|
|
77
136
|
* 🎯 Pressables:
|
|
78
|
-
* (
|
|
137
|
+
* (210, 175) 🖼 Image 420x350 src="…"
|
|
138
|
+
* (146, 394) 📝 "Valya product" frame:(20,382 251x24)
|
|
139
|
+
* (210, 838) <Button /> "In cart" frame:(20,810 380x56)
|
|
140
|
+
* pressablesOnly restores the lean pressable-only snapshot; fullText disables the
|
|
141
|
+
* 80-char text truncation.
|
|
79
142
|
*/
|
|
80
|
-
export declare function formatScreenStateSummary(ss: ScreenState, convert?:
|
|
143
|
+
export declare function formatScreenStateSummary(ss: ScreenState, convert?: ItemCoordConverter, opts?: {
|
|
144
|
+
pressablesOnly?: boolean;
|
|
145
|
+
fullText?: boolean;
|
|
146
|
+
}): string;
|
|
81
147
|
export declare function describePressHandler(raw: unknown): string | null;
|
|
82
148
|
/**
|
|
83
149
|
* Fallback handler context from the custom component's on* props, for when the
|
|
84
150
|
* direct onPress is anonymous (Hermes discards source even in dev bundles, and
|
|
85
151
|
* names like navigation.goBack are lost to computed assignment):
|
|
86
|
-
* -
|
|
87
|
-
*
|
|
152
|
+
* - only trust the prop whose value IS the touchable's onPress (pass-through);
|
|
153
|
+
* a non-matched candidate is an unverifiable guess that mislabels every button
|
|
154
|
+
* in a multi-button container (FloatingHeader exposes only onBack, but its menu
|
|
155
|
+
* and cart buttons run internal handlers) — so it yields null
|
|
88
156
|
* - named fn → "onBack=goBack()"; nameless → "onPress→onBack" (the prop name
|
|
89
157
|
* alone is greppable context); a bare nameless "onPress" prop adds nothing → null
|
|
90
158
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"screenState.d.ts","sourceRoot":"","sources":["../../src/core/screenState.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAQlD,MAAM,WAAW,oBAAoB;IACjC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,mGAAmG;IACnG,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,iGAAiG;IACjG,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,yFAAyF;IACzF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kGAAkG;IAClG,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,sFAAsF;IACtF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,sLAAsL;IACtL,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,kBAAkB;IAC/B,IAAI,EAAE,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,aAAa,GAAG,SAAS,CAAC;IACpE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,UAAU,EAAE,oBAAoB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"screenState.d.ts","sourceRoot":"","sources":["../../src/core/screenState.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAQlD,MAAM,WAAW,oBAAoB;IACjC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,mGAAmG;IACnG,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,iGAAiG;IACjG,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,yFAAyF;IACzF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kGAAkG;IAClG,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,sFAAsF;IACtF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,sLAAsL;IACtL,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC7B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,IAAI,EAAE,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,aAAa,GAAG,SAAS,CAAC;IACpE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,UAAU,EAAE,oBAAoB,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACvC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IACxB,KAAK,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC/B,QAAQ,EAAE,kBAAkB,EAAE,CAAC;IAC/B,UAAU,EAAE,oBAAoB,EAAE,CAAC;IACnC,KAAK,EAAE,eAAe,EAAE,CAAC;IACzB,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC9B;AAMD,wBAAgB,8BAA8B,CAC1C,UAAU,EAAE,oBAAoB,EAAE,EAClC,aAAa,EAAE;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACvE,oBAAoB,EAAE,CAWxB;AAKD;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,oBAAoB,GAAG,oBAAoB,CAYlF;AAED;;;;;GAKG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,IAAI,EAAE;IACpC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CACnE,KAAK;IACF,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,KAAK,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAClE,CAAC;AACF,qEAAqE;AACrE,MAAM,MAAM,uBAAuB,GAAG,kBAAkB,CAAC;AAWzD,uFAAuF;AACvF,wBAAgB,eAAe,CAC3B,CAAC,EAAE,eAAe,EAClB,OAAO,GAAE,kBAAmC,EAC5C,IAAI,GAAE;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAO,GAClC,MAAM,CAIR;AAED,kFAAkF;AAClF,wBAAgB,gBAAgB,CAC5B,GAAG,EAAE,gBAAgB,EACrB,OAAO,GAAE,kBAAmC,GAC7C,MAAM,CAKR;AAKD;;;;;;;;;;;;GAYG;AACH,wBAAgB,wBAAwB,CACpC,EAAE,EAAE,WAAW,EACf,OAAO,GAAE,kBAAmC,EAC5C,IAAI,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAO,GAC5D,MAAM,CAmFR;AAoBD,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAQhE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAWhE;AAED,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,OAAO,GAAG,WAAW,GAAG,IAAI,CAWzE;AAMD,wBAAsB,cAAc,CAChC,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,OAAO,CAAC,eAAe,GAAG;IAAE,WAAW,CAAC,EAAE,WAAW,CAAA;CAAE,CAAC,CA6lC1D"}
|
|
@@ -15,32 +15,68 @@ export function markPressablesCoveredByOverlay(pressables, overlayBounds) {
|
|
|
15
15
|
}
|
|
16
16
|
return pressables;
|
|
17
17
|
}
|
|
18
|
+
/** A count badge such as "1" or "99+" — the only "label" an icon button like a cart carries. */
|
|
19
|
+
const COUNT_BADGE_LABEL = /^\d{1,3}\+?$/;
|
|
18
20
|
/**
|
|
19
21
|
* Replace component-name fallback labels with semantic icon labels when the
|
|
20
22
|
* pressable's icon child name carries recognizable semantics:
|
|
21
23
|
* { label: "[FloatingHeader]", icon: "SvgChevronBackward" }
|
|
22
24
|
* → label "[SvgChevronBackward — possibly back button]"
|
|
25
|
+
* An icon button whose only text is a count badge (cart with "1") keeps that
|
|
26
|
+
* count as nearby context while the label is upgraded to the icon's meaning:
|
|
27
|
+
* { label: "1", icon: "SvgCartNew" } → label "[SvgCartNew — possibly cart button]", nearbyText "1"
|
|
23
28
|
* Labels from text/a11y are never touched (icon is null for those).
|
|
24
29
|
*/
|
|
25
30
|
export function applyIconHintToLabel(p) {
|
|
26
31
|
if (!p.icon)
|
|
27
32
|
return p;
|
|
28
33
|
const hint = iconSemanticHint(p.icon);
|
|
29
|
-
if (hint)
|
|
34
|
+
if (hint) {
|
|
35
|
+
if (p.label && COUNT_BADGE_LABEL.test(p.label.trim()) && !p.nearbyText) {
|
|
36
|
+
p.nearbyText = p.label;
|
|
37
|
+
}
|
|
30
38
|
p.label = `[${p.icon} — ${hint}]`;
|
|
31
|
-
|
|
39
|
+
}
|
|
40
|
+
else if (!p.label) {
|
|
32
41
|
p.label = `[${p.icon}]`;
|
|
42
|
+
}
|
|
33
43
|
return p;
|
|
34
44
|
}
|
|
35
|
-
const identityCoords = (
|
|
45
|
+
const identityCoords = (item) => ({ center: item.center, frame: item.bounds });
|
|
46
|
+
const TEXT_DISPLAY_MAX = 80;
|
|
47
|
+
const IMAGE_SRC_DISPLAY_MAX = 60;
|
|
48
|
+
function truncate(s, max) {
|
|
49
|
+
return s.length > max ? s.slice(0, max) + "…" : s;
|
|
50
|
+
}
|
|
51
|
+
/** One merged-list line for a static text node — coordinates match pressable lines. */
|
|
52
|
+
export function formatTextEntry(t, convert = identityCoords, opts = {}) {
|
|
53
|
+
const { center, frame } = convert(t);
|
|
54
|
+
const body = opts.fullText ? t.text : truncate(t.text, TEXT_DISPLAY_MAX);
|
|
55
|
+
return ` (${center.x}, ${center.y}) 📝 "${body}" frame:(${frame.x},${frame.y} ${frame.width}x${frame.height})`;
|
|
56
|
+
}
|
|
57
|
+
/** One merged-list line for an image node — coordinates match pressable lines. */
|
|
58
|
+
export function formatImageEntry(img, convert = identityCoords) {
|
|
59
|
+
const { center, frame } = convert(img);
|
|
60
|
+
const src = img.src ? ` src="${truncate(img.src, IMAGE_SRC_DISPLAY_MAX)}"` : "";
|
|
61
|
+
const alt = img.alt ? ` alt="${img.alt}"` : "";
|
|
62
|
+
return ` (${center.x}, ${center.y}) 🖼 Image ${frame.width}x${frame.height}${src}${alt} frame:(${frame.x},${frame.y} ${frame.width}x${frame.height})`;
|
|
63
|
+
}
|
|
64
|
+
const TEXT_CAP = 60;
|
|
65
|
+
const IMAGE_CAP = 40;
|
|
36
66
|
/**
|
|
37
|
-
* Render a ScreenState as the
|
|
38
|
-
*
|
|
39
|
-
*
|
|
67
|
+
* Render a ScreenState as the orientation summary used by get_screen_state and the
|
|
68
|
+
* screenshot tools. By default it merges pressables, static text (📝), and images
|
|
69
|
+
* (🖼) into one spatially-ordered list per reachability group — enough to read and
|
|
70
|
+
* navigate the screen without a screenshot:
|
|
71
|
+
* 📍 Detail [Detail]
|
|
40
72
|
* 🎯 Pressables:
|
|
41
|
-
* (
|
|
73
|
+
* (210, 175) 🖼 Image 420x350 src="…"
|
|
74
|
+
* (146, 394) 📝 "Valya product" frame:(20,382 251x24)
|
|
75
|
+
* (210, 838) <Button /> "In cart" frame:(20,810 380x56)
|
|
76
|
+
* pressablesOnly restores the lean pressable-only snapshot; fullText disables the
|
|
77
|
+
* 80-char text truncation.
|
|
42
78
|
*/
|
|
43
|
-
export function formatScreenStateSummary(ss, convert = identityCoords) {
|
|
79
|
+
export function formatScreenStateSummary(ss, convert = identityCoords, opts = {}) {
|
|
44
80
|
const lines = [];
|
|
45
81
|
if (ss.route) {
|
|
46
82
|
lines.push(`📍 Currently focused screen: "${ss.route.name}" [navigation stack: ${ss.route.stack.join(" > ")}]`);
|
|
@@ -54,38 +90,75 @@ export function formatScreenStateSummary(ss, convert = identityCoords) {
|
|
|
54
90
|
}
|
|
55
91
|
const formatPressable = (p) => {
|
|
56
92
|
const { center, frame } = convert(p);
|
|
57
|
-
|
|
93
|
+
// 🔘 marks tap targets in the enriched view (where 📝 text / 🖼 images also
|
|
94
|
+
// appear); omitted under pressablesOnly to keep the lean legacy lines.
|
|
95
|
+
const marker = opts.pressablesOnly ? "" : " 🔘";
|
|
96
|
+
return ` (${center.x}, ${center.y})${marker}${p.component ? ` <${p.component} />` : ""} ${p.label ? `"${p.label}"` : "(unlabeled)"}` +
|
|
58
97
|
`${p.nearbyText ? ` near "${p.nearbyText}"` : ""}${p.onPressHint ? ` ${p.onPressHint}` : ""}` +
|
|
59
98
|
`${p.testID ? ` testID="${p.testID}"` : ""}${p.isInput ? " [input]" : ""}` +
|
|
60
99
|
` frame:(${frame.x},${frame.y} ${frame.width}x${frame.height})`;
|
|
61
100
|
};
|
|
101
|
+
// Merge pressables + (unless pressablesOnly) texts + images for one reachability
|
|
102
|
+
// group, sorted spatially (top→bottom, then left→right). A text duplicating a
|
|
103
|
+
// pressable's nearbyText is dropped; texts/images cap with an explicit marker.
|
|
104
|
+
const renderGroup = (pressables, texts, images) => {
|
|
105
|
+
const out = [];
|
|
106
|
+
const nearby = new Set(pressables.map((p) => (p.nearbyText || "").trim()).filter(Boolean));
|
|
107
|
+
const freshTexts = opts.pressablesOnly ? [] : texts.filter((t) => !nearby.has(t.text.trim()));
|
|
108
|
+
const useTexts = freshTexts.slice(0, TEXT_CAP);
|
|
109
|
+
const useImages = opts.pressablesOnly ? [] : images.slice(0, IMAGE_CAP);
|
|
110
|
+
const rows = [
|
|
111
|
+
...pressables.map((p) => ({ y: p.center.y, x: p.center.x, line: formatPressable(p) })),
|
|
112
|
+
...useTexts.map((t) => ({ y: t.center.y, x: t.center.x, line: formatTextEntry(t, convert, opts) })),
|
|
113
|
+
...useImages.map((img) => ({ y: img.center.y, x: img.center.x, line: formatImageEntry(img, convert) })),
|
|
114
|
+
];
|
|
115
|
+
rows.sort((a, b) => (a.y !== b.y ? a.y - b.y : a.x - b.x));
|
|
116
|
+
rows.forEach((r) => out.push(r.line));
|
|
117
|
+
if (!opts.pressablesOnly) {
|
|
118
|
+
const droppedT = freshTexts.length - useTexts.length;
|
|
119
|
+
const droppedI = images.length - useImages.length;
|
|
120
|
+
if (droppedT > 0)
|
|
121
|
+
out.push(` … +${droppedT} more text`);
|
|
122
|
+
if (droppedI > 0)
|
|
123
|
+
out.push(` … +${droppedI} more images`);
|
|
124
|
+
}
|
|
125
|
+
return out;
|
|
126
|
+
};
|
|
62
127
|
if (ss.overlays.length > 0) {
|
|
63
128
|
for (const overlay of ss.overlays) {
|
|
64
129
|
lines.push(`\n🔲 ${overlay.type}${overlay.title ? ` — "${overlay.title}"` : ""}:`);
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
else
|
|
130
|
+
const body = renderGroup(overlay.pressables, overlay.texts ?? [], overlay.images ?? []);
|
|
131
|
+
if (body.length > 0)
|
|
132
|
+
body.forEach((l) => lines.push(l));
|
|
133
|
+
else
|
|
69
134
|
lines.push(" (no pressables)");
|
|
70
|
-
}
|
|
71
135
|
}
|
|
72
|
-
const
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
136
|
+
const reachableP = ss.pressables.filter((p) => !p.blockedByOverlay);
|
|
137
|
+
const blockedP = ss.pressables.filter((p) => p.blockedByOverlay);
|
|
138
|
+
const reachableT = ss.texts.filter((t) => !t.blockedByOverlay);
|
|
139
|
+
const blockedT = ss.texts.filter((t) => t.blockedByOverlay);
|
|
140
|
+
const reachableI = ss.images.filter((i) => !i.blockedByOverlay);
|
|
141
|
+
const blockedI = ss.images.filter((i) => i.blockedByOverlay);
|
|
142
|
+
const rootBody = renderGroup(reachableP, reachableT, reachableI);
|
|
143
|
+
if (opts.pressablesOnly) {
|
|
144
|
+
lines.push(`\n🎯 Root pressables: ${reachableP.length > 0 ? "" : "(none reachable)"}`);
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
lines.push(`\n🎯 Reachable (outside any overlay):${rootBody.length > 0 ? "" : " (nothing reachable)"}`);
|
|
148
|
+
}
|
|
149
|
+
rootBody.forEach((l) => lines.push(l));
|
|
150
|
+
if (blockedP.length > 0 || blockedT.length > 0 || blockedI.length > 0) {
|
|
77
151
|
lines.push(`\n🚫 Blocked by overlay (visible on the underlying screen but taps will NOT reach them until the overlay closes):`);
|
|
78
|
-
|
|
152
|
+
renderGroup(blockedP, blockedT, blockedI).forEach((l) => lines.push(l));
|
|
79
153
|
}
|
|
80
154
|
}
|
|
81
155
|
else {
|
|
82
|
-
lines.push("\n🎯 Pressables:");
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
else
|
|
156
|
+
lines.push(opts.pressablesOnly ? "\n🎯 Pressables:" : "\n🎯 On screen:");
|
|
157
|
+
const body = renderGroup(ss.pressables, ss.texts, ss.images);
|
|
158
|
+
if (body.length > 0)
|
|
159
|
+
body.forEach((l) => lines.push(l));
|
|
160
|
+
else
|
|
87
161
|
lines.push(" (none)");
|
|
88
|
-
}
|
|
89
162
|
}
|
|
90
163
|
return lines.join("\n");
|
|
91
164
|
}
|
|
@@ -124,8 +197,10 @@ export function describePressHandler(raw) {
|
|
|
124
197
|
* Fallback handler context from the custom component's on* props, for when the
|
|
125
198
|
* direct onPress is anonymous (Hermes discards source even in dev bundles, and
|
|
126
199
|
* names like navigation.goBack are lost to computed assignment):
|
|
127
|
-
* -
|
|
128
|
-
*
|
|
200
|
+
* - only trust the prop whose value IS the touchable's onPress (pass-through);
|
|
201
|
+
* a non-matched candidate is an unverifiable guess that mislabels every button
|
|
202
|
+
* in a multi-button container (FloatingHeader exposes only onBack, but its menu
|
|
203
|
+
* and cart buttons run internal handlers) — so it yields null
|
|
129
204
|
* - named fn → "onBack=goBack()"; nameless → "onPress→onBack" (the prop name
|
|
130
205
|
* alone is greppable context); a bare nameless "onPress" prop adds nothing → null
|
|
131
206
|
*/
|
|
@@ -133,7 +208,7 @@ export function describePropHandlers(raw) {
|
|
|
133
208
|
if (!Array.isArray(raw) || raw.length === 0)
|
|
134
209
|
return null;
|
|
135
210
|
const entries = raw.filter((e) => !!e && typeof e === "object" && typeof e.p === "string");
|
|
136
|
-
const pick = entries.find((e) => e.same) ??
|
|
211
|
+
const pick = entries.find((e) => e.same) ?? null;
|
|
137
212
|
if (!pick)
|
|
138
213
|
return null;
|
|
139
214
|
const name = meaningfulHandlerName(pick.n);
|
|
@@ -153,6 +228,8 @@ export function parseScreenStateResponse(raw) {
|
|
|
153
228
|
route: r.route ?? null,
|
|
154
229
|
overlays: r.overlays ?? [],
|
|
155
230
|
pressables: r.pressables ?? [],
|
|
231
|
+
texts: r.texts ?? [],
|
|
232
|
+
images: r.images ?? [],
|
|
156
233
|
};
|
|
157
234
|
}
|
|
158
235
|
// ============================================================================
|
|
@@ -488,6 +565,9 @@ export async function getScreenState(options = {}) {
|
|
|
488
565
|
|
|
489
566
|
var hostFibers = [];
|
|
490
567
|
var fiberMeta = [];
|
|
568
|
+
// A count badge ("1", "99+") is the only text on icon buttons like a cart — treat it
|
|
569
|
+
// as no-own-label so the icon child still surfaces (the count is kept as nearby text).
|
|
570
|
+
var COUNT_BADGE = /^\\d{1,3}\\+?$/;
|
|
491
571
|
|
|
492
572
|
var RN_PRIMITIVES = /^(Animated\\(.*|withAnimated.*|AnimatedComponent.*|ForwardRef.*|memo\\(.*|Context\\.Consumer|Context\\.Provider|ScrollViewContext(Base)?|VirtualizedListContext(Resetter)?|TextInputContext|KeyboardAvoidingViewContext|RCT.*|RNS.*|RNC.*|ViewManagerAdapter_.*|VirtualizedList.*|CellRenderer.*|FrameSizeProvider.*|MaybeScreenContainer|MaybeScreen|Navigation.*|Screen$|ScreenStack|ScreenContainer|ScreenContentWrapper|SceneView|DelayedFreeze|Freeze|Suspender|DebugContainer|StaticContainer|SafeAreaProvider.*|SafeAreaFrameContext|SafeAreaInsetsContext|ExpoRoot|ExpoRootComponent|GestureHandler.*|NativeViewGestureHandler|GestureDetector|PanGestureHandler|Reanimated.*|BottomTabNavigator|TabLayout|RouteNode|Route$|KeyboardProvider|PortalProviderComponent|BottomSheetModalProviderWrapper|ThemeContext|ThemeProvider|TextAncestorContext|PressabilityDebugView|TouchableHighlightImpl|StatusBarOverlay|BottomSheetHostingContainerComponent|BottomSheetGestureHandlersProvider|BottomSheetBackdropContainerComponent|BottomSheetContainerComponent|BottomSheetDraggableViewComponent|BottomSheetHandleContainerComponent|BottomSheetBackgroundContainerComponent|DebuggingOverlay|InspectorDeferred|Inspector|InspectorOverlay|InspectorPanel|StyleInspector|BoxInspector|BoxContainer|ElementBox|BorderBox|InspectorPanelButton)$/;
|
|
493
573
|
var GENERIC_COMPONENT = /^(View|TouchableOpacity|TouchableHighlight|TouchableWithoutFeedback|Pressable|TouchableNativeFeedback|Text|RCTView|RCTText|Unknown)$/;
|
|
@@ -663,7 +743,8 @@ export async function getScreenState(options = {}) {
|
|
|
663
743
|
var a11y = hProps.accessibilityLabel || pProps.accessibilityLabel || null;
|
|
664
744
|
var baseLabel = a11y || (text && text.length > 0 ? text.slice(0, 80) : null) || null;
|
|
665
745
|
var label = resolveLabel(pressableFiber, hostFiber, baseLabel, testID);
|
|
666
|
-
var
|
|
746
|
+
var badgeOnly = baseLabel ? COUNT_BADGE.test(baseLabel.trim()) : false;
|
|
747
|
+
var icon = (baseLabel && !badgeOnly) ? null : findMeaningfulChildName(pressableFiber);
|
|
667
748
|
if (!baseLabel && !icon) {
|
|
668
749
|
var ownName = getComponentName(pressableFiber);
|
|
669
750
|
if (ownName && !GENERIC_COMPONENT.test(ownName) && !RN_PRIMITIVES.test(ownName)) icon = ownName;
|
|
@@ -687,7 +768,8 @@ export async function getScreenState(options = {}) {
|
|
|
687
768
|
var testID2 = p2.testID || p2.nativeID || null;
|
|
688
769
|
var baseLabel2 = a11y2 || (text2 && text2.length > 0 ? text2.slice(0, 80) : null) || null;
|
|
689
770
|
var label2 = resolveLabel(fiber, hosts3[0], baseLabel2, testID2);
|
|
690
|
-
var
|
|
771
|
+
var badgeOnly2 = baseLabel2 ? COUNT_BADGE.test(baseLabel2.trim()) : false;
|
|
772
|
+
var icon2 = (baseLabel2 && !badgeOnly2) ? null : findMeaningfulChildName(fiber);
|
|
691
773
|
if (!baseLabel2 && !icon2) {
|
|
692
774
|
var ownName2 = getComponentName(fiber);
|
|
693
775
|
if (ownName2 && !GENERIC_COMPONENT.test(ownName2) && !RN_PRIMITIVES.test(ownName2)) icon2 = ownName2;
|
|
@@ -733,6 +815,7 @@ export async function getScreenState(options = {}) {
|
|
|
733
815
|
|
|
734
816
|
var textFibers = [];
|
|
735
817
|
var textContents = [];
|
|
818
|
+
var textOverlayIdx = [];
|
|
736
819
|
|
|
737
820
|
function extractTextString(fiber) {
|
|
738
821
|
var p = fiber.memoizedProps;
|
|
@@ -751,12 +834,20 @@ export async function getScreenState(options = {}) {
|
|
|
751
834
|
return '';
|
|
752
835
|
}
|
|
753
836
|
|
|
754
|
-
function walkTexts(fiber, depth, insidePressable, inHidden) {
|
|
837
|
+
function walkTexts(fiber, depth, insidePressable, inHidden, ovIdx) {
|
|
755
838
|
if (!fiber || depth > 5000) return;
|
|
756
839
|
var name = getComponentName(fiber);
|
|
757
840
|
var props = fiber.memoizedProps;
|
|
758
841
|
var nextHidden = inHidden || isScreenHidden(name, props);
|
|
759
842
|
|
|
843
|
+
// Overlay membership by ancestry (same as the pressable walk) so a sheet's
|
|
844
|
+
// text is grouped with the sheet, not flagged as blocked behind it.
|
|
845
|
+
if (ovIdx == null) {
|
|
846
|
+
for (var ofiT = 0; ofiT < overlayFiberMeta.length; ofiT++) {
|
|
847
|
+
if (overlayFiberMeta[ofiT].fiber === fiber) { ovIdx = ofiT; break; }
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
|
|
760
851
|
var hasOnPress = props && typeof props.onPress === 'function';
|
|
761
852
|
var isInputHere = props && (typeof props.onChangeText === 'function' || typeof props.onFocus === 'function');
|
|
762
853
|
var nextInside = insidePressable || !!hasOnPress || !!isInputHere;
|
|
@@ -766,7 +857,7 @@ export async function getScreenState(options = {}) {
|
|
|
766
857
|
// for proxy bounds (Fabric RCTText has no publicInstance).
|
|
767
858
|
if (!insidePressable && !nextHidden && name !== 'RCTText' && typeof fiber.type !== 'string') {
|
|
768
859
|
var str = extractTextString(fiber);
|
|
769
|
-
if (str && str.length > 0 && str.length <=
|
|
860
|
+
if (str && str.length > 0 && str.length <= 300) {
|
|
770
861
|
var up = fiber;
|
|
771
862
|
var upDepth = 0;
|
|
772
863
|
var measurableT = null;
|
|
@@ -781,17 +872,73 @@ export async function getScreenState(options = {}) {
|
|
|
781
872
|
if (measurableT) {
|
|
782
873
|
textFibers.push(measurableT);
|
|
783
874
|
textContents.push(str);
|
|
875
|
+
textOverlayIdx.push(ovIdx);
|
|
784
876
|
}
|
|
785
877
|
}
|
|
786
878
|
}
|
|
787
879
|
|
|
788
880
|
var child = fiber.child;
|
|
789
881
|
while (child) {
|
|
790
|
-
walkTexts(child, depth + 1, nextInside, nextHidden);
|
|
882
|
+
walkTexts(child, depth + 1, nextInside, nextHidden, ovIdx);
|
|
791
883
|
child = child.sibling;
|
|
792
884
|
}
|
|
793
885
|
}
|
|
794
|
-
walkTexts(roots[0].current, 0, false, false);
|
|
886
|
+
walkTexts(roots[0].current, 0, false, false, null);
|
|
887
|
+
|
|
888
|
+
// ------------------------------------------------------------------
|
|
889
|
+
// 3c. Images — host image components, with source + a11y label. Climb to
|
|
890
|
+
// the nearest measurable host (same proxy-bounds trick as text).
|
|
891
|
+
// ------------------------------------------------------------------
|
|
892
|
+
|
|
893
|
+
var imageFibers = [];
|
|
894
|
+
var imageMeta = []; // { src, alt }
|
|
895
|
+
var IMG_NAME = /^(Image|RCTImageView|ExpoImage|FastImage|ImageBackground)$/;
|
|
896
|
+
|
|
897
|
+
function imageSource(props) {
|
|
898
|
+
if (!props) return null;
|
|
899
|
+
var s = props.source;
|
|
900
|
+
if (s == null) return null;
|
|
901
|
+
if (Array.isArray(s)) s = s[0];
|
|
902
|
+
if (typeof s === 'number') return 'asset#' + s;
|
|
903
|
+
if (s && typeof s === 'object' && typeof s.uri === 'string') {
|
|
904
|
+
return s.uri.length > 200 ? s.uri.slice(0, 200) : s.uri;
|
|
905
|
+
}
|
|
906
|
+
if (typeof s === 'string') return s.length > 200 ? s.slice(0, 200) : s;
|
|
907
|
+
return null;
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
function walkImages(fiber, depth, inHidden, ovIdx) {
|
|
911
|
+
if (!fiber || depth > 5000) return;
|
|
912
|
+
var name = getComponentName(fiber);
|
|
913
|
+
var props = fiber.memoizedProps;
|
|
914
|
+
var nextHidden = inHidden || isScreenHidden(name, props);
|
|
915
|
+
if (ovIdx == null) {
|
|
916
|
+
for (var ofiI = 0; ofiI < overlayFiberMeta.length; ofiI++) {
|
|
917
|
+
if (overlayFiberMeta[ofiI].fiber === fiber) { ovIdx = ofiI; break; }
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
if (!nextHidden && name && IMG_NAME.test(name) && typeof fiber.type !== 'string') {
|
|
921
|
+
var up = fiber;
|
|
922
|
+
var upD = 0;
|
|
923
|
+
var measurableI = null;
|
|
924
|
+
while (up && upD < 20) {
|
|
925
|
+
if (typeof up.type === 'string' && getMeasurable(up)) { measurableI = up; break; }
|
|
926
|
+
up = up.return;
|
|
927
|
+
upD++;
|
|
928
|
+
}
|
|
929
|
+
if (measurableI) {
|
|
930
|
+
imageFibers.push(measurableI);
|
|
931
|
+
imageMeta.push({
|
|
932
|
+
src: imageSource(props),
|
|
933
|
+
alt: (props && typeof props.accessibilityLabel === 'string') ? props.accessibilityLabel.slice(0, 80) : null,
|
|
934
|
+
overlayIdx: ovIdx
|
|
935
|
+
});
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
var child = fiber.child;
|
|
939
|
+
while (child) { walkImages(child, depth + 1, nextHidden, ovIdx); child = child.sibling; }
|
|
940
|
+
}
|
|
941
|
+
walkImages(roots[0].current, 0, false, null);
|
|
795
942
|
|
|
796
943
|
// ------------------------------------------------------------------
|
|
797
944
|
// 4. Store everything in globalThis for the resolve call
|
|
@@ -830,6 +977,9 @@ export async function getScreenState(options = {}) {
|
|
|
830
977
|
globalThis.__screenStateOverlayMeasurements = new Array(overlayHostFibers.length).fill(null);
|
|
831
978
|
globalThis.__screenStateTextContents = textContents;
|
|
832
979
|
globalThis.__screenStateTextMeasurements = new Array(textFibers.length).fill(null);
|
|
980
|
+
globalThis.__screenStateTextOverlayIdx = textOverlayIdx;
|
|
981
|
+
globalThis.__screenStateImageMeta = imageMeta;
|
|
982
|
+
globalThis.__screenStateImageMeasurements = new Array(imageFibers.length).fill(null);
|
|
833
983
|
|
|
834
984
|
// Dispatch all measureInWindow calls (pressables + root + overlay hosts)
|
|
835
985
|
for (var i = 0; i < hostFibers.length; i++) {
|
|
@@ -859,6 +1009,15 @@ export async function getScreenState(options = {}) {
|
|
|
859
1009
|
})(txi);
|
|
860
1010
|
} catch(e) {}
|
|
861
1011
|
}
|
|
1012
|
+
for (var imi = 0; imi < imageFibers.length; imi++) {
|
|
1013
|
+
try {
|
|
1014
|
+
(function(idx) {
|
|
1015
|
+
getMeasurable(imageFibers[idx]).measureInWindow(function(fx, fy, fw, fh) {
|
|
1016
|
+
globalThis.__screenStateImageMeasurements[idx] = { x: fx, y: fy, width: fw, height: fh };
|
|
1017
|
+
});
|
|
1018
|
+
})(imi);
|
|
1019
|
+
} catch(e) {}
|
|
1020
|
+
}
|
|
862
1021
|
|
|
863
1022
|
return { count: hostFibers.length, overlayCount: overlayFiberMeta.length };
|
|
864
1023
|
})()
|
|
@@ -884,8 +1043,14 @@ export async function getScreenState(options = {}) {
|
|
|
884
1043
|
var overlayMeasurements = globalThis.__screenStateOverlayMeasurements || [];
|
|
885
1044
|
var textContents = globalThis.__screenStateTextContents || [];
|
|
886
1045
|
var textMeasurements = globalThis.__screenStateTextMeasurements || [];
|
|
1046
|
+
var textOverlayIdx = globalThis.__screenStateTextOverlayIdx || [];
|
|
1047
|
+
var imageMeta = globalThis.__screenStateImageMeta || [];
|
|
1048
|
+
var imageMeasurements = globalThis.__screenStateImageMeasurements || [];
|
|
887
1049
|
globalThis.__screenStateTextContents = null;
|
|
888
1050
|
globalThis.__screenStateTextMeasurements = null;
|
|
1051
|
+
globalThis.__screenStateTextOverlayIdx = null;
|
|
1052
|
+
globalThis.__screenStateImageMeta = null;
|
|
1053
|
+
globalThis.__screenStateImageMeasurements = null;
|
|
889
1054
|
globalThis.__screenStateFibers = null;
|
|
890
1055
|
globalThis.__screenStateMeta = null;
|
|
891
1056
|
globalThis.__screenStateMeasurements = null;
|
|
@@ -1043,6 +1208,46 @@ export async function getScreenState(options = {}) {
|
|
|
1043
1208
|
delete allPressables[si].hasOwnLabel;
|
|
1044
1209
|
}
|
|
1045
1210
|
|
|
1211
|
+
// Build typed text/image lists (pair each measurement with its meta by index,
|
|
1212
|
+
// viewport-filter) and assign each to a reachability group via the same overlay
|
|
1213
|
+
// logic used for pressables (content containment vs blocking backdrop).
|
|
1214
|
+
var rootTexts = [], rootImages = [];
|
|
1215
|
+
for (var ov = 0; ov < overlays.length; ov++) { overlays[ov].texts = []; overlays[ov].images = []; }
|
|
1216
|
+
// Group an item: by fiber ancestry (overlayIdx) first — a sheet's own text/image
|
|
1217
|
+
// belongs to the sheet; else geometric containment in an overlay's content; else
|
|
1218
|
+
// root, flagged blocked when it sits under a blocking overlay (the screen behind).
|
|
1219
|
+
function pushClassified(entry, kind, ovIdx) {
|
|
1220
|
+
for (var a = 0; a < overlays.length; a++) {
|
|
1221
|
+
if (ovIdx != null && ovIdx === overlays[a].origIdx) { overlays[a][kind].push(entry); return; }
|
|
1222
|
+
}
|
|
1223
|
+
for (var c = 0; c < overlays.length; c++) {
|
|
1224
|
+
if (overlays[c].hasContent && inside(entry.bounds, overlays[c].contentBounds)) { overlays[c][kind].push(entry); return; }
|
|
1225
|
+
}
|
|
1226
|
+
var blk = false;
|
|
1227
|
+
for (var b = 0; b < overlays.length; b++) { if (inside(entry.bounds, overlays[b].blockBounds)) { blk = true; break; } }
|
|
1228
|
+
if (blk) entry.blockedByOverlay = true;
|
|
1229
|
+
(kind === 'texts' ? rootTexts : rootImages).push(entry);
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
for (var ti = 0; ti < textContents.length; ti++) {
|
|
1233
|
+
var tm = textMeasurements[ti];
|
|
1234
|
+
if (!tm || tm.width <= 0 || tm.height <= 0) continue;
|
|
1235
|
+
if (tm.x + tm.width < 0 || tm.y + tm.height < 0) continue;
|
|
1236
|
+
if (tm.x > viewportW || tm.y > viewportH) continue;
|
|
1237
|
+
pushClassified({ text: textContents[ti],
|
|
1238
|
+
center: { x: Math.round(tm.x + tm.width/2), y: Math.round(tm.y + tm.height/2) },
|
|
1239
|
+
bounds: { x: Math.round(tm.x), y: Math.round(tm.y), width: Math.round(tm.width), height: Math.round(tm.height) } }, 'texts', textOverlayIdx[ti]);
|
|
1240
|
+
}
|
|
1241
|
+
for (var ii = 0; ii < imageMeta.length; ii++) {
|
|
1242
|
+
var im = imageMeasurements[ii];
|
|
1243
|
+
if (!im || im.width <= 0 || im.height <= 0) continue;
|
|
1244
|
+
if (im.x + im.width < 0 || im.y + im.height < 0) continue;
|
|
1245
|
+
if (im.x > viewportW || im.y > viewportH) continue;
|
|
1246
|
+
pushClassified({ src: imageMeta[ii].src, alt: imageMeta[ii].alt,
|
|
1247
|
+
center: { x: Math.round(im.x + im.width/2), y: Math.round(im.y + im.height/2) },
|
|
1248
|
+
bounds: { x: Math.round(im.x), y: Math.round(im.y), width: Math.round(im.width), height: Math.round(im.height) } }, 'images', imageMeta[ii].overlayIdx);
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1046
1251
|
// Sort visually (top-to-bottom, left-to-right) — walk order is mount order,
|
|
1047
1252
|
// which puts late-mounted overlays like floating headers at the end.
|
|
1048
1253
|
function byPosition(a, b) {
|
|
@@ -1050,14 +1255,20 @@ export async function getScreenState(options = {}) {
|
|
|
1050
1255
|
return a.center.x - b.center.x;
|
|
1051
1256
|
}
|
|
1052
1257
|
rootPressables.sort(byPosition);
|
|
1053
|
-
|
|
1258
|
+
rootTexts.sort(byPosition);
|
|
1259
|
+
rootImages.sort(byPosition);
|
|
1260
|
+
for (var so = 0; so < overlays.length; so++) {
|
|
1261
|
+
overlays[so].pressables.sort(byPosition);
|
|
1262
|
+
overlays[so].texts.sort(byPosition);
|
|
1263
|
+
overlays[so].images.sort(byPosition);
|
|
1264
|
+
}
|
|
1054
1265
|
|
|
1055
1266
|
// Strip bounds from overlay objects (not in public interface)
|
|
1056
1267
|
var cleanOverlays = overlays.map(function(o) {
|
|
1057
|
-
return { type: o.type, title: o.title, pressables: o.pressables };
|
|
1268
|
+
return { type: o.type, title: o.title, pressables: o.pressables, texts: o.texts, images: o.images };
|
|
1058
1269
|
});
|
|
1059
1270
|
|
|
1060
|
-
return { route: route, overlays: cleanOverlays, pressables: rootPressables };
|
|
1271
|
+
return { route: route, overlays: cleanOverlays, pressables: rootPressables, texts: rootTexts, images: rootImages };
|
|
1061
1272
|
})()
|
|
1062
1273
|
`;
|
|
1063
1274
|
const resolveResult = await executeInApp(resolveExpression, false, { timeoutMs: 15000, originatingToolName: "get_screen_state" }, device);
|
|
@@ -1090,6 +1301,26 @@ export async function getScreenState(options = {}) {
|
|
|
1090
1301
|
for (const overlay of screenState.overlays) {
|
|
1091
1302
|
overlay.pressables = dedupPressables(overlay.pressables);
|
|
1092
1303
|
}
|
|
1304
|
+
// Dedup texts/images: a single visual element often maps to nested host nodes
|
|
1305
|
+
// (Image > ExpoImage) or repeated text spans that all measure to the same box.
|
|
1306
|
+
const dedupBy = (list, keyExtra) => {
|
|
1307
|
+
const seen = new Set();
|
|
1308
|
+
return list.filter((item) => {
|
|
1309
|
+
const key = `${item.center.x},${item.center.y}|${keyExtra(item)}`;
|
|
1310
|
+
if (seen.has(key))
|
|
1311
|
+
return false;
|
|
1312
|
+
seen.add(key);
|
|
1313
|
+
return true;
|
|
1314
|
+
});
|
|
1315
|
+
};
|
|
1316
|
+
screenState.texts = dedupBy(screenState.texts, (t) => t.text);
|
|
1317
|
+
screenState.images = dedupBy(screenState.images, (i) => i.src ?? "");
|
|
1318
|
+
for (const overlay of screenState.overlays) {
|
|
1319
|
+
if (overlay.texts)
|
|
1320
|
+
overlay.texts = dedupBy(overlay.texts, (t) => t.text);
|
|
1321
|
+
if (overlay.images)
|
|
1322
|
+
overlay.images = dedupBy(overlay.images, (i) => i.src ?? "");
|
|
1323
|
+
}
|
|
1093
1324
|
// Semantic icon hints + onPress handler hints
|
|
1094
1325
|
const decorate = (p) => {
|
|
1095
1326
|
applyIconHintToLabel(p);
|