@blario/react-native 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/BlarProvider.d.ts +2 -0
- package/dist/actions/select.d.ts +1 -1
- package/dist/actions/text-input.d.ts +1 -0
- package/dist/agent/compose.d.ts +1 -0
- package/dist/agent-mode/keyboard-guard.d.ts +13 -0
- package/dist/snapshot/capture.d.ts +1 -0
- package/dist/snapshot/visibility.d.ts +1 -1
- package/dist/snapshot/wrappers.d.ts +0 -2
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
- package/src/BlarProvider.tsx +13 -0
- package/src/actions/executor.ts +6 -3
- package/src/actions/select.ts +16 -2
- package/src/actions/text-input.ts +30 -1
- package/src/agent/compose.ts +2 -0
- package/src/agent/controller.ts +3 -1
- package/src/agent-mode/keyboard-guard.ts +33 -0
- package/src/chat/ChatSheet.tsx +29 -9
- package/src/chat/ChatView.tsx +1 -1
- package/src/chat/widget-bundle.generated.ts +2 -2
- package/src/snapshot/capture.ts +8 -6
- package/src/snapshot/render.ts +1 -1
- package/src/snapshot/visibility.ts +11 -10
- package/src/snapshot/wrappers.ts +0 -13
- package/src/version.ts +1 -1
package/README.md
CHANGED
|
@@ -63,6 +63,7 @@ export default function App() {
|
|
|
63
63
|
Screenshots use `react-native-view-shot` when it is installed, otherwise `@shopify/react-native-skia`
|
|
64
64
|
(`makeImageFromView`) on iOS. Android requires view-shot: Skia's Android capture can raise a fatal native error
|
|
65
65
|
when a view unmounts during capture. With no supported capture module, a screenshot answers `sdk_error`.
|
|
66
|
+
Android on the New Architecture needs view-shot 4 or later: 3.x fails every capture there, while actions keep working.
|
|
66
67
|
Each module is resolved once when the SDK loads:
|
|
67
68
|
|
|
68
69
|
- view-shot leaves a host `Modal` out (it is a separate window), so the response carries `partial: true` and
|
|
@@ -230,7 +231,7 @@ A run looks and behaves like the web widget's agent mode (`web-sdk/src/managers/
|
|
|
230
231
|
installed (both optional peers); without one this is a no-op, reported once with `debug`.
|
|
231
232
|
|
|
232
233
|
Known differences from the web: the pending dim covers the whole screen (the web dims only the chat window); the
|
|
233
|
-
|
|
234
|
+
keyboard stays closed during a run, unless the chat is open; `session_closed` ends the run (the web only
|
|
234
235
|
tears it down). Visual effects (dot grid, glow, slime) come in a later milestone.
|
|
235
236
|
|
|
236
237
|
### Test nudges without a real delivery
|
|
@@ -333,7 +334,8 @@ Wire envelope (mirrors the backend): `agent_command` / `request_dom_state` / `re
|
|
|
333
334
|
| `highlight` | `duration_ms` (default 3000) | any element |
|
|
334
335
|
|
|
335
336
|
`click`, `type`, `check` and `highlight` scroll the target into view first. Elements that sit beyond their scroll
|
|
336
|
-
viewport are listed with `offscreen=above|below|left|right
|
|
337
|
+
viewport are listed with `offscreen=above|below|left|right`, however many there are. The page text also includes what a
|
|
338
|
+
list holds off screen in any direction; only blocks parked beyond the window with no list behind them are left out. A wheel picker is a snapping
|
|
337
339
|
list (`snapToInterval` or `snapToOffsets`) of plain text rows: each row is a `btn-N`, and all of them stay listed. Lists
|
|
338
340
|
carry `list=<name>` from their own or their wrapper's `accessibilityLabel` or `testID`, and so do those rows. `select`
|
|
339
341
|
needs no snap props: it reaches any list, so a wheel that snaps in JS works through it.
|
package/dist/BlarProvider.d.ts
CHANGED
|
@@ -42,6 +42,7 @@ export declare class BlarProvider extends Component<BlarProviderProps, BlarProvi
|
|
|
42
42
|
private chatControls;
|
|
43
43
|
private shownNudge;
|
|
44
44
|
private readonly releases;
|
|
45
|
+
private readonly keyboardGuard;
|
|
45
46
|
private stackingCheckQueued;
|
|
46
47
|
private loggedPhase;
|
|
47
48
|
private readonly stacking;
|
|
@@ -62,6 +63,7 @@ export declare class BlarProvider extends Component<BlarProviderProps, BlarProvi
|
|
|
62
63
|
private createTransport;
|
|
63
64
|
private createChat;
|
|
64
65
|
private attachBackend;
|
|
66
|
+
private syncKeyboardGuard;
|
|
65
67
|
private queueStackingCheck;
|
|
66
68
|
private keepLayerOnTop;
|
|
67
69
|
private countHostModals;
|
package/dist/actions/select.d.ts
CHANGED
|
@@ -11,6 +11,6 @@ export declare function scrollListTo(target: ResolvedTarget, list: Fiber, offset
|
|
|
11
11
|
export declare function selectWheelOption(target: ResolvedTarget): Promise<HandlerResult>;
|
|
12
12
|
export declare function rowsOf(list: Fiber): ListRow[];
|
|
13
13
|
export declare function matchesRow(text: string, value: string): boolean;
|
|
14
|
-
export declare function selectValue(target: ResolvedTarget, value: string, services: ActionServices): Promise<HandlerResult>;
|
|
14
|
+
export declare function selectValue(target: ResolvedTarget, value: string, services: ActionServices, isCancelled?: () => boolean): Promise<HandlerResult>;
|
|
15
15
|
export declare function walk(from: number, to: number, step: number): number[];
|
|
16
16
|
export {};
|
|
@@ -4,6 +4,7 @@ import type { AgentAction, HandlerResult } from './types.ts';
|
|
|
4
4
|
type TypeAction = Extract<AgentAction, {
|
|
5
5
|
type: 'type';
|
|
6
6
|
}>;
|
|
7
|
+
export declare function restoreKeyboards(): void;
|
|
7
8
|
export declare function typeText(target: ResolvedTarget, action: TypeAction, services: ActionServices): HandlerResult;
|
|
8
9
|
export declare function submitInput(target: ResolvedTarget): HandlerResult;
|
|
9
10
|
export {};
|
package/dist/agent/compose.d.ts
CHANGED
|
@@ -11,6 +11,7 @@ export interface AgentEnvironment {
|
|
|
11
11
|
transport: AgentTransport;
|
|
12
12
|
getAnchorFiber: () => Fiber | null;
|
|
13
13
|
getViewport: () => Size;
|
|
14
|
+
getWindow: () => Size;
|
|
14
15
|
getNavigationRef: () => NavigationRefLike | null;
|
|
15
16
|
getCaptureTarget: () => unknown;
|
|
16
17
|
reading: ReadingPort;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export interface KeyboardPort {
|
|
2
|
+
addListener: (event: 'keyboardWillShow' | 'keyboardDidShow', listener: () => void) => {
|
|
3
|
+
remove: () => void;
|
|
4
|
+
};
|
|
5
|
+
dismiss: () => void;
|
|
6
|
+
}
|
|
7
|
+
export declare class KeyboardGuard {
|
|
8
|
+
private readonly keyboard;
|
|
9
|
+
private readonly onRelease;
|
|
10
|
+
private subscriptions;
|
|
11
|
+
constructor(keyboard: KeyboardPort, onRelease?: () => void);
|
|
12
|
+
setActive(active: boolean): void;
|
|
13
|
+
}
|
|
@@ -9,9 +9,9 @@ export type Visibility = {
|
|
|
9
9
|
} | {
|
|
10
10
|
kind: 'offscreen';
|
|
11
11
|
direction: Offscreen;
|
|
12
|
-
distance: number;
|
|
13
12
|
} | {
|
|
14
13
|
kind: 'hidden';
|
|
15
14
|
};
|
|
16
15
|
export declare function classifyVisibility(rect: Rect, window: Size, containers: Fiber[], geometries: ScrollGeometries): Visibility;
|
|
16
|
+
export declare function markOffscreenText(visibility: Visibility, text: string): string;
|
|
17
17
|
export declare function keepInTextTree(visibility: Visibility, rect: Rect, window: Size, viewportOnly: boolean): boolean;
|
|
@@ -5,7 +5,5 @@ export interface MeasuredCandidate {
|
|
|
5
5
|
candidate: Candidate;
|
|
6
6
|
rect: Rect;
|
|
7
7
|
offscreen: Offscreen | null;
|
|
8
|
-
distance: number;
|
|
9
8
|
}
|
|
10
9
|
export declare function dropWrapperDuplicates(measured: MeasuredCandidate[]): MeasuredCandidate[];
|
|
11
|
-
export declare function limitOffscreen(measured: MeasuredCandidate[]): MeasuredCandidate[];
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const SDK_VERSION = "0.1.
|
|
1
|
+
export declare const SDK_VERSION = "0.1.6";
|
package/package.json
CHANGED
package/src/BlarProvider.tsx
CHANGED
|
@@ -13,6 +13,8 @@ import type { AgentModePhase } from './agent-mode/state.ts';
|
|
|
13
13
|
import { KEEP_AWAKE_ADAPTER, KeepAwake } from './agent-mode/keep-awake.ts';
|
|
14
14
|
import { LayerStacking } from './agent-mode/layer-stacking.ts';
|
|
15
15
|
import { AgentLayer } from './agent-mode/ui/AgentLayer.tsx';
|
|
16
|
+
import { restoreKeyboards } from './actions/text-input.ts';
|
|
17
|
+
import { KeyboardGuard } from './agent-mode/keyboard-guard.ts';
|
|
16
18
|
import { AgentLayerHost } from './agent-mode/ui/AgentLayerHost.tsx';
|
|
17
19
|
import { BackendTransport, type WidgetProof } from './backend/backend-transport.ts';
|
|
18
20
|
import type { BlarIdentity } from './backend/identity.ts';
|
|
@@ -104,6 +106,8 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
|
|
|
104
106
|
|
|
105
107
|
private readonly releases: Array<() => void> = [];
|
|
106
108
|
|
|
109
|
+
private readonly keyboardGuard = new KeyboardGuard(Keyboard, restoreKeyboards);
|
|
110
|
+
|
|
107
111
|
private stackingCheckQueued = false;
|
|
108
112
|
|
|
109
113
|
private loggedPhase: AgentModePhase = 'idle';
|
|
@@ -163,6 +167,7 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
|
|
|
163
167
|
transport,
|
|
164
168
|
getAnchorFiber: () => (this as WithReactInternals)._reactInternals ?? null,
|
|
165
169
|
getViewport: () => visibleViewport(Dimensions.get('window'), Keyboard.metrics()?.height ?? 0),
|
|
170
|
+
getWindow: () => Dimensions.get('window'),
|
|
166
171
|
getNavigationRef: () => this.props.navigationRef ?? null,
|
|
167
172
|
getCaptureTarget: () => this.contentRef.current,
|
|
168
173
|
reading: this.spotlight,
|
|
@@ -316,8 +321,10 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
|
|
|
316
321
|
this.debug(`agent mode ${snapshot.phase}`);
|
|
317
322
|
}
|
|
318
323
|
this.setState({ agentMode: snapshot });
|
|
324
|
+
this.syncKeyboardGuard();
|
|
319
325
|
}),
|
|
320
326
|
);
|
|
327
|
+
this.releases.push(() => this.keyboardGuard.setActive(false));
|
|
321
328
|
return agentMode;
|
|
322
329
|
}
|
|
323
330
|
|
|
@@ -373,6 +380,7 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
|
|
|
373
380
|
coordinator.subscribe((chat) => {
|
|
374
381
|
this.noteNudge(chat);
|
|
375
382
|
this.setState({ chat });
|
|
383
|
+
this.syncKeyboardGuard();
|
|
376
384
|
}),
|
|
377
385
|
);
|
|
378
386
|
this.releases.push(backend.onFrame((frame) => coordinator.handleFrame(frame)));
|
|
@@ -395,6 +403,11 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
|
|
|
395
403
|
Blar.attach(backend, (options) => controller.readMessageMetadata({ ...options, tabId: backend.getTabId() }));
|
|
396
404
|
}
|
|
397
405
|
|
|
406
|
+
private syncKeyboardGuard(): void {
|
|
407
|
+
const running = this.agentMode?.getSnapshot().phase === 'active';
|
|
408
|
+
this.keyboardGuard.setActive(running && this.chatControls?.isOpen() !== true);
|
|
409
|
+
}
|
|
410
|
+
|
|
398
411
|
private queueStackingCheck(): void {
|
|
399
412
|
if (this.stackingCheckQueued || (this.agentMode?.getSnapshot().phase ?? 'idle') === 'idle') {
|
|
400
413
|
return;
|
package/src/actions/executor.ts
CHANGED
|
@@ -41,6 +41,9 @@ export class ActionExecutor {
|
|
|
41
41
|
if (command.action.type === 'navigate') {
|
|
42
42
|
return { ok: true, target: null };
|
|
43
43
|
}
|
|
44
|
+
if (command.action.type === 'press_key' && command.action.key !== 'Enter') {
|
|
45
|
+
return { ok: false, outcome: outcomeOf(failed('unsupported_action', `key ${command.action.key} is not supported: only Enter is. To move a list use scroll or select on its scr-N`)) };
|
|
46
|
+
}
|
|
44
47
|
const target = this.resolveTarget(command);
|
|
45
48
|
if (target === null) {
|
|
46
49
|
return { ok: false, outcome: outcomeOf(this.missingTarget(command)) };
|
|
@@ -79,7 +82,7 @@ export class ActionExecutor {
|
|
|
79
82
|
if (options.isCancelled()) {
|
|
80
83
|
return failed('user_cancelled', CANCELLED_MESSAGE);
|
|
81
84
|
}
|
|
82
|
-
return await this.dispatch(command, target);
|
|
85
|
+
return await this.dispatch(command, target, options);
|
|
83
86
|
} catch (error) {
|
|
84
87
|
return failedFromException(error);
|
|
85
88
|
}
|
|
@@ -124,7 +127,7 @@ export class ActionExecutor {
|
|
|
124
127
|
return failed('element_not_found', reason);
|
|
125
128
|
}
|
|
126
129
|
|
|
127
|
-
private dispatch(command: AgentCommandData, target: ResolvedTarget): Promise<HandlerResult> | HandlerResult {
|
|
130
|
+
private dispatch(command: AgentCommandData, target: ResolvedTarget, options: ExecutionOptions): Promise<HandlerResult> | HandlerResult {
|
|
128
131
|
const { action } = command;
|
|
129
132
|
switch (action.type) {
|
|
130
133
|
case 'click':
|
|
@@ -134,7 +137,7 @@ export class ActionExecutor {
|
|
|
134
137
|
case 'check':
|
|
135
138
|
return toggleSwitch(target, action.checked);
|
|
136
139
|
case 'select':
|
|
137
|
-
return selectValue(target, action.value, this.services);
|
|
140
|
+
return selectValue(target, action.value, this.services, options.isCancelled);
|
|
138
141
|
case 'press_key':
|
|
139
142
|
return action.key === 'Enter' ? submitInput(target) : failed('unsupported_action', `key ${action.key} is not supported`);
|
|
140
143
|
case 'scroll':
|
package/src/actions/select.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { childrenOf, isHostComponent, propsOf } from '../fiber/reader.ts';
|
|
|
5
5
|
import { collectText, TEXT_HOST_TYPE } from '../fiber/text.ts';
|
|
6
6
|
import type { Fiber } from '../fiber/types.ts';
|
|
7
7
|
import { measureHost } from '../host/measure.ts';
|
|
8
|
+
import { scrollInstanceOf } from '../snapshot/classify.ts';
|
|
8
9
|
import { maxScrollOffsetOf, measureScrollGeometry, offsetCentering, type ScrollGeometry, scrollOffsetOf, viewportExtentOf } from '../snapshot/scroll.ts';
|
|
9
10
|
import { runBatched } from './batched.ts';
|
|
10
11
|
import { failed, succeeded } from './result.ts';
|
|
@@ -20,6 +21,7 @@ const WALK_STEP_FRACTION = 3;
|
|
|
20
21
|
const WALK_SETTLE_MS = 120;
|
|
21
22
|
const MOUNT_POLL_MS = 150;
|
|
22
23
|
const LISTED_OPTIONS = 30;
|
|
24
|
+
const CANCELLED_MESSAGE = 'Cancelled by the user before the action ran';
|
|
23
25
|
const MOVEMENT_TOLERANCE = 1;
|
|
24
26
|
const SCROLL_ANIMATION_MS = 400;
|
|
25
27
|
|
|
@@ -38,7 +40,9 @@ function commitScrollEnd(list: Fiber, geometry: ScrollGeometry, offset: number):
|
|
|
38
40
|
const { viewport, content, horizontal } = geometry;
|
|
39
41
|
runBatched(() =>
|
|
40
42
|
onMomentumScrollEnd({
|
|
43
|
+
persist: () => undefined,
|
|
41
44
|
nativeEvent: {
|
|
45
|
+
contentInset: { top: 0, left: 0, bottom: 0, right: 0 },
|
|
42
46
|
contentOffset: horizontal ? { x: offset, y: 0 } : { x: 0, y: offset },
|
|
43
47
|
contentSize: { width: content.width, height: content.height },
|
|
44
48
|
layoutMeasurement: { width: viewport.width, height: viewport.height },
|
|
@@ -81,6 +85,9 @@ export async function selectWheelOption(target: ResolvedTarget): Promise<Handler
|
|
|
81
85
|
export function rowsOf(list: Fiber): ListRow[] {
|
|
82
86
|
const rows: ListRow[] = [];
|
|
83
87
|
const visit = (fiber: Fiber): void => {
|
|
88
|
+
if (fiber !== list && scrollInstanceOf(fiber) !== null) {
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
84
91
|
if (isHostComponent(fiber) && fiber.type === TEXT_HOST_TYPE) {
|
|
85
92
|
const text = collectText(fiber, ROW_TEXT_LIMIT);
|
|
86
93
|
if (text.length > 0) {
|
|
@@ -109,7 +116,7 @@ function listOf(target: ResolvedTarget): Fiber | null {
|
|
|
109
116
|
|
|
110
117
|
// A virtualized list mounts its rows shortly after it opens, and a distant row only once scrolling gets near it: wait
|
|
111
118
|
// a moment, then walk the list. A value that isn't there leaves the list where it was.
|
|
112
|
-
export async function selectValue(target: ResolvedTarget, value: string, services: ActionServices): Promise<HandlerResult> {
|
|
119
|
+
export async function selectValue(target: ResolvedTarget, value: string, services: ActionServices, isCancelled: () => boolean = () => false): Promise<HandlerResult> {
|
|
113
120
|
const first = listOf(target);
|
|
114
121
|
if (first === null) {
|
|
115
122
|
return failed('element_not_interactable', 'select works on one open list: click this field to open its picker, then select each value on its list (scr-N)');
|
|
@@ -125,10 +132,13 @@ export async function selectValue(target: ResolvedTarget, value: string, service
|
|
|
125
132
|
return rows.find((row) => matchesRow(row.text, value));
|
|
126
133
|
};
|
|
127
134
|
let row = findRow();
|
|
128
|
-
for (let waited = 0; row === undefined && waited < MOUNT_WAIT_MS; waited += MOUNT_POLL_MS) {
|
|
135
|
+
for (let waited = 0; row === undefined && waited < MOUNT_WAIT_MS && !isCancelled(); waited += MOUNT_POLL_MS) {
|
|
129
136
|
await SYSTEM_CLOCK.sleep(MOUNT_POLL_MS);
|
|
130
137
|
row = findRow();
|
|
131
138
|
}
|
|
139
|
+
if (isCancelled()) {
|
|
140
|
+
return failed('user_cancelled', CANCELLED_MESSAGE);
|
|
141
|
+
}
|
|
132
142
|
if (row !== undefined) {
|
|
133
143
|
return centerInList(current, list, row.host);
|
|
134
144
|
}
|
|
@@ -139,6 +149,10 @@ export async function selectValue(target: ResolvedTarget, value: string, service
|
|
|
139
149
|
const end = maxScrollOffsetOf(geometry);
|
|
140
150
|
const stops = [...walk(start, end, step), ...walk(start, 0, step)];
|
|
141
151
|
for (const stop of stops) {
|
|
152
|
+
if (isCancelled()) {
|
|
153
|
+
await scrollListTo(current, list, start);
|
|
154
|
+
return failed('user_cancelled', CANCELLED_MESSAGE);
|
|
155
|
+
}
|
|
142
156
|
await scrollListTo(current, list, stop);
|
|
143
157
|
await SYSTEM_CLOCK.sleep(WALK_SETTLE_MS);
|
|
144
158
|
row = findRow();
|
|
@@ -25,6 +25,33 @@ function isControlled(target: ResolvedTarget): boolean {
|
|
|
25
25
|
return typeof target.props.value === 'string';
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
const noKeyboard = new Map<NativeInput, boolean>();
|
|
29
|
+
|
|
30
|
+
interface NativeInput {
|
|
31
|
+
setNativeProps?: (props: { showSoftInputOnFocus: boolean }) => void;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Focus without a keyboard: blurring a field to close it would dismiss what depends on its focus, like the suggestions
|
|
35
|
+
// of a search box. The field gets its keyboard back when the run ends.
|
|
36
|
+
function focusWithoutKeyboard(instance: NativeInput & { focus?: () => void }, target: ResolvedTarget): void {
|
|
37
|
+
if (!noKeyboard.has(instance)) {
|
|
38
|
+
noKeyboard.set(instance, target.props.showSoftInputOnFocus !== false);
|
|
39
|
+
instance.setNativeProps?.({ showSoftInputOnFocus: false });
|
|
40
|
+
}
|
|
41
|
+
instance.focus?.();
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function restoreKeyboards(): void {
|
|
45
|
+
noKeyboard.forEach((enabled, instance) => {
|
|
46
|
+
try {
|
|
47
|
+
instance.setNativeProps?.({ showSoftInputOnFocus: enabled });
|
|
48
|
+
} catch {
|
|
49
|
+
// The field is gone: nothing left to restore.
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
noKeyboard.clear();
|
|
53
|
+
}
|
|
54
|
+
|
|
28
55
|
export function typeText(target: ResolvedTarget, action: TypeAction, services: ActionServices): HandlerResult {
|
|
29
56
|
if (target.element.kind !== 'text_input') {
|
|
30
57
|
return failed('element_not_interactable', 'type needs a text input');
|
|
@@ -36,7 +63,9 @@ export function typeText(target: ResolvedTarget, action: TypeAction, services: A
|
|
|
36
63
|
const next = action.clear_first ? action.value : currentTextOf(target) + action.value;
|
|
37
64
|
const instance = target.host === null ? null : hostInstanceOf(target.host);
|
|
38
65
|
try {
|
|
39
|
-
instance
|
|
66
|
+
if (instance !== null) {
|
|
67
|
+
focusWithoutKeyboard(instance as NativeInput & { focus?: () => void }, target);
|
|
68
|
+
}
|
|
40
69
|
if (!isControlled(target)) {
|
|
41
70
|
if (Platform.OS === 'android') {
|
|
42
71
|
const tag = findNodeHandle(instance as Parameters<typeof findNodeHandle>[0]);
|
package/src/agent/compose.ts
CHANGED
|
@@ -20,6 +20,7 @@ export interface AgentEnvironment {
|
|
|
20
20
|
transport: AgentTransport;
|
|
21
21
|
getAnchorFiber: () => Fiber | null;
|
|
22
22
|
getViewport: () => Size;
|
|
23
|
+
getWindow: () => Size;
|
|
23
24
|
getNavigationRef: () => NavigationRefLike | null;
|
|
24
25
|
getCaptureTarget: () => unknown;
|
|
25
26
|
reading: ReadingPort;
|
|
@@ -40,6 +41,7 @@ export function createAgentController(environment: AgentEnvironment): AgentContr
|
|
|
40
41
|
getAnchorFiber: environment.getAnchorFiber,
|
|
41
42
|
registry,
|
|
42
43
|
getViewport: environment.getViewport,
|
|
44
|
+
getWindow: environment.getWindow,
|
|
43
45
|
getExtraRoutes: (): string[] => navigation.listRefRoutePaths(),
|
|
44
46
|
});
|
|
45
47
|
const commitWatch = new CommitWatch(environment.getAnchorFiber, environment.onCommit);
|
package/src/agent/controller.ts
CHANGED
|
@@ -14,6 +14,8 @@ import { handleElementContext } from './handlers/element-context.ts';
|
|
|
14
14
|
import { handleScreenshot } from './handlers/screenshot.ts';
|
|
15
15
|
import { handleTextDom, readTextDom } from './handlers/text-dom.ts';
|
|
16
16
|
|
|
17
|
+
const MESSAGE_TEXT_DOM_MAX_CHARS = 100_000;
|
|
18
|
+
|
|
17
19
|
export interface ControllerDependencies {
|
|
18
20
|
transport: AgentTransport;
|
|
19
21
|
commitWatch: CommitWatch;
|
|
@@ -44,7 +46,7 @@ export class AgentController {
|
|
|
44
46
|
|
|
45
47
|
async readMessageMetadata(options: { includeTextDom: boolean; tabId: string }): Promise<MessageMetadata> {
|
|
46
48
|
const snapshot = await this.context.capturer.capture('peek');
|
|
47
|
-
const textDom = options.includeTextDom ? await readTextDom(this.context, { viewportOnly: false, maxChars:
|
|
49
|
+
const textDom = options.includeTextDom ? await readTextDom(this.context, { viewportOnly: false, maxChars: MESSAGE_TEXT_DOM_MAX_CHARS }) : null;
|
|
48
50
|
return buildMessageMetadata({
|
|
49
51
|
screen: snapshot.screen,
|
|
50
52
|
tabId: options.tabId,
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export interface KeyboardPort {
|
|
2
|
+
addListener: (event: 'keyboardWillShow' | 'keyboardDidShow', listener: () => void) => { remove: () => void };
|
|
3
|
+
dismiss: () => void;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
// Nobody types during a run, and an open keyboard only covers what the agent is doing: an input that focuses itself would
|
|
7
|
+
// otherwise leave it up. Android has no keyboardWillShow, so it hides a beat later. The agent's own typing never opens it
|
|
8
|
+
// (focusWithoutKeyboard); ending the run closes the field it left focused, so the next tap opens a normal keyboard.
|
|
9
|
+
export class KeyboardGuard {
|
|
10
|
+
private subscriptions: Array<{ remove: () => void }> = [];
|
|
11
|
+
|
|
12
|
+
constructor(
|
|
13
|
+
private readonly keyboard: KeyboardPort,
|
|
14
|
+
private readonly onRelease: () => void = () => undefined,
|
|
15
|
+
) {}
|
|
16
|
+
|
|
17
|
+
setActive(active: boolean): void {
|
|
18
|
+
if (!active) {
|
|
19
|
+
if (this.subscriptions.length > 0) {
|
|
20
|
+
this.subscriptions.splice(0).forEach((subscription) => subscription.remove());
|
|
21
|
+
this.keyboard.dismiss();
|
|
22
|
+
}
|
|
23
|
+
this.onRelease();
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
if (this.subscriptions.length > 0) {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
const dismiss = (): void => this.keyboard.dismiss();
|
|
30
|
+
this.subscriptions = [this.keyboard.addListener('keyboardWillShow', dismiss), this.keyboard.addListener('keyboardDidShow', dismiss)];
|
|
31
|
+
dismiss();
|
|
32
|
+
}
|
|
33
|
+
}
|
package/src/chat/ChatSheet.tsx
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import React, { useEffect, useRef } from 'react';
|
|
2
|
-
import { Animated, BackHandler, Dimensions, Easing,
|
|
2
|
+
import { Animated, BackHandler, Dimensions, Easing, Keyboard, Platform, Pressable, StyleSheet, View } from 'react-native';
|
|
3
3
|
|
|
4
4
|
import { useInsets } from '../ui/useInsets.ts';
|
|
5
5
|
import type { UiStrings } from '../ui/strings.ts';
|
|
@@ -16,12 +16,38 @@ export interface ChatSheetProps {
|
|
|
16
16
|
const TOP_GAP = 12;
|
|
17
17
|
const ANIMATION_MS = 240;
|
|
18
18
|
|
|
19
|
+
// iOS lifts the sheet over the keyboard while the chat is open. Not KeyboardAvoidingView: it reacts to every keyboard in
|
|
20
|
+
// the app even with the chat closed, and its LayoutAnimation applies to the host's whole next commit, which aborted a
|
|
21
|
+
// debug build as soon as a keyboard opened while the host mounted views. On Android the window resizes by itself.
|
|
22
|
+
function useKeyboardInset(active: boolean, homeIndicator: number): Animated.Value {
|
|
23
|
+
const inset = useRef(new Animated.Value(0)).current;
|
|
24
|
+
useEffect(() => {
|
|
25
|
+
if (!active || Platform.OS !== 'ios') {
|
|
26
|
+
inset.setValue(0);
|
|
27
|
+
return undefined;
|
|
28
|
+
}
|
|
29
|
+
const subscription = Keyboard.addListener('keyboardWillChangeFrame', (event) => {
|
|
30
|
+
const covered = Dimensions.get('window').height - event.endCoordinates.screenY;
|
|
31
|
+
Animated.timing(inset, {
|
|
32
|
+
// The keyboard covers the home-indicator inset, so the sheet's bottom padding sits under it, not above it.
|
|
33
|
+
toValue: Math.max(0, covered - homeIndicator),
|
|
34
|
+
duration: event.duration,
|
|
35
|
+
easing: Easing.out(Easing.cubic),
|
|
36
|
+
useNativeDriver: false,
|
|
37
|
+
}).start();
|
|
38
|
+
});
|
|
39
|
+
return () => subscription.remove();
|
|
40
|
+
}, [active, homeIndicator, inset]);
|
|
41
|
+
return inset;
|
|
42
|
+
}
|
|
43
|
+
|
|
19
44
|
export function ChatSheet({ mode, mounted, strings, onClose, children }: ChatSheetProps): React.JSX.Element | null {
|
|
20
45
|
const insets = useInsets();
|
|
21
46
|
const windowHeight = Dimensions.get('window').height;
|
|
22
47
|
const sheetHeight = windowHeight - insets.top - TOP_GAP;
|
|
23
48
|
const progress = useRef(new Animated.Value(mode === 'open' ? 1 : 0)).current;
|
|
24
49
|
const isOpen = mode === 'open';
|
|
50
|
+
const keyboardInset = useKeyboardInset(isOpen, insets.bottom);
|
|
25
51
|
|
|
26
52
|
useEffect(() => {
|
|
27
53
|
Animated.timing(progress, { toValue: isOpen ? 1 : 0, duration: ANIMATION_MS, easing: Easing.out(Easing.cubic), useNativeDriver: true }).start();
|
|
@@ -48,13 +74,7 @@ export function ChatSheet({ mode, mounted, strings, onClose, children }: ChatShe
|
|
|
48
74
|
<Animated.View style={[styles.backdrop, { opacity: backdropOpacity }]}>
|
|
49
75
|
<Pressable accessibilityLabel={strings.chatClose} style={StyleSheet.absoluteFill} onPress={onClose} />
|
|
50
76
|
</Animated.View>
|
|
51
|
-
<
|
|
52
|
-
style={[styles.container, { paddingTop: insets.top + TOP_GAP }]}
|
|
53
|
-
behavior={Platform.OS === 'ios' ? 'padding' : undefined}
|
|
54
|
-
// The keyboard covers the home-indicator inset, so the sheet's bottom padding sits under it, not above it.
|
|
55
|
-
keyboardVerticalOffset={-insets.bottom}
|
|
56
|
-
pointerEvents="box-none"
|
|
57
|
-
>
|
|
77
|
+
<Animated.View style={[styles.container, { paddingTop: insets.top + TOP_GAP, paddingBottom: keyboardInset }]} pointerEvents="box-none">
|
|
58
78
|
<Animated.View
|
|
59
79
|
accessibilityElementsHidden={!isOpen}
|
|
60
80
|
importantForAccessibility={isOpen ? 'yes' : 'no-hide-descendants'}
|
|
@@ -62,7 +82,7 @@ export function ChatSheet({ mode, mounted, strings, onClose, children }: ChatShe
|
|
|
62
82
|
>
|
|
63
83
|
{children}
|
|
64
84
|
</Animated.View>
|
|
65
|
-
</
|
|
85
|
+
</Animated.View>
|
|
66
86
|
</View>
|
|
67
87
|
);
|
|
68
88
|
}
|
package/src/chat/ChatView.tsx
CHANGED
|
@@ -107,7 +107,7 @@ export function ChatView({ shell, unavailableText, getMetadata, getProof, onClos
|
|
|
107
107
|
mixedContentMode="always"
|
|
108
108
|
setSupportMultipleWindows={false}
|
|
109
109
|
keyboardDisplayRequiresUserAction={false}
|
|
110
|
-
// The sheet
|
|
110
|
+
// The sheet already lifts the chat above the keyboard. WKWebView would also scroll its
|
|
111
111
|
// own page up to reveal the composer, pushing the widget out of the sheet; with scrolling off it resets that
|
|
112
112
|
// offset (the message list is an overflow container and keeps scrolling). The form bar (^ v Done) is for
|
|
113
113
|
// multi-field forms, not a one-field composer.
|