guideai-app 0.5.0 → 0.5.1-2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/GuideAI.js +1 -1
  2. package/dist/GuideAI.js.map +1 -1
  3. package/dist/__mocks__/fileMock.d.ts +2 -0
  4. package/dist/components/AnimatedSettingsItem.d.ts +1 -1
  5. package/dist/components/AudioConfigDialog.d.ts +20 -0
  6. package/dist/components/ConfigButton.d.ts +10 -0
  7. package/dist/components/Microphone.d.ts +3 -1
  8. package/dist/components/SettingsMenu.d.ts +4 -1
  9. package/dist/components/TranscriptBox.d.ts +7 -1
  10. package/dist/components/TranscriptTextInput.d.ts +4 -1
  11. package/dist/components/TranscriptToggle.d.ts +1 -1
  12. package/dist/hooks/useGuideAIAPI.d.ts +1 -0
  13. package/dist/hooks/useWebRTC.d.ts +16 -4
  14. package/dist/types/GuideAI.types.d.ts +10 -1
  15. package/dist/types/audioConfig.types.d.ts +71 -0
  16. package/dist/utils/api.d.ts +19 -8
  17. package/dist/utils/clickElement.d.ts +7 -0
  18. package/dist/utils/constants.d.ts +0 -1
  19. package/dist/utils/dataChannel.d.ts +29 -4
  20. package/dist/utils/dataChannelHelpers.d.ts +28 -0
  21. package/dist/utils/elementInteractions.d.ts +110 -21
  22. package/dist/utils/fillField.d.ts +2 -0
  23. package/dist/utils/goToAElmLink.d.ts +6 -1
  24. package/dist/utils/highlight.d.ts +2 -0
  25. package/dist/utils/highlightThenClick.d.ts +6 -2
  26. package/dist/utils/hover.d.ts +1 -1
  27. package/dist/utils/hoverThenClick.d.ts +5 -1
  28. package/dist/utils/localStorageHelper.d.ts +18 -0
  29. package/dist/utils/logger.d.ts +8 -8
  30. package/dist/utils/navigateTo.d.ts +2 -0
  31. package/dist/utils/reactHooks.d.ts +13 -0
  32. package/dist/utils/toolRegistry.d.ts +27 -0
  33. package/dist/utils/webrtcConnection.d.ts +6 -1
  34. package/dist/visualContext/debug-overlay.d.ts +1 -1
  35. package/dist/visualContext/index.d.ts +1 -1
  36. package/package.json +8 -2
@@ -0,0 +1,2 @@
1
+ declare const _exports: string;
2
+ export = _exports;
@@ -5,5 +5,5 @@ interface AnimatedSettingsItemProps {
5
5
  children: React.ReactNode;
6
6
  React: typeof import('react');
7
7
  }
8
- declare const AnimatedSettingsItem: ({ visible, gap, isLast, children, React }: AnimatedSettingsItemProps) => import("react").JSX.Element;
8
+ declare const AnimatedSettingsItem: ({ visible, gap, isLast, children, React, }: AnimatedSettingsItemProps) => import("react").JSX.Element;
9
9
  export default AnimatedSettingsItem;
@@ -0,0 +1,20 @@
1
+ import { AudioConfig, StoredAudioConfig } from '../types/audioConfig.types';
2
+ interface AudioConfigDialogProps {
3
+ isOpen: boolean;
4
+ onClose: () => void;
5
+ initialConfig: StoredAudioConfig | null;
6
+ onApply: (config: AudioConfig, stored: StoredAudioConfig) => void;
7
+ hasRespondedWithAudio: boolean;
8
+ defaultMicLabel: string | null;
9
+ React: typeof import('react');
10
+ ReactDOM?: {
11
+ createPortal: (children: React.ReactNode, container: Element) => React.ReactNode;
12
+ };
13
+ }
14
+ declare const AudioConfigDialog: {
15
+ ({ isOpen, onClose, initialConfig, onApply, hasRespondedWithAudio, defaultMicLabel, React, ReactDOM, }: AudioConfigDialogProps): import("react").FunctionComponentElement<{
16
+ children?: import("react").ReactNode | undefined;
17
+ }> | null;
18
+ displayName: string;
19
+ };
20
+ export default AudioConfigDialog;
@@ -0,0 +1,10 @@
1
+ interface ConfigButtonProps {
2
+ onOpenConfig: () => void;
3
+ hasMic: boolean;
4
+ React: typeof import('react');
5
+ }
6
+ declare const ConfigButton: {
7
+ ({ onOpenConfig, hasMic, React }: ConfigButtonProps): import("react").JSX.Element;
8
+ displayName: string;
9
+ };
10
+ export default ConfigButton;
@@ -8,6 +8,8 @@ interface MicrophoneProps {
8
8
  handleOnboardingComplete: () => void;
9
9
  handleOnboardingClose: () => void;
10
10
  isConnecting: boolean;
11
+ /** Message shown when connecting (e.g. "Initializing..." or "Retrying...") */
12
+ connectionMessage?: string | null;
11
13
  status: RecordingStatus;
12
14
  handleToggleConversation: () => void;
13
15
  showTranscript: boolean;
@@ -15,5 +17,5 @@ interface MicrophoneProps {
15
17
  isMuted: boolean;
16
18
  React: typeof import('react');
17
19
  }
18
- declare const Microphone: ({ componentRef, baseStyles, hasInteracted, popupPosition, showOnboarding, handleOnboardingComplete, handleOnboardingClose, isConnecting, status, handleToggleConversation, showTranscript, hasMic, isMuted, React }: MicrophoneProps) => import("react").JSX.Element;
20
+ declare const Microphone: ({ componentRef, baseStyles, hasInteracted, popupPosition, showOnboarding, handleOnboardingComplete, handleOnboardingClose, isConnecting, connectionMessage, status, handleToggleConversation, showTranscript, hasMic, isMuted, React, }: MicrophoneProps) => import("react").JSX.Element;
19
21
  export default Microphone;
@@ -10,7 +10,10 @@ interface SettingsMenuProps {
10
10
  setShowSettings?: (value: boolean) => void;
11
11
  transcriptPosition: TranscriptPosition;
12
12
  micPosition?: MicButtonPosition;
13
+ showConfigButton?: boolean;
14
+ onOpenConfig?: () => void;
15
+ isConfigDialogOpen?: boolean;
13
16
  React: typeof import('react');
14
17
  }
15
- declare const SettingsMenu: ({ handleResetChat, hasMic, isMuted, handleToggleMute, showTranscript, handleToggleTranscript, showSettings, setShowSettings, transcriptPosition, micPosition, React }: SettingsMenuProps) => import("react").JSX.Element;
18
+ declare const SettingsMenu: ({ handleResetChat, hasMic, isMuted, handleToggleMute, showTranscript, handleToggleTranscript, showSettings, setShowSettings, transcriptPosition, micPosition, showConfigButton, onOpenConfig, isConfigDialogOpen, React, }: SettingsMenuProps) => import("react").JSX.Element;
16
19
  export default SettingsMenu;
@@ -12,6 +12,8 @@ interface TranscriptBoxProps {
12
12
  handleTextSubmit?: () => void;
13
13
  handleTextKeyPress?: (event: React.KeyboardEvent) => void;
14
14
  textPlaceholder?: string;
15
+ /** Connection status message (e.g. "Initializing...", "Retrying...") */
16
+ connectionMessage?: string | null;
15
17
  micPosition?: MicButtonPosition;
16
18
  transcriptPosition: TranscriptPosition;
17
19
  hasMic: boolean;
@@ -22,10 +24,14 @@ interface TranscriptBoxProps {
22
24
  showSettings?: boolean;
23
25
  setShowSettings?: (value: boolean) => void;
24
26
  isDataChannelOpen?: boolean;
27
+ isResponseActive?: boolean;
28
+ showConfigButton?: boolean;
29
+ onOpenConfig?: () => void;
30
+ isConfigDialogOpen?: boolean;
25
31
  React: typeof import('react');
26
32
  }
27
33
  declare const TranscriptBox: {
28
- ({ allMessages, showTranscript, handleCloseTranscript, handleToggleTranscript, showTextInput, textInput, setTextInput, handleTextSubmit, handleTextKeyPress, textPlaceholder, micPosition, transcriptPosition, hasMic, isMuted, handleToggleMute, isConversationActive, handleResetChat, showSettings, setShowSettings, isDataChannelOpen, React }: TranscriptBoxProps): import("react").JSX.Element | null;
34
+ ({ allMessages, showTranscript, handleCloseTranscript: _handleCloseTranscript, handleToggleTranscript, showTextInput, textInput, setTextInput, handleTextSubmit, handleTextKeyPress, textPlaceholder, connectionMessage, micPosition, transcriptPosition, hasMic, isMuted, handleToggleMute, isConversationActive: _isConversationActive, handleResetChat, showSettings, setShowSettings, isDataChannelOpen, isResponseActive, showConfigButton, onOpenConfig, isConfigDialogOpen, React, }: TranscriptBoxProps): import("react").JSX.Element | null;
29
35
  displayName: string;
30
36
  };
31
37
  export default TranscriptBox;
@@ -4,8 +4,11 @@ interface TranscriptTextInputProps {
4
4
  handleTextSubmit?: () => void;
5
5
  handleTextKeyPress?: (event: React.KeyboardEvent) => void;
6
6
  textPlaceholder?: string;
7
+ /** Connection status message (e.g. "Initializing...", "Retrying...") - shown when channel not open */
8
+ connectionMessage?: string | null;
7
9
  isDataChannelOpen?: boolean;
10
+ isResponseActive?: boolean;
8
11
  React: typeof import('react');
9
12
  }
10
- declare const TranscriptTextInput: ({ textInput, setTextInput, handleTextSubmit, handleTextKeyPress, textPlaceholder, isDataChannelOpen, React }: TranscriptTextInputProps) => import("react").JSX.Element;
13
+ declare const TranscriptTextInput: ({ textInput, setTextInput, handleTextSubmit, handleTextKeyPress, textPlaceholder, connectionMessage, isDataChannelOpen, isResponseActive, React, }: TranscriptTextInputProps) => import("react").JSX.Element;
11
14
  export default TranscriptTextInput;
@@ -3,5 +3,5 @@ interface TranscriptToggleProps {
3
3
  handleToggleTranscript?: () => void;
4
4
  React: typeof import('react');
5
5
  }
6
- declare const TranscriptToggle: ({ showTranscript, handleToggleTranscript, React }: TranscriptToggleProps) => import("react").JSX.Element;
6
+ declare const TranscriptToggle: ({ showTranscript, handleToggleTranscript, React, }: TranscriptToggleProps) => import("react").JSX.Element;
7
7
  export default TranscriptToggle;
@@ -18,5 +18,6 @@ export interface UseGuideAIAPIParams {
18
18
  handleToggleInputMode: () => void;
19
19
  handleTextSubmit: (text?: string) => Promise<void>;
20
20
  visualContextWindowAPI: any;
21
+ setShowConfigButton: (value: boolean) => void;
21
22
  }
22
23
  export declare function useGuideAIAPI(params: UseGuideAIAPIParams): void;
@@ -8,6 +8,11 @@ import { ConversationManager } from '../utils/conversationManager';
8
8
  import type { RecordingStatus } from '../types/GuideAI.types';
9
9
  import type { GuideAIEnvironment } from '../types/GuideAI.types';
10
10
  import type { StoredMessage } from '../utils/messageStorage';
11
+ export type PropTools = Record<string, {
12
+ description: string;
13
+ parameters: object;
14
+ execute: (args: Record<string, unknown>) => Promise<unknown>;
15
+ }>;
11
16
  export interface UseWebRTCParams {
12
17
  hooks: {
13
18
  useRef: <T>(initialValue: T) => {
@@ -18,12 +23,9 @@ export interface UseWebRTCParams {
18
23
  organizationKey: string;
19
24
  environment: GuideAIEnvironment;
20
25
  setIsDataChannelOpen: (open: boolean) => void;
26
+ setIsResponseActive: (active: boolean) => void;
21
27
  setIsConnecting: (connecting: boolean) => void;
22
28
  setStatus: (status: RecordingStatus) => void;
23
- handleHighlightThenClick: (out: any) => Promise<boolean>;
24
- handleHoverThenClick: (out: any) => Promise<boolean>;
25
- handleHover: (out: any) => Promise<boolean>;
26
- handleGoToAElmLink: (linkText: string, delay?: number) => Promise<boolean>;
27
29
  handleResetChatRef: {
28
30
  current: () => Promise<void>;
29
31
  };
@@ -38,6 +40,16 @@ export interface UseWebRTCParams {
38
40
  current: ConversationManager | null;
39
41
  };
40
42
  onMessageLogged: () => void;
43
+ /** Optional tools from component prop: Record<functionName, { description, parameters, execute }> */
44
+ tools?: PropTools;
45
+ onSessionUpdated?: (session: any) => void;
46
+ onAudioOutputStarted?: () => void;
47
+ /** Set connection message (e.g. "Retrying...") during init retries */
48
+ setConnectionMessage: (msg: string | null) => void;
49
+ /** Called when initialize fails after all retries */
50
+ onInitializeError?: (error: {
51
+ message: string;
52
+ }) => void;
41
53
  }
42
54
  export declare function useWebRTC(params: UseWebRTCParams): {
43
55
  peerConnectionRef: {
@@ -22,7 +22,6 @@ export type GuideAIPosition = {
22
22
  export type MicButtonPosition = GuideAIPosition;
23
23
  export interface GuideAIProps {
24
24
  organizationKey: string;
25
- workflowKey?: string;
26
25
  environment?: GuideAIEnvironment;
27
26
  React: typeof import('react');
28
27
  ReactDOM: any;
@@ -38,6 +37,16 @@ export interface GuideAIProps {
38
37
  placeholder?: string;
39
38
  };
40
39
  visualContext?: VisualContextConfig;
40
+ /** Custom tools: Record<functionName, { description, parameters, execute }> - always custom handler */
41
+ tools?: Record<string, {
42
+ description: string;
43
+ parameters: object;
44
+ execute: (args: Record<string, unknown>) => Promise<unknown>;
45
+ }>;
46
+ /** Called when initialize fails after all retries. Host can show toast. */
47
+ onInitializeError?: (error: {
48
+ message: string;
49
+ }) => void;
41
50
  }
42
51
  export type TranscriptPosition = 'right' | 'left';
43
52
  export type PopupPosition = 'above' | 'below' | 'above-left' | 'above-right' | 'above-center' | 'below-left' | 'below-right' | 'below-center';
@@ -0,0 +1,71 @@
1
+ export interface ServerVadConfig {
2
+ type: 'server_vad';
3
+ create_response?: boolean;
4
+ interrupt_response?: boolean;
5
+ threshold?: number;
6
+ silence_duration_ms?: number | undefined;
7
+ prefix_padding_ms?: number | undefined;
8
+ idle_timeout_ms?: number | null;
9
+ }
10
+ export interface SemanticVadConfig {
11
+ type: 'semantic_vad';
12
+ create_response?: boolean;
13
+ interrupt_response?: boolean;
14
+ eagerness?: 'low' | 'medium' | 'high' | 'auto';
15
+ }
16
+ export type TurnDetectionConfig = ServerVadConfig | SemanticVadConfig;
17
+ export type NoiseReductionType = 'near_field' | 'far_field';
18
+ export interface NoiseReductionConfig {
19
+ type: NoiseReductionType;
20
+ }
21
+ export interface AudioConfigInput {
22
+ noise_reduction?: NoiseReductionConfig | null;
23
+ turn_detection?: TurnDetectionConfig | null;
24
+ }
25
+ export interface AudioConfigOutput {
26
+ speed?: number;
27
+ voice?: string;
28
+ }
29
+ export interface AudioConfig {
30
+ input?: AudioConfigInput;
31
+ output?: AudioConfigOutput;
32
+ }
33
+ /**
34
+ * Tracks which fields the user has customized (unchecked "Default").
35
+ * If a key is absent or true, that field uses the hardcoded default.
36
+ */
37
+ export interface AudioConfigDefaults {
38
+ noise_reduction?: boolean;
39
+ vad_type?: boolean;
40
+ create_response?: boolean;
41
+ interrupt_response?: boolean;
42
+ threshold?: boolean;
43
+ silence_duration_ms?: boolean;
44
+ prefix_padding_ms?: boolean;
45
+ idle_timeout_ms?: boolean;
46
+ eagerness?: boolean;
47
+ speed?: boolean;
48
+ voice?: boolean;
49
+ }
50
+ export interface StoredAudioConfig {
51
+ config: AudioConfig;
52
+ useDefaults: AudioConfigDefaults;
53
+ }
54
+ export declare const VOICES: readonly ["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse", "marin", "cedar"];
55
+ export type VoiceName = (typeof VOICES)[number];
56
+ export declare const DEFAULT_AUDIO_CONFIG: AudioConfig;
57
+ export declare const DEFAULT_SERVER_VAD: ServerVadConfig;
58
+ export declare const DEFAULT_SEMANTIC_VAD: SemanticVadConfig;
59
+ export declare const ALL_DEFAULTS_TRUE: AudioConfigDefaults;
60
+ export declare const FIELD_INFO: Record<string, string>;
61
+ export declare function mergeWithDefaults(stored: StoredAudioConfig | null): AudioConfig;
62
+ /**
63
+ * Builds the audio payload for session.update from an AudioConfig.
64
+ * All types match the API shape directly -- no conversion needed.
65
+ */
66
+ export declare function buildAudioSessionPayload(config: AudioConfig): Record<string, any>;
67
+ export interface ValidationError {
68
+ field: string;
69
+ message: string;
70
+ }
71
+ export declare function validateAudioConfig(config: AudioConfig): ValidationError[];
@@ -1,15 +1,10 @@
1
1
  import { GuideAIEnvironment } from '../types/GuideAI.types';
2
2
  import { MessageData } from './conversationManager';
3
3
  export declare function getApiHeaders(): Record<string, string>;
4
- export interface Workflow {
5
- id: string;
4
+ export interface PropToolDefinition {
6
5
  name: string;
7
- triggers: string[];
8
- organizationKey: string;
9
- prompt: string;
10
- archived?: boolean;
11
- createdAt?: string;
12
- updatedAt?: string;
6
+ description: string;
7
+ parameters: object;
13
8
  }
14
9
  export interface InitializeWebRTCPayload {
15
10
  sdp: string;
@@ -17,9 +12,25 @@ export interface InitializeWebRTCPayload {
17
12
  user?: string;
18
13
  conversationId?: string;
19
14
  }
15
+ export interface ToolFromServer {
16
+ toolConfig: object;
17
+ functionConfig: object;
18
+ }
20
19
  export interface InitializeWebRTCResponse {
21
20
  answerSdp: string;
22
21
  id: string;
22
+ tools?: ToolFromServer[];
23
23
  }
24
24
  export declare const initializeWebRTC: (endpoint: string, payload: InitializeWebRTCPayload) => Promise<InitializeWebRTCResponse>;
25
+ /** Fire-and-forget: register prop tools (customer-scoped) and built-in tools (global) not in registry. Does not await. */
26
+ export declare const registerTools: (endpoint: string, payload: {
27
+ organizationKey: string;
28
+ propTools?: PropToolDefinition[];
29
+ builtInTools?: PropToolDefinition[];
30
+ }) => void;
31
+ /** Built-in tool definition for server registration */
32
+ export interface BuiltInToolDefinition {
33
+ toolConfig: object;
34
+ functionConfig: object;
35
+ }
25
36
  export declare const logMessage: (messageData: MessageData, conversationId: string | null, organizationKey: string, user?: string, environment?: GuideAIEnvironment) => Promise<void>;
@@ -0,0 +1,7 @@
1
+ import { ToolInput } from './elementInteractions';
2
+ /**
3
+ * Perform a single click on an element (used by click tool and by highlightThenClick/hoverThenClick).
4
+ * Resolves clickable subelement, dispatches click, optionally shows ripple/emphasis.
5
+ */
6
+ export declare function performClick(element: Element, rect: DOMRect, withEffect: boolean, cursor: HTMLElement | null, interactionTime?: number): Promise<void>;
7
+ export declare const clickElement: (out: ToolInput) => Promise<boolean>;
@@ -5,7 +5,6 @@ export declare const GUIDE_AI_API_BASE_LOCAL = "http://localhost:3000/api";
5
5
  export declare const getApiBaseUrl: (environment?: "development" | "production" | "local") => string;
6
6
  export declare const GUIDE_AI_API_BASE = "https://app.getguide.ai/api";
7
7
  export declare const OPENAI_REALTIME_BASE = "https://api.openai.com/v1/realtime/calls";
8
- export type ElementInteractionType = "HIGHLIGHT" | "HOVERANDCLICK";
9
8
  export declare const UI_LAYOUT: {
10
9
  /** Mic/icon wrapper size - matches .guideai-icon-wrapper */
11
10
  readonly micButtonSize: 50;
@@ -1,18 +1,17 @@
1
1
  import { StoredMessage } from './messageStorage';
2
+ import { type ToolWithConfig } from './toolRegistry';
2
3
  import { ConversationManager } from './conversationManager';
4
+ import { AudioConfig } from '../types/audioConfig.types';
3
5
  export type RecordingStatus = 'idle' | 'recording' | 'active' | 'processing' | 'playing';
4
6
  export declare const IGNORE_MESSAGE_TYPES: string[];
5
7
  export interface DataChannelCallbacks {
6
8
  setIsDataChannelOpen: (open: boolean) => void;
7
9
  setIsConnecting: (connecting: boolean) => void;
8
10
  setStatus: (status: RecordingStatus) => void;
9
- handleHighlightThenClick: (out: any) => Promise<boolean>;
10
- handleHoverThenClick: (out: any) => Promise<boolean>;
11
- handleHover: (out: any) => Promise<boolean>;
12
- handleGoToAElmLink: (linkText: string, delay?: number) => Promise<boolean>;
13
11
  onDataChannelClose: (pc: RTCPeerConnection) => void;
14
12
  onDataChannelError: (event: Event) => void;
15
13
  onDataChannelMessageError: (message: any) => void;
14
+ setIsResponseActive: (active: boolean) => void;
16
15
  updateStreamingMessage: (delta: string) => void;
17
16
  /** Reset streaming state when a new response starts (call on response.created) */
18
17
  resetStreamingMessage?: () => void;
@@ -24,10 +23,20 @@ export interface DataChannelCallbacks {
24
23
  current: ConversationManager | null;
25
24
  };
26
25
  updateSessionId?: (sessionId: string | null) => void;
26
+ onSessionUpdated?: (session: any) => void;
27
+ onAudioOutputStarted?: () => void;
27
28
  }
28
29
  export interface DataChannelConfig {
29
30
  audioStream: MediaStream | null;
30
31
  callbacks: DataChannelCallbacks;
32
+ /** Tools from server (initialize response). If undefined = old server, use built-in. If [] = new server, no tools. */
33
+ tools?: ToolWithConfig[];
34
+ /** Prop tools: Record<functionName, { description, parameters, execute }> - always custom handler */
35
+ propTools?: Record<string, {
36
+ description: string;
37
+ parameters: object;
38
+ execute: (args: Record<string, unknown>) => Promise<unknown>;
39
+ }>;
31
40
  }
32
41
  export declare const BUFFERED_AMOUNT_THRESHOLD = 65536;
33
42
  export declare class DataChannelManager {
@@ -36,7 +45,12 @@ export declare class DataChannelManager {
36
45
  private callbacks;
37
46
  private audioStream;
38
47
  private sendQueue;
48
+ private toolsConfig;
49
+ private serverToolsReceived;
50
+ private propTools?;
39
51
  constructor(config: DataChannelConfig);
52
+ /** Set tools from server (call after initialize, before channel opens) */
53
+ setTools(tools: ToolWithConfig[]): void;
40
54
  /**
41
55
  * Create a new data channel on the peer connection
42
56
  */
@@ -66,6 +80,17 @@ export declare class DataChannelManager {
66
80
  * Check if the data channel is open and ready
67
81
  */
68
82
  isOpen(): boolean;
83
+ /**
84
+ * Build an audio session payload by merging hardcoded defaults with localStorage.
85
+ * Converts internal types to the API format (e.g. noise_reduction string -> object).
86
+ */
87
+ private buildMergedAudioPayload;
88
+ /**
89
+ * Send a session.update with an AudioConfig payload.
90
+ * Converts internal types to API format via buildAudioSessionPayload.
91
+ * Used by the config dialog Apply and Reset buttons.
92
+ */
93
+ sendAudioSessionUpdate(config: AudioConfig): void;
69
94
  /**
70
95
  * Get the current buffered amount in bytes
71
96
  */
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Helper functions for data channel message handling.
3
+ * Extracted from dataChannel.ts to keep the main module focused on orchestration.
4
+ */
5
+ /** Check if message is a conversation.item.create with input_text */
6
+ export declare function hasInputText(message: unknown): boolean;
7
+ /** Check if message is image-only (no text) */
8
+ export declare function isImageOnlyMessage(message: unknown): boolean;
9
+ /** Check if payload string is an image-only message */
10
+ export declare function isImageOnlyPayload(payload: string): boolean;
11
+ /** Persist server-accepted audio config from session.updated to localStorage */
12
+ export declare function persistAudioConfigFromSession(session: {
13
+ audio?: {
14
+ input?: {
15
+ noise_reduction?: unknown;
16
+ turn_detection?: unknown;
17
+ };
18
+ output?: {
19
+ speed?: unknown;
20
+ voice?: unknown;
21
+ };
22
+ };
23
+ }): void;
24
+ /** Build conversation context message to send after session.updated, or null if no stored messages */
25
+ export declare function buildConversationContextMessage(organizationKey: string): {
26
+ type: string;
27
+ item: object;
28
+ } | null;
@@ -1,11 +1,11 @@
1
- import { ElementInteractionType } from './constants';
2
1
  export interface ToolInput {
3
2
  arguments: string;
4
3
  name?: string;
5
4
  }
6
- export declare const HOVER_EFFECT_KEEP_ALIVE_TIME = 3000;
5
+ export declare const HOVER_EFFECT_KEEP_ALIVE_TIME = 1500;
6
+ export declare const getLastNavigateTimestamp: () => number | null;
7
+ export declare const waitForElement: (selector: string, timeout?: number, interval?: number) => Promise<Element | null>;
7
8
  export declare const createHoverEffect: (element: Element, initialRect: DOMRect, cursorElement?: HTMLElement | null, duration?: number) => Promise<void>;
8
- export declare const clickElement: (element: Element, rect: DOMRect, withEffects?: boolean, cursorElement?: HTMLElement | null) => Promise<void>;
9
9
  export declare const parseCustomSelector: (selector: string) => {
10
10
  type: "text" | "css" | "xpath";
11
11
  cssSelector?: string;
@@ -14,10 +14,79 @@ export declare const parseCustomSelector: (selector: string) => {
14
14
  };
15
15
  export declare const findElementBySelectorStrict: (selector: string) => Element | null;
16
16
  export declare const findElementBySelector: (selector: string) => Element | null;
17
- export declare const interactWithElement: (element: Element, cursorElement: HTMLElement | null, state: ElementInteractionType, hoverTime?: number) => Promise<HTMLElement | null>;
18
- export declare const processSelectorsInteraction: (selectors: string | string[], isInteracting: boolean, setIsInteracting: (interacting: boolean) => void, onEachElement: (element: Element, cursor: HTMLElement | null, index: number) => Promise<HTMLElement | null>, onComplete?: (lastElement: Element | null, cursor: HTMLElement | null) => Promise<void>) => Promise<boolean>;
17
+ export declare function getEffectiveZIndex(element: Element | null): number;
18
+ export declare const moveCursorToElement: (element: Element, cursorElement: HTMLElement | null) => Promise<HTMLElement | null>;
19
+ type InteractionJob = () => Promise<void>;
20
+ export declare const enqueueInteractionJob: (job: InteractionJob) => void;
19
21
  export declare const parseArgs: (argsToUse: string) => Promise<any>;
20
- export declare const Highlight_Click_Tool: {
22
+ export declare const Highlight_Tool: {
23
+ type: string;
24
+ name: string;
25
+ description: string;
26
+ parameters: {
27
+ type: string;
28
+ properties: {
29
+ selector: {
30
+ type: string;
31
+ description: string;
32
+ };
33
+ interactionTime: {
34
+ type: string;
35
+ description: string;
36
+ minimum: number;
37
+ maximum: number;
38
+ };
39
+ };
40
+ required: string[];
41
+ };
42
+ };
43
+ export declare const Click_Tool: {
44
+ type: string;
45
+ name: string;
46
+ description: string;
47
+ parameters: {
48
+ type: string;
49
+ properties: {
50
+ selector: {
51
+ type: string;
52
+ description: string;
53
+ };
54
+ withEffect: {
55
+ type: string;
56
+ description: string;
57
+ };
58
+ interactionTime: {
59
+ type: string;
60
+ description: string;
61
+ minimum: number;
62
+ maximum: number;
63
+ };
64
+ };
65
+ required: string[];
66
+ };
67
+ };
68
+ export declare const Hover_Tool: {
69
+ type: string;
70
+ name: string;
71
+ description: string;
72
+ parameters: {
73
+ type: string;
74
+ properties: {
75
+ selector: {
76
+ type: string;
77
+ description: string;
78
+ };
79
+ interactionTime: {
80
+ type: string;
81
+ description: string;
82
+ minimum: number;
83
+ maximum: number;
84
+ };
85
+ };
86
+ required: string[];
87
+ };
88
+ };
89
+ export declare const Highlight_Then_Click_Tool: {
21
90
  type: string;
22
91
  name: string;
23
92
  description: string;
@@ -42,7 +111,7 @@ export declare const Highlight_Click_Tool: {
42
111
  required: string[];
43
112
  };
44
113
  };
45
- export declare const Hover_Click_Tool: {
114
+ export declare const Hover_Then_Click_Tool: {
46
115
  type: string;
47
116
  name: string;
48
117
  description: string;
@@ -73,28 +142,18 @@ export declare const Hover_Click_Tool: {
73
142
  required: string[];
74
143
  };
75
144
  };
76
- export declare const Hover_Tool: {
145
+ export declare const Navigate_To_Tool: {
77
146
  type: string;
78
147
  name: string;
79
148
  description: string;
80
149
  parameters: {
81
150
  type: string;
82
151
  properties: {
83
- selector: {
84
- oneOf: ({
85
- type: string;
86
- description: string;
87
- items?: undefined;
88
- } | {
89
- type: string;
90
- items: {
91
- type: string;
92
- };
93
- description: string;
94
- })[];
152
+ url: {
153
+ type: string;
95
154
  description: string;
96
155
  };
97
- hoverTime: {
156
+ delay: {
98
157
  type: string;
99
158
  description: string;
100
159
  minimum: number;
@@ -125,3 +184,33 @@ export declare const Go_To_A_Elm_Link_Tool: {
125
184
  required: string[];
126
185
  };
127
186
  };
187
+ export declare const Fill_Field_Tool: {
188
+ type: string;
189
+ name: string;
190
+ description: string;
191
+ parameters: {
192
+ type: string;
193
+ properties: {
194
+ fieldName: {
195
+ type: string;
196
+ description: string;
197
+ };
198
+ selector: {
199
+ type: string;
200
+ description: string;
201
+ };
202
+ value: {
203
+ type: string;
204
+ description: string;
205
+ };
206
+ delay: {
207
+ type: string;
208
+ description: string;
209
+ minimum: number;
210
+ maximum: number;
211
+ };
212
+ };
213
+ required: string[];
214
+ };
215
+ };
216
+ export {};
@@ -0,0 +1,2 @@
1
+ import { ToolInput } from './elementInteractions';
2
+ export declare const fillField: (out: ToolInput) => Promise<boolean>;
@@ -1 +1,6 @@
1
- export declare const goToAElmLink: (linkText: string, delay?: number) => Promise<boolean>;
1
+ import { ToolInput } from './elementInteractions';
2
+ /**
3
+ * Tool entry for goToAElmLink. Parses linkText and delay from args and enqueues the navigation job.
4
+ * @deprecated Prefer navigateTo. This tool is for legacy Momentum-style markup (span text + data-momentum-client-link).
5
+ */
6
+ export declare const goToAElmLink: (out: ToolInput) => Promise<boolean>;
@@ -0,0 +1,2 @@
1
+ import { ToolInput } from './elementInteractions';
2
+ export declare const highlight: (out: ToolInput) => Promise<boolean>;
@@ -1,2 +1,6 @@
1
- import { ToolInput } from "./elementInteractions";
2
- export declare const highlightThenClick: (out: ToolInput, isHighlighting: boolean, setIsHighlighting: (highlighting: boolean) => void) => Promise<boolean>;
1
+ import { ToolInput } from './elementInteractions';
2
+ /**
3
+ * Highlight (move cursor to) each element then click it. Supports a single selector or array of selectors.
4
+ * Same behavior as legacy: cursor moves to each element in sequence and clicks with visual effect.
5
+ */
6
+ export declare const highlightThenClick: (out: ToolInput) => Promise<boolean>;
@@ -1,2 +1,2 @@
1
1
  import { ToolInput } from './elementInteractions';
2
- export declare const hover: (out: ToolInput, isHovering: boolean, setIsHovering: (hovering: boolean) => void) => Promise<boolean>;
2
+ export declare const hover: (out: ToolInput) => Promise<boolean>;
@@ -1,2 +1,6 @@
1
1
  import { ToolInput } from './elementInteractions';
2
- export declare const hoverThenClick: (out: ToolInput, isHovering: boolean, setIsHovering: (hovering: boolean) => void) => Promise<boolean>;
2
+ /**
3
+ * Hover over each element (with hover effect), then click the last element. Supports single selector or array.
4
+ * Same behavior as legacy: cursor moves to each, hover effect runs, then last element is clicked (no click effect).
5
+ */
6
+ export declare const hoverThenClick: (out: ToolInput) => Promise<boolean>;