@pillar-ai/sdk 0.1.27 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/index.d.ts +1 -1
- package/dist/actions/types.d.ts +0 -89
- package/dist/api/client.d.ts +2 -0
- package/dist/cli/sync.js +18 -3
- package/dist/components/Button/EdgeTrigger.d.ts +5 -0
- package/dist/components/PagePilot/styles.d.ts +1 -1
- package/dist/components/Panel/Panel.d.ts +5 -0
- package/dist/components/Panel/TabNavigation.d.ts +16 -0
- package/dist/components/Panel/styles.d.ts +1 -1
- package/dist/components/Progress/SecretReveal.d.ts +12 -0
- package/dist/components/Views/ArticleChatView.d.ts +2 -1
- package/dist/components/Views/HelpCenterArticles.d.ts +17 -0
- package/dist/components/Views/SupportView.d.ts +15 -0
- package/dist/components/context.d.ts +6 -2
- package/dist/core/config.d.ts +29 -0
- package/dist/pillar.esm.js +1 -1
- package/dist/store/suggestions.d.ts +1 -1
- package/dist/tools/types.d.ts +40 -14
- package/dist/utils/helpdesk.d.ts +33 -0
- package/dist/utils/normalize-tool-result.d.ts +10 -0
- package/package.json +1 -1
- package/dist/button/FloatingButton.d.ts +0 -44
- package/dist/components/DevTools/DOMScannerPreview.d.ts +0 -21
- package/dist/components/Views/DeveloperView.d.ts +0 -6
- package/dist/panel/Panel.d.ts +0 -53
- package/dist/panel/PanelUI.d.ts +0 -43
- package/dist/panel/components/ArticleCard.d.ts +0 -10
- package/dist/panel/components/CategoryCard.d.ts +0 -10
- package/dist/panel/components/ChatInput.d.ts +0 -36
- package/dist/panel/components/Header.d.ts +0 -16
- package/dist/panel/components/SearchInput.d.ts +0 -11
- package/dist/panel/styles.d.ts +0 -5
- package/dist/panel/views/ArticleView.d.ts +0 -21
- package/dist/panel/views/CategoryView.d.ts +0 -20
- package/dist/panel/views/ChatView.d.ts +0 -30
- package/dist/panel/views/HomeView.d.ts +0 -18
- package/dist/panel/views/SearchView.d.ts +0 -22
- package/dist/store/developer.d.ts +0 -19
- package/dist/tooltips/Tooltip.d.ts +0 -63
- package/dist/tooltips/TooltipManager.d.ts +0 -42
- package/dist/tooltips/styles.d.ts +0 -5
- package/dist/ui/config.d.ts +0 -96
- package/dist/ui/executor.d.ts +0 -75
- package/dist/ui/index.d.ts +0 -11
- package/dist/ui/scanner.d.ts +0 -105
- package/dist/ui/types.d.ts +0 -293
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Tooltip Manager
|
|
3
|
-
* Scans DOM for tooltip elements and manages their lifecycle
|
|
4
|
-
*/
|
|
5
|
-
import type { EventEmitter } from '../core/events';
|
|
6
|
-
import type { ResolvedConfig } from '../core/config';
|
|
7
|
-
import type { APIClient } from '../api/client';
|
|
8
|
-
export declare class TooltipManager {
|
|
9
|
-
private config;
|
|
10
|
-
private api;
|
|
11
|
-
private events;
|
|
12
|
-
private tooltips;
|
|
13
|
-
private tooltipData;
|
|
14
|
-
private observer;
|
|
15
|
-
private stylesInjected;
|
|
16
|
-
constructor(config: ResolvedConfig, api: APIClient, events: EventEmitter);
|
|
17
|
-
/**
|
|
18
|
-
* Initialize the tooltip manager
|
|
19
|
-
*/
|
|
20
|
-
init(): Promise<void>;
|
|
21
|
-
/**
|
|
22
|
-
* Scan DOM for tooltip elements and initialize them
|
|
23
|
-
*/
|
|
24
|
-
scan(): Promise<void>;
|
|
25
|
-
/**
|
|
26
|
-
* Show a specific tooltip
|
|
27
|
-
*/
|
|
28
|
-
show(tooltipId: string): void;
|
|
29
|
-
/**
|
|
30
|
-
* Hide a specific tooltip
|
|
31
|
-
*/
|
|
32
|
-
hide(tooltipId: string): void;
|
|
33
|
-
/**
|
|
34
|
-
* Destroy the tooltip manager
|
|
35
|
-
*/
|
|
36
|
-
destroy(): void;
|
|
37
|
-
private createTooltip;
|
|
38
|
-
private setupObserver;
|
|
39
|
-
private handleRemovedElement;
|
|
40
|
-
private getElementKey;
|
|
41
|
-
private getTooltipDataForElement;
|
|
42
|
-
}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Tooltip CSS Styles
|
|
3
|
-
* Injected into the document head
|
|
4
|
-
*/
|
|
5
|
-
export declare const TOOLTIP_STYLES = "\n/* Pillar Tooltip Styles */\n.pillar-tooltip {\n position: absolute;\n z-index: 99999;\n max-width: 320px;\n padding: 12px 16px;\n font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;\n font-size: 14px;\n line-height: 1.5;\n color: #1a1a1a;\n background: #ffffff;\n border-radius: 8px;\n box-shadow: 0 4px 20px rgba(0, 0, 0, 0.15), 0 0 0 1px rgba(0, 0, 0, 0.05);\n pointer-events: auto;\n opacity: 0;\n transform: scale(0.95);\n transition: opacity 0.15s ease, transform 0.15s ease;\n}\n\n.pillar-tooltip.pillar-tooltip--visible {\n opacity: 1;\n transform: scale(1);\n}\n\n.pillar-tooltip__content {\n margin: 0;\n}\n\n.pillar-tooltip__content p {\n margin: 0 0 8px;\n}\n\n.pillar-tooltip__content p:last-child {\n margin-bottom: 0;\n}\n\n.pillar-tooltip__content a {\n color: #2563eb;\n text-decoration: none;\n}\n\n.pillar-tooltip__content a:hover {\n text-decoration: underline;\n}\n\n.pillar-tooltip__learn-more {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n margin-top: 12px;\n padding: 6px 12px;\n font-size: 13px;\n font-weight: 500;\n color: #2563eb;\n background: #eff6ff;\n border: none;\n border-radius: 6px;\n cursor: pointer;\n transition: background 0.15s ease;\n}\n\n.pillar-tooltip__learn-more:hover {\n background: #dbeafe;\n}\n\n.pillar-tooltip__learn-more svg {\n width: 14px;\n height: 14px;\n}\n\n/* Arrow */\n.pillar-tooltip__arrow {\n position: absolute;\n width: 12px;\n height: 12px;\n background: #ffffff;\n transform: rotate(45deg);\n box-shadow: -1px -1px 0 0 rgba(0, 0, 0, 0.05);\n}\n\n.pillar-tooltip--top .pillar-tooltip__arrow {\n bottom: -6px;\n box-shadow: 1px 1px 0 0 rgba(0, 0, 0, 0.05);\n}\n\n.pillar-tooltip--bottom .pillar-tooltip__arrow {\n top: -6px;\n}\n\n.pillar-tooltip--left .pillar-tooltip__arrow {\n right: -6px;\n box-shadow: 1px -1px 0 0 rgba(0, 0, 0, 0.05);\n}\n\n.pillar-tooltip--right .pillar-tooltip__arrow {\n left: -6px;\n box-shadow: -1px 1px 0 0 rgba(0, 0, 0, 0.05);\n}\n\n/* Tooltip Icon (for icon trigger) */\n.pillar-tooltip-icon {\n display: inline-flex;\n align-items: center;\n justify-content: center;\n width: 16px;\n height: 16px;\n margin-left: 4px;\n vertical-align: middle;\n color: #6b7280;\n cursor: help;\n transition: color 0.15s ease;\n}\n\n.pillar-tooltip-icon:hover {\n color: #2563eb;\n}\n\n.pillar-tooltip-icon svg {\n width: 100%;\n height: 100%;\n}\n\n/* Close button */\n.pillar-tooltip__close {\n position: absolute;\n top: 8px;\n right: 8px;\n display: flex;\n align-items: center;\n justify-content: center;\n width: 20px;\n height: 20px;\n padding: 0;\n color: #9ca3af;\n background: none;\n border: none;\n border-radius: 4px;\n cursor: pointer;\n transition: color 0.15s ease, background 0.15s ease;\n}\n\n.pillar-tooltip__close:hover {\n color: #1a1a1a;\n background: #f3f4f6;\n}\n\n.pillar-tooltip__close svg {\n width: 12px;\n height: 12px;\n}\n";
|
package/dist/ui/config.d.ts
DELETED
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* UI Scanner Configuration
|
|
3
|
-
*
|
|
4
|
-
* Configuration options for the UI scanning system.
|
|
5
|
-
*/
|
|
6
|
-
/**
|
|
7
|
-
* Configuration for the UI Scanner.
|
|
8
|
-
*/
|
|
9
|
-
export interface UIScannerConfig {
|
|
10
|
-
/**
|
|
11
|
-
* Enable automatic UI scanning.
|
|
12
|
-
* When true, the page is scanned on init and after interactions.
|
|
13
|
-
* @default true
|
|
14
|
-
*/
|
|
15
|
-
enabled: boolean;
|
|
16
|
-
/**
|
|
17
|
-
* CSS selectors for containers to scan.
|
|
18
|
-
* Only elements within these containers will be included.
|
|
19
|
-
* Use ['body'] to scan the entire page.
|
|
20
|
-
* @default ['body']
|
|
21
|
-
*/
|
|
22
|
-
includeSelectors: string[];
|
|
23
|
-
/**
|
|
24
|
-
* CSS selectors to exclude from scanning.
|
|
25
|
-
* Elements matching these selectors (or within them) will be skipped.
|
|
26
|
-
* @default []
|
|
27
|
-
*/
|
|
28
|
-
excludeSelectors: string[];
|
|
29
|
-
/**
|
|
30
|
-
* Automatically exclude sensitive elements.
|
|
31
|
-
* When true, password fields, payment forms, etc. are excluded.
|
|
32
|
-
* @default true
|
|
33
|
-
*/
|
|
34
|
-
excludeSensitive: boolean;
|
|
35
|
-
/**
|
|
36
|
-
* Additional selectors for sensitive elements to exclude.
|
|
37
|
-
* These are added to the default sensitive selectors.
|
|
38
|
-
* @default []
|
|
39
|
-
*/
|
|
40
|
-
sensitiveSelectors: string[];
|
|
41
|
-
/**
|
|
42
|
-
* Maximum number of elements to include in the scan.
|
|
43
|
-
* Elements are prioritized by visibility and interactability.
|
|
44
|
-
* @default 100
|
|
45
|
-
*/
|
|
46
|
-
maxElements: number;
|
|
47
|
-
/**
|
|
48
|
-
* Automatically rescan after each UI interaction.
|
|
49
|
-
* @default true
|
|
50
|
-
*/
|
|
51
|
-
rescanAfterInteraction: boolean;
|
|
52
|
-
/**
|
|
53
|
-
* Debounce delay for rescanning in milliseconds.
|
|
54
|
-
* Prevents excessive scans during rapid interactions.
|
|
55
|
-
* @default 300
|
|
56
|
-
*/
|
|
57
|
-
rescanDebounceMs: number;
|
|
58
|
-
/**
|
|
59
|
-
* Include elements that are currently not visible (scrolled out of view).
|
|
60
|
-
* @default false
|
|
61
|
-
*/
|
|
62
|
-
includeOffscreen: boolean;
|
|
63
|
-
/**
|
|
64
|
-
* Maximum text length for element labels.
|
|
65
|
-
* Longer text will be truncated.
|
|
66
|
-
* @default 100
|
|
67
|
-
*/
|
|
68
|
-
maxLabelLength: number;
|
|
69
|
-
/**
|
|
70
|
-
* Maximum text length for context extraction.
|
|
71
|
-
* @default 200
|
|
72
|
-
*/
|
|
73
|
-
maxContextLength: number;
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Default sensitive element selectors.
|
|
77
|
-
* These are excluded when excludeSensitive is true.
|
|
78
|
-
*/
|
|
79
|
-
export declare const DEFAULT_SENSITIVE_SELECTORS: string[];
|
|
80
|
-
/**
|
|
81
|
-
* Default selectors for interactable elements.
|
|
82
|
-
* Used by the scanner to find elements to include.
|
|
83
|
-
*/
|
|
84
|
-
export declare const DEFAULT_INTERACTABLE_SELECTORS: string[];
|
|
85
|
-
/**
|
|
86
|
-
* Default configuration for the UI Scanner.
|
|
87
|
-
*/
|
|
88
|
-
export declare const DEFAULT_UI_SCANNER_CONFIG: UIScannerConfig;
|
|
89
|
-
/**
|
|
90
|
-
* Merge user config with defaults.
|
|
91
|
-
*/
|
|
92
|
-
export declare function resolveUIScannerConfig(config?: Partial<UIScannerConfig>): UIScannerConfig;
|
|
93
|
-
/**
|
|
94
|
-
* Get all sensitive selectors (default + custom).
|
|
95
|
-
*/
|
|
96
|
-
export declare function getSensitiveSelectors(config: UIScannerConfig): string[];
|
package/dist/ui/executor.d.ts
DELETED
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* UI Interaction Executor
|
|
3
|
-
*
|
|
4
|
-
* Executes AI-requested interactions on DOM elements.
|
|
5
|
-
* Handles clicking, typing, selecting, scrolling, and other interactions.
|
|
6
|
-
*/
|
|
7
|
-
import type { UIScanner } from './scanner';
|
|
8
|
-
import type { UIInteraction, UIInteractionResult } from './types';
|
|
9
|
-
/**
|
|
10
|
-
* Executes UI interactions on DOM elements.
|
|
11
|
-
*/
|
|
12
|
-
export declare class UIInteractionExecutor {
|
|
13
|
-
private scanner;
|
|
14
|
-
constructor(scanner: UIScanner);
|
|
15
|
-
/**
|
|
16
|
-
* Execute a UI interaction.
|
|
17
|
-
*/
|
|
18
|
-
execute(interaction: UIInteraction): Promise<UIInteractionResult>;
|
|
19
|
-
/**
|
|
20
|
-
* Find an element by its Pillar ID.
|
|
21
|
-
*/
|
|
22
|
-
private findElement;
|
|
23
|
-
/**
|
|
24
|
-
* Wait for an element to become visible.
|
|
25
|
-
*/
|
|
26
|
-
private waitForVisible;
|
|
27
|
-
/**
|
|
28
|
-
* Check if an element is visible.
|
|
29
|
-
*/
|
|
30
|
-
private isVisible;
|
|
31
|
-
/**
|
|
32
|
-
* Scroll an element into view.
|
|
33
|
-
*/
|
|
34
|
-
private scrollIntoView;
|
|
35
|
-
/**
|
|
36
|
-
* Click on an element.
|
|
37
|
-
*/
|
|
38
|
-
private click;
|
|
39
|
-
/**
|
|
40
|
-
* Type text into an input element.
|
|
41
|
-
*/
|
|
42
|
-
private type;
|
|
43
|
-
/**
|
|
44
|
-
* Select an option from a select element.
|
|
45
|
-
*/
|
|
46
|
-
private select;
|
|
47
|
-
/**
|
|
48
|
-
* Scroll to an element.
|
|
49
|
-
*/
|
|
50
|
-
private scroll;
|
|
51
|
-
/**
|
|
52
|
-
* Hover over an element.
|
|
53
|
-
*/
|
|
54
|
-
private hover;
|
|
55
|
-
/**
|
|
56
|
-
* Focus an element.
|
|
57
|
-
*/
|
|
58
|
-
private focus;
|
|
59
|
-
/**
|
|
60
|
-
* Clear an input element's value.
|
|
61
|
-
*/
|
|
62
|
-
private clear;
|
|
63
|
-
/**
|
|
64
|
-
* Create an error result.
|
|
65
|
-
*/
|
|
66
|
-
private errorResult;
|
|
67
|
-
/**
|
|
68
|
-
* Sleep for a given number of milliseconds.
|
|
69
|
-
*/
|
|
70
|
-
private sleep;
|
|
71
|
-
/**
|
|
72
|
-
* Flush microtasks to allow React/Vue state updates.
|
|
73
|
-
*/
|
|
74
|
-
private flushMicrotasks;
|
|
75
|
-
}
|
package/dist/ui/index.d.ts
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* UI Interaction Module
|
|
3
|
-
*
|
|
4
|
-
* Exports for the UI scanning and interaction system.
|
|
5
|
-
*/
|
|
6
|
-
export type { UIElement, UIElementRect, UIElementSummary, UIElementType, UIInteraction, UIInteractionOptions, UIInteractionResult, UIInteractionType, UIPageState, UIScrollPosition, } from './types';
|
|
7
|
-
export { toElementSummary, toPageStateSummary } from './types';
|
|
8
|
-
export type { UIScannerConfig } from './config';
|
|
9
|
-
export { DEFAULT_INTERACTABLE_SELECTORS, DEFAULT_SENSITIVE_SELECTORS, DEFAULT_UI_SCANNER_CONFIG, getSensitiveSelectors, resolveUIScannerConfig, } from './config';
|
|
10
|
-
export { UIScanner } from './scanner';
|
|
11
|
-
export { UIInteractionExecutor } from './executor';
|
package/dist/ui/scanner.d.ts
DELETED
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* UI Scanner
|
|
3
|
-
*
|
|
4
|
-
* Scans the DOM for interactable elements and builds a structured
|
|
5
|
-
* representation of the page's UI state for AI understanding.
|
|
6
|
-
*/
|
|
7
|
-
import { type UIScannerConfig } from './config';
|
|
8
|
-
import type { UIPageState } from './types';
|
|
9
|
-
/**
|
|
10
|
-
* Scans the page for interactable UI elements.
|
|
11
|
-
*/
|
|
12
|
-
export declare class UIScanner {
|
|
13
|
-
private config;
|
|
14
|
-
private elementIdCounter;
|
|
15
|
-
private elementMap;
|
|
16
|
-
constructor(config: UIScannerConfig);
|
|
17
|
-
/**
|
|
18
|
-
* Update scanner configuration.
|
|
19
|
-
*/
|
|
20
|
-
updateConfig(config: Partial<UIScannerConfig>): void;
|
|
21
|
-
/**
|
|
22
|
-
* Scan the page and return the UI state.
|
|
23
|
-
*/
|
|
24
|
-
scan(): UIPageState;
|
|
25
|
-
/**
|
|
26
|
-
* Get an element by its Pillar ID.
|
|
27
|
-
*/
|
|
28
|
-
getElementById(id: string): HTMLElement | null;
|
|
29
|
-
/**
|
|
30
|
-
* Find all interactable elements on the page.
|
|
31
|
-
*/
|
|
32
|
-
private findElements;
|
|
33
|
-
/**
|
|
34
|
-
* Get containers to scan based on config.
|
|
35
|
-
*/
|
|
36
|
-
private getContainers;
|
|
37
|
-
/**
|
|
38
|
-
* Check if element is in an excluded area.
|
|
39
|
-
*/
|
|
40
|
-
private isExcluded;
|
|
41
|
-
/**
|
|
42
|
-
* Check if element is sensitive (should be excluded for security).
|
|
43
|
-
*/
|
|
44
|
-
private isSensitive;
|
|
45
|
-
/**
|
|
46
|
-
* Check if element is visible.
|
|
47
|
-
*/
|
|
48
|
-
private isVisible;
|
|
49
|
-
/**
|
|
50
|
-
* Convert an HTML element to a UIElement.
|
|
51
|
-
*/
|
|
52
|
-
private elementToUIElement;
|
|
53
|
-
/**
|
|
54
|
-
* Generate a unique ID for an element.
|
|
55
|
-
*/
|
|
56
|
-
private generateId;
|
|
57
|
-
/**
|
|
58
|
-
* Determine the semantic type of an element.
|
|
59
|
-
*/
|
|
60
|
-
private getElementType;
|
|
61
|
-
/**
|
|
62
|
-
* Get the available interactions for an element.
|
|
63
|
-
*/
|
|
64
|
-
private getInteractions;
|
|
65
|
-
/**
|
|
66
|
-
* Extract a human-readable label for an element.
|
|
67
|
-
*/
|
|
68
|
-
private extractLabel;
|
|
69
|
-
/**
|
|
70
|
-
* Extract context from surrounding elements.
|
|
71
|
-
*/
|
|
72
|
-
private extractContext;
|
|
73
|
-
/**
|
|
74
|
-
* Check if element is disabled.
|
|
75
|
-
*/
|
|
76
|
-
private isDisabled;
|
|
77
|
-
/**
|
|
78
|
-
* Get element bounding rectangle.
|
|
79
|
-
*/
|
|
80
|
-
private getRect;
|
|
81
|
-
/**
|
|
82
|
-
* Get relevant attributes from element.
|
|
83
|
-
*/
|
|
84
|
-
private getRelevantAttributes;
|
|
85
|
-
/**
|
|
86
|
-
* Generate XPath for element.
|
|
87
|
-
*/
|
|
88
|
-
private getXPath;
|
|
89
|
-
/**
|
|
90
|
-
* Get the ID of the currently focused element.
|
|
91
|
-
*/
|
|
92
|
-
private getFocusedElementId;
|
|
93
|
-
/**
|
|
94
|
-
* Get current scroll position.
|
|
95
|
-
*/
|
|
96
|
-
private getScrollPosition;
|
|
97
|
-
/**
|
|
98
|
-
* Truncate text to max length.
|
|
99
|
-
*/
|
|
100
|
-
private truncate;
|
|
101
|
-
/**
|
|
102
|
-
* Convert camelCase/snake_case name to human readable.
|
|
103
|
-
*/
|
|
104
|
-
private humanizeName;
|
|
105
|
-
}
|
package/dist/ui/types.d.ts
DELETED
|
@@ -1,293 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* UI Interaction Types
|
|
3
|
-
*
|
|
4
|
-
* Type definitions for the UI scanning and interaction system.
|
|
5
|
-
* This enables AI agents to understand and interact with page elements.
|
|
6
|
-
*/
|
|
7
|
-
/**
|
|
8
|
-
* Supported UI interaction types.
|
|
9
|
-
*
|
|
10
|
-
* - click: Click/tap on an element
|
|
11
|
-
* - type: Enter text into an input field
|
|
12
|
-
* - select: Select an option from a dropdown/select
|
|
13
|
-
* - scroll: Scroll element into view
|
|
14
|
-
* - hover: Hover over an element
|
|
15
|
-
* - focus: Focus an element
|
|
16
|
-
* - clear: Clear an input field's value
|
|
17
|
-
*/
|
|
18
|
-
export type UIInteractionType = 'click' | 'type' | 'select' | 'scroll' | 'hover' | 'focus' | 'clear';
|
|
19
|
-
/**
|
|
20
|
-
* Element types detected by the scanner.
|
|
21
|
-
*/
|
|
22
|
-
export type UIElementType = 'button' | 'link' | 'input' | 'textarea' | 'select' | 'checkbox' | 'radio' | 'switch' | 'slider' | 'menu' | 'menuitem' | 'tab' | 'dialog' | 'custom' | 'unknown';
|
|
23
|
-
/**
|
|
24
|
-
* Bounding rectangle for an element.
|
|
25
|
-
*/
|
|
26
|
-
export interface UIElementRect {
|
|
27
|
-
x: number;
|
|
28
|
-
y: number;
|
|
29
|
-
width: number;
|
|
30
|
-
height: number;
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Represents a scanned UI element.
|
|
34
|
-
*
|
|
35
|
-
* Contains all information needed for the AI to understand
|
|
36
|
-
* and interact with a page element.
|
|
37
|
-
*/
|
|
38
|
-
export interface UIElement {
|
|
39
|
-
/**
|
|
40
|
-
* Unique identifier for this element.
|
|
41
|
-
* Generated during scanning (e.g., "pillar-el-0").
|
|
42
|
-
* Used to target the element for interactions.
|
|
43
|
-
*/
|
|
44
|
-
id: string;
|
|
45
|
-
/**
|
|
46
|
-
* HTML tag name (lowercase).
|
|
47
|
-
* @example "button", "input", "div"
|
|
48
|
-
*/
|
|
49
|
-
tag: string;
|
|
50
|
-
/**
|
|
51
|
-
* Semantic element type.
|
|
52
|
-
* Derived from tag, role, or type attribute.
|
|
53
|
-
*/
|
|
54
|
-
type: UIElementType;
|
|
55
|
-
/**
|
|
56
|
-
* Human-readable label for the AI.
|
|
57
|
-
* Extracted from aria-label, text content, title, etc.
|
|
58
|
-
*/
|
|
59
|
-
label: string;
|
|
60
|
-
/**
|
|
61
|
-
* Whether the element can be interacted with.
|
|
62
|
-
* False if disabled or not interactive.
|
|
63
|
-
*/
|
|
64
|
-
interactable: boolean;
|
|
65
|
-
/**
|
|
66
|
-
* List of interactions this element supports.
|
|
67
|
-
* Determined by element type and attributes.
|
|
68
|
-
*/
|
|
69
|
-
interactions: UIInteractionType[];
|
|
70
|
-
/**
|
|
71
|
-
* Current value for inputs, textareas, selects.
|
|
72
|
-
*/
|
|
73
|
-
value?: string;
|
|
74
|
-
/**
|
|
75
|
-
* Available options for select elements.
|
|
76
|
-
*/
|
|
77
|
-
options?: string[];
|
|
78
|
-
/**
|
|
79
|
-
* Placeholder text for inputs/textareas.
|
|
80
|
-
*/
|
|
81
|
-
placeholder?: string;
|
|
82
|
-
/**
|
|
83
|
-
* Whether checkbox/radio/switch is checked.
|
|
84
|
-
*/
|
|
85
|
-
checked?: boolean;
|
|
86
|
-
/**
|
|
87
|
-
* Whether the element is disabled.
|
|
88
|
-
*/
|
|
89
|
-
disabled?: boolean;
|
|
90
|
-
/**
|
|
91
|
-
* Element's bounding rectangle.
|
|
92
|
-
* Useful for understanding layout context.
|
|
93
|
-
*/
|
|
94
|
-
rect: UIElementRect;
|
|
95
|
-
/**
|
|
96
|
-
* Relevant attributes from the element.
|
|
97
|
-
* Includes data-pillar-* attributes and key attributes.
|
|
98
|
-
*/
|
|
99
|
-
attributes: Record<string, string>;
|
|
100
|
-
/**
|
|
101
|
-
* XPath for precise element targeting.
|
|
102
|
-
* Used as a fallback if ID-based lookup fails.
|
|
103
|
-
*/
|
|
104
|
-
xpath: string;
|
|
105
|
-
/**
|
|
106
|
-
* Nearby text or labels for additional context.
|
|
107
|
-
* Helps AI understand the element's purpose.
|
|
108
|
-
*/
|
|
109
|
-
context?: string;
|
|
110
|
-
/**
|
|
111
|
-
* Reference to the actual DOM element.
|
|
112
|
-
* Only available client-side, not serialized.
|
|
113
|
-
* @internal
|
|
114
|
-
*/
|
|
115
|
-
_element?: HTMLElement;
|
|
116
|
-
}
|
|
117
|
-
/**
|
|
118
|
-
* Scroll position on the page.
|
|
119
|
-
*/
|
|
120
|
-
export interface UIScrollPosition {
|
|
121
|
-
x: number;
|
|
122
|
-
y: number;
|
|
123
|
-
}
|
|
124
|
-
/**
|
|
125
|
-
* Page-level UI state.
|
|
126
|
-
*
|
|
127
|
-
* Represents a snapshot of the page's interactable elements
|
|
128
|
-
* at a specific point in time.
|
|
129
|
-
*/
|
|
130
|
-
export interface UIPageState {
|
|
131
|
-
/**
|
|
132
|
-
* Current page URL.
|
|
133
|
-
*/
|
|
134
|
-
url: string;
|
|
135
|
-
/**
|
|
136
|
-
* Page title.
|
|
137
|
-
*/
|
|
138
|
-
title: string;
|
|
139
|
-
/**
|
|
140
|
-
* Timestamp when the scan was performed.
|
|
141
|
-
*/
|
|
142
|
-
timestamp: number;
|
|
143
|
-
/**
|
|
144
|
-
* List of interactable elements on the page.
|
|
145
|
-
*/
|
|
146
|
-
elements: UIElement[];
|
|
147
|
-
/**
|
|
148
|
-
* ID of the currently focused element, if any.
|
|
149
|
-
*/
|
|
150
|
-
focusedElementId?: string;
|
|
151
|
-
/**
|
|
152
|
-
* Current scroll position.
|
|
153
|
-
*/
|
|
154
|
-
scrollPosition: UIScrollPosition;
|
|
155
|
-
}
|
|
156
|
-
/**
|
|
157
|
-
* Represents a UI interaction request from the AI.
|
|
158
|
-
*/
|
|
159
|
-
export interface UIInteraction {
|
|
160
|
-
/**
|
|
161
|
-
* ID of the element to interact with.
|
|
162
|
-
* Must match an element from UIPageState.elements.
|
|
163
|
-
*/
|
|
164
|
-
elementId: string;
|
|
165
|
-
/**
|
|
166
|
-
* Type of interaction to perform.
|
|
167
|
-
*/
|
|
168
|
-
action: UIInteractionType;
|
|
169
|
-
/**
|
|
170
|
-
* Value for type/select interactions.
|
|
171
|
-
*/
|
|
172
|
-
value?: string;
|
|
173
|
-
/**
|
|
174
|
-
* Additional options for the interaction.
|
|
175
|
-
*/
|
|
176
|
-
options?: UIInteractionOptions;
|
|
177
|
-
}
|
|
178
|
-
/**
|
|
179
|
-
* Options for UI interactions.
|
|
180
|
-
*/
|
|
181
|
-
export interface UIInteractionOptions {
|
|
182
|
-
/**
|
|
183
|
-
* Whether to clear existing value before typing.
|
|
184
|
-
* Only applies to 'type' action.
|
|
185
|
-
* @default true
|
|
186
|
-
*/
|
|
187
|
-
clearBeforeType?: boolean;
|
|
188
|
-
/**
|
|
189
|
-
* Delay in ms between keystrokes for type action.
|
|
190
|
-
* Use 0 for instant typing.
|
|
191
|
-
* @default 0
|
|
192
|
-
*/
|
|
193
|
-
typeDelay?: number;
|
|
194
|
-
/**
|
|
195
|
-
* Whether to scroll element into view before interaction.
|
|
196
|
-
* @default true
|
|
197
|
-
*/
|
|
198
|
-
scrollIntoView?: boolean;
|
|
199
|
-
/**
|
|
200
|
-
* Whether to wait for element to be visible.
|
|
201
|
-
* @default true
|
|
202
|
-
*/
|
|
203
|
-
waitForVisible?: boolean;
|
|
204
|
-
/**
|
|
205
|
-
* Timeout in ms for waiting operations.
|
|
206
|
-
* @default 5000
|
|
207
|
-
*/
|
|
208
|
-
timeout?: number;
|
|
209
|
-
}
|
|
210
|
-
/**
|
|
211
|
-
* Result of a UI interaction.
|
|
212
|
-
*/
|
|
213
|
-
export interface UIInteractionResult {
|
|
214
|
-
/**
|
|
215
|
-
* Whether the interaction succeeded.
|
|
216
|
-
*/
|
|
217
|
-
success: boolean;
|
|
218
|
-
/**
|
|
219
|
-
* Error message if the interaction failed.
|
|
220
|
-
*/
|
|
221
|
-
error?: string;
|
|
222
|
-
/**
|
|
223
|
-
* The element ID that was interacted with.
|
|
224
|
-
*/
|
|
225
|
-
elementId: string;
|
|
226
|
-
/**
|
|
227
|
-
* The action that was performed.
|
|
228
|
-
*/
|
|
229
|
-
action: UIInteractionType;
|
|
230
|
-
/**
|
|
231
|
-
* New value after the interaction (for inputs).
|
|
232
|
-
*/
|
|
233
|
-
newValue?: string;
|
|
234
|
-
/**
|
|
235
|
-
* Whether the element is still present after interaction.
|
|
236
|
-
* May be false if clicking caused navigation or element removal.
|
|
237
|
-
*/
|
|
238
|
-
elementStillPresent?: boolean;
|
|
239
|
-
/**
|
|
240
|
-
* Any data extracted from the interaction.
|
|
241
|
-
* For example, selected option text.
|
|
242
|
-
*/
|
|
243
|
-
data?: Record<string, unknown>;
|
|
244
|
-
}
|
|
245
|
-
/**
|
|
246
|
-
* Simplified element representation for AI context.
|
|
247
|
-
*
|
|
248
|
-
* Contains only essential information to reduce token usage
|
|
249
|
-
* while still providing enough context for the AI.
|
|
250
|
-
*/
|
|
251
|
-
export interface UIElementSummary {
|
|
252
|
-
/**
|
|
253
|
-
* Element ID for targeting.
|
|
254
|
-
*/
|
|
255
|
-
id: string;
|
|
256
|
-
/**
|
|
257
|
-
* Human-readable label.
|
|
258
|
-
*/
|
|
259
|
-
label: string;
|
|
260
|
-
/**
|
|
261
|
-
* Element type (button, input, etc.).
|
|
262
|
-
*/
|
|
263
|
-
type: string;
|
|
264
|
-
/**
|
|
265
|
-
* Available interactions.
|
|
266
|
-
*/
|
|
267
|
-
interactions: string[];
|
|
268
|
-
/**
|
|
269
|
-
* Current value (if applicable).
|
|
270
|
-
*/
|
|
271
|
-
value?: string;
|
|
272
|
-
/**
|
|
273
|
-
* Available options (for selects).
|
|
274
|
-
*/
|
|
275
|
-
options?: string[];
|
|
276
|
-
/**
|
|
277
|
-
* Additional context about the element.
|
|
278
|
-
*/
|
|
279
|
-
context?: string;
|
|
280
|
-
}
|
|
281
|
-
/**
|
|
282
|
-
* Convert a full UIElement to a summary for AI context.
|
|
283
|
-
*/
|
|
284
|
-
export declare function toElementSummary(element: UIElement): UIElementSummary;
|
|
285
|
-
/**
|
|
286
|
-
* Convert UIPageState to a minimal representation for AI context.
|
|
287
|
-
*/
|
|
288
|
-
export declare function toPageStateSummary(state: UIPageState): {
|
|
289
|
-
url: string;
|
|
290
|
-
title: string;
|
|
291
|
-
elements: UIElementSummary[];
|
|
292
|
-
focusedElementId?: string;
|
|
293
|
-
};
|