handhold-sdk 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +427 -0
- package/dist/d44f1fe8001e32f8e74d.svg +3056 -0
- package/dist/fonts/Phosphor.ttf +0 -0
- package/dist/fonts/Phosphor.woff +0 -0
- package/dist/fonts/Phosphor.woff2 +0 -0
- package/dist/handhold.min.js +1 -0
- package/package.json +114 -0
- package/src/HandHold.js +1599 -0
- package/src/agent/VENDOR.md +203 -0
- package/src/agent/agent/AgentLoop.js +366 -0
- package/src/agent/agent/prompts.js +163 -0
- package/src/agent/agent/tools.js +148 -0
- package/src/agent/controller.js +235 -0
- package/src/agent/dom/actions.js +456 -0
- package/src/agent/dom/domTree.js +1761 -0
- package/src/agent/dom/redact.js +98 -0
- package/src/agent/dom/serializer.js +332 -0
- package/src/agent/dom/utils.js +99 -0
- package/src/agent/index.js +169 -0
- package/src/agent/llm/connector.js +143 -0
- package/src/agent/package.json +3 -0
- package/src/agent/policy/PolicyGuard.js +172 -0
- package/src/agent/site/manifest.js +145 -0
- package/src/agent/ui/ChatWidget.js +219 -0
- package/src/agent-mode/AgentMode.js +429 -0
- package/src/api/APIClient.js +258 -0
- package/src/core/Config.js +207 -0
- package/src/core/UiState.js +83 -0
- package/src/core/defaults.js +20 -0
- package/src/events.js +59 -0
- package/src/index.js +77 -0
- package/src/intent/ActionVerbMap.js +324 -0
- package/src/intent/IntentExtractor.js +317 -0
- package/src/observers/ElementScanner.js +284 -0
- package/src/observers/NavigationObserver.js +182 -0
- package/src/observers/PageObserver.js +293 -0
- package/src/session/SessionManager.js +402 -0
- package/src/session/SessionStorage.js +148 -0
- package/src/ui/HelpWidget.js +1189 -0
- package/src/ui/UICoach.js +1725 -0
- package/src/ui/components/Button.css +137 -0
- package/src/ui/components/Button.js +102 -0
- package/src/ui/widget.css +599 -0
- package/src/utils/Logger.js +132 -0
- package/src/utils/MarkdownRenderer.js +39 -0
- package/src/utils/domUtils.js +350 -0
- package/src/utils/eventBus.js +102 -0
- package/src/utils/index.js +8 -0
- package/src/utils/stringUtils.js +202 -0
- package/types/index.d.ts +538 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hand Hold SDK - Element Scanner
|
|
3
|
+
* Scans and extracts interactive UI elements from the page
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
generateSelector,
|
|
8
|
+
getElementText,
|
|
9
|
+
getElementRole,
|
|
10
|
+
getElementPosition,
|
|
11
|
+
isElementVisible,
|
|
12
|
+
isInteractiveElement,
|
|
13
|
+
} from '../utils/domUtils.js';
|
|
14
|
+
|
|
15
|
+
class ElementScanner {
|
|
16
|
+
constructor(config) {
|
|
17
|
+
this.config = config;
|
|
18
|
+
this.excludeSelectors = [
|
|
19
|
+
'#handhold-container',
|
|
20
|
+
'#handhold-widget',
|
|
21
|
+
'#handhold-overlay',
|
|
22
|
+
'[data-handhold-ignore]',
|
|
23
|
+
];
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Scan page and extract all relevant UI elements
|
|
28
|
+
* @param {Element} [root=document.body] - Root element to scan from
|
|
29
|
+
* @returns {Array} - Array of element objects
|
|
30
|
+
*/
|
|
31
|
+
scan(root = document.body) {
|
|
32
|
+
const elements = [];
|
|
33
|
+
const seen = new Set();
|
|
34
|
+
|
|
35
|
+
// Get interactive elements
|
|
36
|
+
const interactiveElements = this._getInteractiveElements(root);
|
|
37
|
+
|
|
38
|
+
for (const element of interactiveElements) {
|
|
39
|
+
// Skip excluded elements
|
|
40
|
+
if (this._isExcluded(element)) continue;
|
|
41
|
+
|
|
42
|
+
// Skip invisible elements
|
|
43
|
+
if (!isElementVisible(element)) continue;
|
|
44
|
+
|
|
45
|
+
// Generate selector
|
|
46
|
+
const selector = generateSelector(element);
|
|
47
|
+
|
|
48
|
+
// Skip duplicates
|
|
49
|
+
if (seen.has(selector)) continue;
|
|
50
|
+
seen.add(selector);
|
|
51
|
+
|
|
52
|
+
// Extract element data
|
|
53
|
+
const elementData = this._extractElementData(element, selector);
|
|
54
|
+
|
|
55
|
+
if (elementData) {
|
|
56
|
+
elements.push(elementData);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return elements;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Get all interactive elements
|
|
65
|
+
* @param {Element} root - Root element
|
|
66
|
+
* @returns {Element[]}
|
|
67
|
+
*/
|
|
68
|
+
_getInteractiveElements(root) {
|
|
69
|
+
const selectors = [
|
|
70
|
+
'button',
|
|
71
|
+
'a[href]',
|
|
72
|
+
'input:not([type="hidden"])',
|
|
73
|
+
'select',
|
|
74
|
+
'textarea',
|
|
75
|
+
'[role="button"]',
|
|
76
|
+
'[role="link"]',
|
|
77
|
+
'[role="checkbox"]',
|
|
78
|
+
'[role="radio"]',
|
|
79
|
+
'[role="menuitem"]',
|
|
80
|
+
'[role="tab"]',
|
|
81
|
+
'[role="switch"]',
|
|
82
|
+
'[onclick]',
|
|
83
|
+
'[tabindex]:not([tabindex="-1"])',
|
|
84
|
+
'.btn',
|
|
85
|
+
'.button',
|
|
86
|
+
];
|
|
87
|
+
|
|
88
|
+
return Array.from(root.querySelectorAll(selectors.join(', ')));
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Check if element should be excluded
|
|
93
|
+
* @param {Element} element
|
|
94
|
+
* @returns {boolean}
|
|
95
|
+
*/
|
|
96
|
+
_isExcluded(element) {
|
|
97
|
+
// Check if element matches exclude selectors
|
|
98
|
+
for (const selector of this.excludeSelectors) {
|
|
99
|
+
if (element.matches(selector) || element.closest(selector)) {
|
|
100
|
+
return true;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Exclude hidden inputs
|
|
105
|
+
if (element.tagName === 'INPUT' && element.type === 'hidden') {
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Extract data from element
|
|
114
|
+
* @param {Element} element
|
|
115
|
+
* @param {string} selector
|
|
116
|
+
* @returns {Object|null}
|
|
117
|
+
*/
|
|
118
|
+
_extractElementData(element, selector) {
|
|
119
|
+
let text = getElementText(element);
|
|
120
|
+
const ariaLabel = element.getAttribute('aria-label');
|
|
121
|
+
const title = element.getAttribute('title');
|
|
122
|
+
const role = getElementRole(element);
|
|
123
|
+
const id = element.id;
|
|
124
|
+
const name = element.getAttribute('name');
|
|
125
|
+
|
|
126
|
+
// For inputs, try to find label if text/aria-label is missing
|
|
127
|
+
if (['input', 'textarea', 'select'].includes(element.tagName.toLowerCase())) {
|
|
128
|
+
const label = this._findInputLabel(element);
|
|
129
|
+
if (label) {
|
|
130
|
+
text = text ? `${label} (${text})` : label;
|
|
131
|
+
} else if (name) {
|
|
132
|
+
// Fallback to name if no label found
|
|
133
|
+
text = text ? `${name} (${text})` : name;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Skip elements with no identifiable text
|
|
138
|
+
if (!text && !ariaLabel && !title && !id && !name) {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const position = getElementPosition(element);
|
|
143
|
+
|
|
144
|
+
return {
|
|
145
|
+
text: text || '',
|
|
146
|
+
aria_label: ariaLabel || '',
|
|
147
|
+
title: title || '',
|
|
148
|
+
role,
|
|
149
|
+
selector,
|
|
150
|
+
tag: element.tagName.toLowerCase(),
|
|
151
|
+
type: element.type || null,
|
|
152
|
+
disabled: element.disabled || element.getAttribute('aria-disabled') === 'true',
|
|
153
|
+
position,
|
|
154
|
+
data_testid: element.getAttribute('data-testid') || null,
|
|
155
|
+
id: id || null,
|
|
156
|
+
name: name || null,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Get page context information
|
|
162
|
+
* @returns {Object}
|
|
163
|
+
*/
|
|
164
|
+
getPageContext() {
|
|
165
|
+
return {
|
|
166
|
+
title: document.title,
|
|
167
|
+
route: window.location.pathname,
|
|
168
|
+
url: window.location.href,
|
|
169
|
+
headings: this._extractHeadings(),
|
|
170
|
+
forms: this._extractForms(),
|
|
171
|
+
meta: this._extractMeta(),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Extract page headings
|
|
177
|
+
* @returns {string[]}
|
|
178
|
+
*/
|
|
179
|
+
_extractHeadings() {
|
|
180
|
+
const headings = [];
|
|
181
|
+
const headingElements = document.querySelectorAll('h1, h2, h3');
|
|
182
|
+
|
|
183
|
+
headingElements.forEach(h => {
|
|
184
|
+
const text = h.textContent?.trim();
|
|
185
|
+
if (text && !this._isExcluded(h)) {
|
|
186
|
+
headings.push(text);
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
return headings.slice(0, 10); // Limit to 10 headings
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Extract form information
|
|
195
|
+
* @returns {Object[]}
|
|
196
|
+
*/
|
|
197
|
+
_extractForms() {
|
|
198
|
+
const forms = [];
|
|
199
|
+
const formElements = document.querySelectorAll('form');
|
|
200
|
+
|
|
201
|
+
formElements.forEach(form => {
|
|
202
|
+
if (this._isExcluded(form)) return;
|
|
203
|
+
|
|
204
|
+
const inputs = Array.from(form.querySelectorAll('input, select, textarea'))
|
|
205
|
+
.filter(el => el.type !== 'hidden' && isElementVisible(el))
|
|
206
|
+
.map(el => ({
|
|
207
|
+
name: el.name || el.id || '',
|
|
208
|
+
type: el.type || el.tagName.toLowerCase(),
|
|
209
|
+
label: this._findInputLabel(el),
|
|
210
|
+
}));
|
|
211
|
+
|
|
212
|
+
if (inputs.length > 0) {
|
|
213
|
+
forms.push({
|
|
214
|
+
id: form.id || null,
|
|
215
|
+
action: form.action || null,
|
|
216
|
+
method: form.method || 'get',
|
|
217
|
+
inputs,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
return forms;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Find label for an input element
|
|
227
|
+
* @param {Element} input
|
|
228
|
+
* @returns {string}
|
|
229
|
+
*/
|
|
230
|
+
_findInputLabel(input) {
|
|
231
|
+
// Check for associated label
|
|
232
|
+
if (input.id) {
|
|
233
|
+
const label = document.querySelector(`label[for="${input.id}"]`);
|
|
234
|
+
if (label) return label.textContent?.trim() || '';
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Check for parent label
|
|
238
|
+
const parentLabel = input.closest('label');
|
|
239
|
+
if (parentLabel) {
|
|
240
|
+
return parentLabel.textContent?.trim() || '';
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// Check for aria-label
|
|
244
|
+
return input.getAttribute('aria-label') || input.placeholder || '';
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Extract page meta information
|
|
249
|
+
* @returns {Object}
|
|
250
|
+
*/
|
|
251
|
+
_extractMeta() {
|
|
252
|
+
return {
|
|
253
|
+
description: document.querySelector('meta[name="description"]')?.content || '',
|
|
254
|
+
keywords: document.querySelector('meta[name="keywords"]')?.content || '',
|
|
255
|
+
viewport: document.querySelector('meta[name="viewport"]')?.content || '',
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Find element by selector
|
|
261
|
+
* @param {string} selector
|
|
262
|
+
* @returns {Element|null}
|
|
263
|
+
*/
|
|
264
|
+
findElement(selector) {
|
|
265
|
+
try {
|
|
266
|
+
return document.querySelector(selector);
|
|
267
|
+
} catch (e) {
|
|
268
|
+
console.warn('[HandHold] Invalid selector:', selector);
|
|
269
|
+
return null;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Check if an element exists and is visible
|
|
275
|
+
* @param {string} selector
|
|
276
|
+
* @returns {boolean}
|
|
277
|
+
*/
|
|
278
|
+
elementExists(selector) {
|
|
279
|
+
const element = this.findElement(selector);
|
|
280
|
+
return element && isElementVisible(element);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
export default ElementScanner;
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hand Hold SDK - Navigation Observer
|
|
3
|
+
* Detects route/URL changes in SPAs and traditional navigation
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import eventBus from '../utils/eventBus.js';
|
|
7
|
+
|
|
8
|
+
class NavigationObserver {
|
|
9
|
+
constructor(config) {
|
|
10
|
+
this.config = config;
|
|
11
|
+
this.currentRoute = this._getCurrentRoute();
|
|
12
|
+
this.isObserving = false;
|
|
13
|
+
|
|
14
|
+
// Bound handlers for cleanup
|
|
15
|
+
this._boundPopstateHandler = this._onPopstate.bind(this);
|
|
16
|
+
this._boundHashchangeHandler = this._onHashchange.bind(this);
|
|
17
|
+
this._boundClickHandler = this._onLinkClick.bind(this);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Get current route
|
|
22
|
+
* @returns {string}
|
|
23
|
+
*/
|
|
24
|
+
_getCurrentRoute() {
|
|
25
|
+
return window.location.pathname + window.location.search + window.location.hash;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Start observing navigation changes
|
|
30
|
+
*/
|
|
31
|
+
observe() {
|
|
32
|
+
if (this.isObserving) return;
|
|
33
|
+
|
|
34
|
+
// Listen for popstate (back/forward navigation)
|
|
35
|
+
window.addEventListener('popstate', this._boundPopstateHandler);
|
|
36
|
+
|
|
37
|
+
// Listen for hashchange
|
|
38
|
+
window.addEventListener('hashchange', this._boundHashchangeHandler);
|
|
39
|
+
|
|
40
|
+
// Intercept link clicks for SPA navigation detection
|
|
41
|
+
document.addEventListener('click', this._boundClickHandler, true);
|
|
42
|
+
|
|
43
|
+
// Patch pushState and replaceState for SPA frameworks
|
|
44
|
+
this._patchHistoryMethods();
|
|
45
|
+
|
|
46
|
+
this.isObserving = true;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Stop observing navigation changes
|
|
51
|
+
*/
|
|
52
|
+
disconnect() {
|
|
53
|
+
if (!this.isObserving) return;
|
|
54
|
+
|
|
55
|
+
window.removeEventListener('popstate', this._boundPopstateHandler);
|
|
56
|
+
window.removeEventListener('hashchange', this._boundHashchangeHandler);
|
|
57
|
+
document.removeEventListener('click', this._boundClickHandler, true);
|
|
58
|
+
|
|
59
|
+
this._unpatchHistoryMethods();
|
|
60
|
+
|
|
61
|
+
this.isObserving = false;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Patch history.pushState and history.replaceState
|
|
66
|
+
*/
|
|
67
|
+
_patchHistoryMethods() {
|
|
68
|
+
// Store original methods
|
|
69
|
+
this._originalPushState = window.history.pushState;
|
|
70
|
+
this._originalReplaceState = window.history.replaceState;
|
|
71
|
+
|
|
72
|
+
const self = this;
|
|
73
|
+
|
|
74
|
+
// Patch pushState
|
|
75
|
+
window.history.pushState = function(...args) {
|
|
76
|
+
self._originalPushState.apply(this, args);
|
|
77
|
+
self._checkRouteChange();
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
// Patch replaceState
|
|
81
|
+
window.history.replaceState = function(...args) {
|
|
82
|
+
self._originalReplaceState.apply(this, args);
|
|
83
|
+
self._checkRouteChange();
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Restore original history methods
|
|
89
|
+
*/
|
|
90
|
+
_unpatchHistoryMethods() {
|
|
91
|
+
if (this._originalPushState) {
|
|
92
|
+
window.history.pushState = this._originalPushState;
|
|
93
|
+
}
|
|
94
|
+
if (this._originalReplaceState) {
|
|
95
|
+
window.history.replaceState = this._originalReplaceState;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Handle popstate event
|
|
101
|
+
*/
|
|
102
|
+
_onPopstate(event) {
|
|
103
|
+
this._checkRouteChange();
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Handle hashchange event
|
|
108
|
+
*/
|
|
109
|
+
_onHashchange(event) {
|
|
110
|
+
this._checkRouteChange();
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Handle link clicks
|
|
115
|
+
*/
|
|
116
|
+
_onLinkClick(event) {
|
|
117
|
+
// Find closest anchor element
|
|
118
|
+
const anchor = event.target.closest('a');
|
|
119
|
+
if (!anchor) return;
|
|
120
|
+
|
|
121
|
+
const href = anchor.getAttribute('href');
|
|
122
|
+
if (!href) return;
|
|
123
|
+
|
|
124
|
+
// Skip external links
|
|
125
|
+
if (href.startsWith('http') && !href.startsWith(window.location.origin)) {
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Skip special links
|
|
130
|
+
if (href.startsWith('javascript:') || href.startsWith('mailto:') || href.startsWith('tel:')) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// Check for route change after a short delay (to allow SPA routing to complete)
|
|
135
|
+
setTimeout(() => this._checkRouteChange(), 50);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Check if route has changed and emit event
|
|
140
|
+
*/
|
|
141
|
+
_checkRouteChange() {
|
|
142
|
+
const newRoute = this._getCurrentRoute();
|
|
143
|
+
|
|
144
|
+
if (newRoute !== this.currentRoute) {
|
|
145
|
+
const previousRoute = this.currentRoute;
|
|
146
|
+
this.currentRoute = newRoute;
|
|
147
|
+
|
|
148
|
+
eventBus.emit('navigation:changed', {
|
|
149
|
+
route: newRoute,
|
|
150
|
+
previousRoute,
|
|
151
|
+
pathname: window.location.pathname,
|
|
152
|
+
search: window.location.search,
|
|
153
|
+
hash: window.location.hash,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Get current pathname only
|
|
160
|
+
* @returns {string}
|
|
161
|
+
*/
|
|
162
|
+
getPathname() {
|
|
163
|
+
return window.location.pathname;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Get current full route
|
|
168
|
+
* @returns {string}
|
|
169
|
+
*/
|
|
170
|
+
getRoute() {
|
|
171
|
+
return this.currentRoute;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Force route check (useful after known navigation)
|
|
176
|
+
*/
|
|
177
|
+
forceCheck() {
|
|
178
|
+
this._checkRouteChange();
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export default NavigationObserver;
|