handhold-sdk 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +427 -0
  2. package/dist/d44f1fe8001e32f8e74d.svg +3056 -0
  3. package/dist/fonts/Phosphor.ttf +0 -0
  4. package/dist/fonts/Phosphor.woff +0 -0
  5. package/dist/fonts/Phosphor.woff2 +0 -0
  6. package/dist/handhold.min.js +1 -0
  7. package/package.json +114 -0
  8. package/src/HandHold.js +1599 -0
  9. package/src/agent/VENDOR.md +203 -0
  10. package/src/agent/agent/AgentLoop.js +366 -0
  11. package/src/agent/agent/prompts.js +163 -0
  12. package/src/agent/agent/tools.js +148 -0
  13. package/src/agent/controller.js +235 -0
  14. package/src/agent/dom/actions.js +456 -0
  15. package/src/agent/dom/domTree.js +1761 -0
  16. package/src/agent/dom/redact.js +98 -0
  17. package/src/agent/dom/serializer.js +332 -0
  18. package/src/agent/dom/utils.js +99 -0
  19. package/src/agent/index.js +169 -0
  20. package/src/agent/llm/connector.js +143 -0
  21. package/src/agent/package.json +3 -0
  22. package/src/agent/policy/PolicyGuard.js +172 -0
  23. package/src/agent/site/manifest.js +145 -0
  24. package/src/agent/ui/ChatWidget.js +219 -0
  25. package/src/agent-mode/AgentMode.js +429 -0
  26. package/src/api/APIClient.js +258 -0
  27. package/src/core/Config.js +207 -0
  28. package/src/core/UiState.js +83 -0
  29. package/src/core/defaults.js +20 -0
  30. package/src/events.js +59 -0
  31. package/src/index.js +77 -0
  32. package/src/intent/ActionVerbMap.js +324 -0
  33. package/src/intent/IntentExtractor.js +317 -0
  34. package/src/observers/ElementScanner.js +284 -0
  35. package/src/observers/NavigationObserver.js +182 -0
  36. package/src/observers/PageObserver.js +293 -0
  37. package/src/session/SessionManager.js +402 -0
  38. package/src/session/SessionStorage.js +148 -0
  39. package/src/ui/HelpWidget.js +1189 -0
  40. package/src/ui/UICoach.js +1725 -0
  41. package/src/ui/components/Button.css +137 -0
  42. package/src/ui/components/Button.js +102 -0
  43. package/src/ui/widget.css +599 -0
  44. package/src/utils/Logger.js +132 -0
  45. package/src/utils/MarkdownRenderer.js +39 -0
  46. package/src/utils/domUtils.js +350 -0
  47. package/src/utils/eventBus.js +102 -0
  48. package/src/utils/index.js +8 -0
  49. package/src/utils/stringUtils.js +202 -0
  50. package/types/index.d.ts +538 -0
@@ -0,0 +1,98 @@
1
+ /**
2
+ * redact.js — PII redaction applied to page text/attributes BEFORE they are
3
+ * serialized into the LLM prompt.
4
+ *
5
+ * Upstream (alibaba/page-agent) left this as an open TODO in dom/index.ts
6
+ * ("@todo 数据脱敏过滤器" — data-masking filter) and their own
7
+ * terms-and-privacy.md admits HTML cleaning "does not guarantee removal of
8
+ * sensitive information". This module is the missing piece.
9
+ *
10
+ * Design notes:
11
+ * - Redaction is ON by default. Secure by default; opt out per category.
12
+ * - Card numbers are only redacted when Luhn-valid, so 16-digit order ids
13
+ * and tracking numbers survive (a naive \d{16} rule destroys e-commerce
14
+ * pages and makes the agent useless on them).
15
+ * - This is a best-effort net, not a guarantee. Do not point the agent at
16
+ * pages containing data you are not allowed to send to your LLM provider.
17
+ */
18
+
19
+ function luhnValid(digits) {
20
+ let sum = 0
21
+ let alt = false
22
+ for (let i = digits.length - 1; i >= 0; i--) {
23
+ let d = digits.charCodeAt(i) - 48
24
+ if (alt) {
25
+ d *= 2
26
+ if (d > 9) d -= 9
27
+ }
28
+ sum += d
29
+ alt = !alt
30
+ }
31
+ return sum % 10 === 0
32
+ }
33
+
34
+ const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g
35
+
36
+ // Phones must show phone-ish intent: a leading +country code, parentheses,
37
+ // or dash/dot separators. Plain space-separated digit groups (order numbers,
38
+ // non-Luhn ids) are deliberately NOT treated as phones.
39
+ const PHONE_RE = /(?:\+\d{1,3}(?:[ .-]?\d{2,4}){2,4}|(?:\(\d{2,4}\)[ .-]?)\d{3,4}[ .-]\d{3,4}|\d{3,4}[.-]\d{3,4}[.-]\d{3,4})/g
40
+
41
+ // Last char must be a digit so trailing whitespace is never consumed.
42
+ const CARD_CANDIDATE_RE = /\b\d(?:[ -]?\d){12,18}\b/g
43
+
44
+ const SSN_RE = /\b\d{3}-\d{2}-\d{4}\b/g
45
+
46
+ function countDigits(s) {
47
+ let n = 0
48
+ for (const ch of s) if (ch >= '0' && ch <= '9') n++
49
+ return n
50
+ }
51
+
52
+ /**
53
+ * @param {Object} [options]
54
+ * @param {boolean} [options.email=true]
55
+ * @param {boolean} [options.phone=true]
56
+ * @param {boolean} [options.card=true]
57
+ * @param {boolean} [options.ssn=true]
58
+ * @param {{pattern: RegExp, label: string}[]} [options.extraPatterns]
59
+ * @returns {(text: string) => string}
60
+ */
61
+ export function createRedactor(options = {}) {
62
+ const { email = true, phone = true, card = true, ssn = true, extraPatterns = [] } = options
63
+
64
+ return function redact(text) {
65
+ if (typeof text !== 'string' || text.length === 0) return text
66
+ let out = text
67
+
68
+ if (email) out = out.replace(EMAIL_RE, '[EMAIL]')
69
+ if (ssn) out = out.replace(SSN_RE, '[SSN]')
70
+
71
+ if (card) {
72
+ out = out.replace(CARD_CANDIDATE_RE, (match) => {
73
+ const digits = match.replace(/[ -]/g, '')
74
+ if (digits.length >= 13 && digits.length <= 19 && luhnValid(digits)) return '[CARD]'
75
+ return match
76
+ })
77
+ }
78
+
79
+ if (phone) {
80
+ out = out.replace(PHONE_RE, (match) => {
81
+ // require enough digits to plausibly be a phone number,
82
+ // and at least one separator (so plain long numbers pass through
83
+ // to the card check instead)
84
+ if (countDigits(match) >= 9 && /[ .-]/.test(match)) return '[PHONE]'
85
+ return match
86
+ })
87
+ }
88
+
89
+ for (const { pattern, label } of extraPatterns) {
90
+ out = out.replace(pattern, label)
91
+ }
92
+
93
+ return out
94
+ }
95
+ }
96
+
97
+ /** Default redactor with everything on. */
98
+ export const redactText = createRedactor()
@@ -0,0 +1,332 @@
1
+ /**
2
+ * serializer.js — turns the domTree engine's flat tree into (1) the indexed
3
+ * text snapshot the LLM reads and (2) a selector map of index → live element.
4
+ *
5
+ * Ported from alibaba/page-agent packages/page-controller/src/dom/index.ts (MIT),
6
+ * with two deliberate changes:
7
+ * 1. REDACTION (new): every text fragment and attribute value passes through
8
+ * a redactor before entering the snapshot. Upstream left this as an open
9
+ * TODO ("@todo 数据脱敏过滤器"). Default redactor strips emails, phones,
10
+ * Luhn-valid card numbers, SSNs. See dom/redact.js.
11
+ * 2. No import-time side effects: upstream registers window listeners at
12
+ * module load; here installNavigationCleanup() must be called explicitly.
13
+ */
14
+
15
+ import domTree from './domTree.js'
16
+ import { redactText } from './redact.js'
17
+
18
+ const DEFAULT_VIEWPORT_EXPANSION = -1
19
+
20
+ const SEMANTIC_TAGS = new Set(['nav', 'menu', 'header', 'footer', 'aside', 'dialog'])
21
+
22
+ const DEFAULT_INCLUDE_ATTRIBUTES = [
23
+ 'title', 'type', 'checked', 'name', 'role', 'value', 'placeholder',
24
+ 'data-date-format', 'alt', 'aria-label', 'aria-expanded', 'data-state',
25
+ 'aria-checked',
26
+ 'id', 'for',
27
+ 'target',
28
+ 'aria-haspopup', 'aria-controls', 'aria-owns',
29
+ 'contenteditable',
30
+ // needed by PolicyGuard's credential/payment detection
31
+ 'autocomplete',
32
+ ]
33
+
34
+ const newElementsCache = new WeakMap()
35
+
36
+ /**
37
+ * @handhold-patch H5 (see sdk/src/agent/VENDOR.md, patch #5)
38
+ * Resolve blacklist/whitelist entries. Upstream mapped fn->value 1:1, which
39
+ * only supports blacklisting exact elements. Handhold must hide its WHOLE
40
+ * widget subtree (#handhold-container and every descendant) from snapshots,
41
+ * so functions may now return element collections (array/NodeList/iterable):
42
+ * results are flattened and nullish results dropped. Entries are re-evaluated
43
+ * on every snapshot, catching dynamically rendered widget UI.
44
+ */
45
+ export const resolveElementList = (list = []) =>
46
+ list.flatMap((item) => {
47
+ const v = typeof item === 'function' ? item() : item
48
+ if (v == null) return []
49
+ if (typeof v === 'object' && typeof v[Symbol.iterator] === 'function') return [...v]
50
+ return [v]
51
+ })
52
+
53
+ export function getFlatTree(config = {}) {
54
+ const viewportExpansion = config.viewportExpansion ?? DEFAULT_VIEWPORT_EXPANSION
55
+
56
+ const resolve = resolveElementList
57
+
58
+ const elements = domTree({
59
+ doHighlightElements: config.highlight ?? false,
60
+ debugMode: false,
61
+ focusHighlightIndex: -1,
62
+ viewportExpansion,
63
+ interactiveBlacklist: resolve(config.interactiveBlacklist),
64
+ interactiveWhitelist: resolve(config.interactiveWhitelist),
65
+ highlightOpacity: config.highlightOpacity ?? 0.0,
66
+ highlightLabelOpacity: config.highlightLabelOpacity ?? 0.1,
67
+ })
68
+
69
+ const currentUrl = window.location.href
70
+ for (const nodeId in elements.map) {
71
+ const node = elements.map[nodeId]
72
+ if (node.isInteractive && node.ref) {
73
+ if (!newElementsCache.has(node.ref)) {
74
+ newElementsCache.set(node.ref, currentUrl)
75
+ node.isNew = true
76
+ }
77
+ }
78
+ }
79
+
80
+ return elements
81
+ }
82
+
83
+ const globRegexCache = new Map()
84
+ function globToRegex(pattern) {
85
+ let regex = globRegexCache.get(pattern)
86
+ if (!regex) {
87
+ const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&')
88
+ regex = new RegExp(`^${escaped.replace(/\*/g, '.*')}$`)
89
+ globRegexCache.set(pattern, regex)
90
+ }
91
+ return regex
92
+ }
93
+
94
+ function matchAttributes(attrs, patterns) {
95
+ const result = {}
96
+ for (const pattern of patterns) {
97
+ if (pattern.includes('*')) {
98
+ const regex = globToRegex(pattern)
99
+ for (const key of Object.keys(attrs)) {
100
+ if (regex.test(key) && attrs[key].trim()) result[key] = attrs[key].trim()
101
+ }
102
+ } else {
103
+ const value = attrs[pattern]
104
+ if (value && value.trim()) result[pattern] = value.trim()
105
+ }
106
+ }
107
+ return result
108
+ }
109
+
110
+ /**
111
+ * @param {Object} flatTree output of getFlatTree / domTree
112
+ * @param {Object} [options]
113
+ * @param {string[]} [options.includeAttributes]
114
+ * @param {boolean} [options.keepSemanticTags=false]
115
+ * @param {(s: string) => string} [options.redactor] defaults to full redaction; pass (s)=>s to disable (NOT recommended)
116
+ * @returns {string}
117
+ */
118
+ export function flatTreeToString(flatTree, options = {}) {
119
+ const {
120
+ includeAttributes = [],
121
+ keepSemanticTags = false,
122
+ redactor = redactText,
123
+ } = options
124
+
125
+ const includeAttrs = [...includeAttributes, ...DEFAULT_INCLUDE_ATTRIBUTES]
126
+
127
+ const capTextLength = (text, maxLength) =>
128
+ text.length > maxLength ? text.substring(0, maxLength) + '...' : text
129
+
130
+ const buildTreeNode = (nodeId) => {
131
+ const node = flatTree.map[nodeId]
132
+ if (!node) return null
133
+
134
+ if (node.type === 'TEXT_NODE') {
135
+ return {
136
+ type: 'text',
137
+ text: redactor(node.text ?? ''),
138
+ isVisible: node.isVisible,
139
+ parent: null,
140
+ children: [],
141
+ }
142
+ }
143
+
144
+ const children = []
145
+ if (node.children) {
146
+ for (const childId of node.children) {
147
+ const child = buildTreeNode(childId)
148
+ if (child) children.push(child)
149
+ }
150
+ }
151
+
152
+ const attributes = {}
153
+ for (const [k, v] of Object.entries(node.attributes ?? {})) {
154
+ attributes[k] = redactor(String(v))
155
+ }
156
+
157
+ return {
158
+ type: 'element',
159
+ tagName: node.tagName,
160
+ attributes,
161
+ isVisible: node.isVisible ?? false,
162
+ isInteractive: node.isInteractive ?? false,
163
+ isTopElement: node.isTopElement ?? false,
164
+ isNew: node.isNew ?? false,
165
+ highlightIndex: node.highlightIndex,
166
+ parent: null,
167
+ children,
168
+ extra: node.extra ?? {},
169
+ }
170
+ }
171
+
172
+ const setParentReferences = (node, parent = null) => {
173
+ node.parent = parent
174
+ for (const child of node.children) setParentReferences(child, node)
175
+ }
176
+
177
+ const rootNode = buildTreeNode(flatTree.rootId)
178
+ if (!rootNode) return ''
179
+ setParentReferences(rootNode)
180
+
181
+ const hasParentWithHighlightIndex = (node) => {
182
+ let current = node.parent
183
+ while (current) {
184
+ if (current.type === 'element' && current.highlightIndex !== undefined) return true
185
+ current = current.parent
186
+ }
187
+ return false
188
+ }
189
+
190
+ const processNode = (node, depth, result) => {
191
+ let nextDepth = depth
192
+ const depthStr = '\t'.repeat(depth)
193
+
194
+ if (node.type === 'element') {
195
+ const isSemantic = keepSemanticTags && node.tagName && SEMANTIC_TAGS.has(node.tagName)
196
+
197
+ if (node.highlightIndex !== undefined) {
198
+ nextDepth += 1
199
+
200
+ const text = getAllTextTillNextClickableElement(node)
201
+ let attributesHtmlStr = ''
202
+
203
+ if (includeAttrs.length > 0 && node.attributes) {
204
+ const attributesToInclude = matchAttributes(node.attributes, includeAttrs)
205
+
206
+ const keys = Object.keys(attributesToInclude)
207
+ if (keys.length > 1) {
208
+ const keysToRemove = new Set()
209
+ const seenValues = {}
210
+ for (const key of keys) {
211
+ const value = attributesToInclude[key]
212
+ if (value.length > 5) {
213
+ if (value in seenValues) keysToRemove.add(key)
214
+ else seenValues[value] = key
215
+ }
216
+ }
217
+ for (const key of keysToRemove) delete attributesToInclude[key]
218
+ }
219
+
220
+ if (attributesToInclude.role === node.tagName) delete attributesToInclude.role
221
+
222
+ for (const attr of ['aria-label', 'placeholder', 'title']) {
223
+ if (
224
+ attributesToInclude[attr] &&
225
+ attributesToInclude[attr].toLowerCase().trim() === text.toLowerCase().trim()
226
+ ) {
227
+ delete attributesToInclude[attr]
228
+ }
229
+ }
230
+
231
+ if (Object.keys(attributesToInclude).length > 0) {
232
+ attributesHtmlStr = Object.entries(attributesToInclude)
233
+ .map(([key, value]) => `${key}=${capTextLength(value, 20)}`)
234
+ .join(' ')
235
+ }
236
+ }
237
+
238
+ const highlightIndicator = node.isNew ? `*[${node.highlightIndex}]` : `[${node.highlightIndex}]`
239
+ let line = `${depthStr}${highlightIndicator}<${node.tagName ?? ''}`
240
+ if (attributesHtmlStr) line += ` ${attributesHtmlStr}`
241
+
242
+ if (node.extra && node.extra.scrollable) {
243
+ let scrollDataText = ''
244
+ if (node.extra.scrollData?.left) scrollDataText += `left=${node.extra.scrollData.left}, `
245
+ if (node.extra.scrollData?.top) scrollDataText += `top=${node.extra.scrollData.top}, `
246
+ if (node.extra.scrollData?.right) scrollDataText += `right=${node.extra.scrollData.right}, `
247
+ if (node.extra.scrollData?.bottom) scrollDataText += `bottom=${node.extra.scrollData.bottom}`
248
+ line += ` data-scrollable="${scrollDataText}"`
249
+ }
250
+
251
+ if (text) {
252
+ const trimmedText = text.trim()
253
+ if (!attributesHtmlStr) line += ' '
254
+ line += `>${trimmedText}`
255
+ } else if (!attributesHtmlStr) {
256
+ line += ' '
257
+ }
258
+
259
+ line += ' />'
260
+ result.push(line)
261
+ }
262
+
263
+ const emitSemantic = isSemantic && node.highlightIndex === undefined
264
+ const mark = emitSemantic ? result.length : -1
265
+ if (emitSemantic) {
266
+ result.push(`${depthStr}<${node.tagName}>`)
267
+ nextDepth += 1
268
+ }
269
+
270
+ for (const child of node.children) processNode(child, nextDepth, result)
271
+
272
+ if (emitSemantic) {
273
+ if (result.length === mark + 1) result.pop()
274
+ else result.push(`${depthStr}</${node.tagName}>`)
275
+ }
276
+ } else if (node.type === 'text') {
277
+ if (hasParentWithHighlightIndex(node)) return
278
+ if (node.parent && node.parent.type === 'element' && node.parent.isVisible && node.parent.isTopElement) {
279
+ result.push(`${depthStr}${node.text ?? ''}`)
280
+ }
281
+ }
282
+ }
283
+
284
+ const result = []
285
+ processNode(rootNode, 0, result)
286
+ return result.join('\n')
287
+ }
288
+
289
+ export const getAllTextTillNextClickableElement = (node, maxDepth = -1) => {
290
+ const textParts = []
291
+ const collectText = (currentNode, currentDepth) => {
292
+ if (maxDepth !== -1 && currentDepth > maxDepth) return
293
+ if (currentNode.type === 'element' && currentNode !== node && currentNode.highlightIndex !== undefined) return
294
+ if (currentNode.type === 'text' && currentNode.text) {
295
+ textParts.push(currentNode.text)
296
+ } else if (currentNode.type === 'element') {
297
+ for (const child of currentNode.children) collectText(child, currentDepth + 1)
298
+ }
299
+ }
300
+ collectText(node, 0)
301
+ return textParts.join('\n').trim()
302
+ }
303
+
304
+ export function getSelectorMap(flatTree) {
305
+ const selectorMap = new Map()
306
+ for (const key of Object.keys(flatTree.map)) {
307
+ const node = flatTree.map[key]
308
+ if (node.isInteractive && typeof node.highlightIndex === 'number') {
309
+ selectorMap.set(node.highlightIndex, node)
310
+ }
311
+ }
312
+ return selectorMap
313
+ }
314
+
315
+ export function cleanUpHighlights() {
316
+ const cleanupFunctions = window._highlightCleanupFunctions || []
317
+ for (const cleanup of cleanupFunctions) {
318
+ if (typeof cleanup === 'function') cleanup()
319
+ }
320
+ window._highlightCleanupFunctions = []
321
+ }
322
+
323
+ /** Call once from the controller. No import-time listeners (unlike upstream). */
324
+ export function installNavigationCleanup() {
325
+ window.addEventListener('popstate', cleanUpHighlights)
326
+ window.addEventListener('hashchange', cleanUpHighlights)
327
+ window.addEventListener('beforeunload', cleanUpHighlights)
328
+ const navigation = window.navigation
329
+ if (navigation && typeof navigation.addEventListener === 'function') {
330
+ navigation.addEventListener('navigate', cleanUpHighlights)
331
+ }
332
+ }
@@ -0,0 +1,99 @@
1
+ /**
2
+ * utils.js — ported from alibaba/page-agent packages/page-controller/src/utils/index.ts (MIT).
3
+ * Changes vs upstream:
4
+ * - Pointer/mask CustomEvents removed. Upstream dispatched `PageAgent::MovePointerTo`
5
+ * etc. to drive a visual cursor overlay; here those are optional hooks the host
6
+ * app can register (see setPointerHooks). No hook → no-op, no artificial wait.
7
+ */
8
+
9
+ // ======= type guards =======
10
+ // @note instanceof fails for elements inside iframes, hence nodeType/tagName checks.
11
+
12
+ export function isHTMLElement(el) {
13
+ return !!el && el.nodeType === 1
14
+ }
15
+
16
+ export function isInputElement(el) {
17
+ return el?.nodeType === 1 && el.tagName === 'INPUT'
18
+ }
19
+
20
+ export function isTextAreaElement(el) {
21
+ return el?.nodeType === 1 && el.tagName === 'TEXTAREA'
22
+ }
23
+
24
+ export function isSelectElement(el) {
25
+ return el?.nodeType === 1 && el.tagName === 'SELECT'
26
+ }
27
+
28
+ export function isAnchorElement(el) {
29
+ return el?.nodeType === 1 && el.tagName === 'A'
30
+ }
31
+
32
+ // ======= iframe helpers =======
33
+
34
+ /** Iframe offset for translating element coordinates to top-frame viewport. */
35
+ export function getIframeOffset(element) {
36
+ const frame = element.ownerDocument.defaultView?.frameElement
37
+ if (!frame) return { x: 0, y: 0 }
38
+ const rect = frame.getBoundingClientRect()
39
+ return { x: rect.left, y: rect.top }
40
+ }
41
+
42
+ /**
43
+ * Get native value setter from the element's own prototype (iframe-safe).
44
+ * Required so React's value-tracking sees the change (React patches the
45
+ * instance property; the prototype setter bypasses the patch, then the
46
+ * dispatched 'input' event makes React re-read the value).
47
+ */
48
+ export function getNativeValueSetter(element) {
49
+ return Object.getOwnPropertyDescriptor(Object.getPrototypeOf(element), 'value').set
50
+ }
51
+
52
+ // ======= general utils =======
53
+
54
+ export async function waitFor(seconds, signal) {
55
+ if (signal?.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError')
56
+ await new Promise((resolve, reject) => {
57
+ const t = setTimeout(done, seconds * 1000)
58
+ function done() {
59
+ signal?.removeEventListener('abort', onAbort)
60
+ resolve()
61
+ }
62
+ function onAbort() {
63
+ clearTimeout(t)
64
+ reject(signal.reason ?? new DOMException('Aborted', 'AbortError'))
65
+ }
66
+ signal?.addEventListener('abort', onAbort, { once: true })
67
+ })
68
+ }
69
+
70
+ // ======= optional pointer-visual hooks (replaces upstream mask CustomEvents) =======
71
+
72
+ let pointerHooks = null
73
+
74
+ /**
75
+ * Register visual hooks: { movePointerTo(x,y), clickPointer(), enablePassThrough(), disablePassThrough() }
76
+ * All optional. Used purely for UX (showing a fake cursor); never load-bearing.
77
+ */
78
+ export function setPointerHooks(hooks) {
79
+ pointerHooks = hooks
80
+ }
81
+
82
+ export async function movePointerToElement(element, x, y) {
83
+ if (!pointerHooks?.movePointerTo) return
84
+ const offset = getIframeOffset(element)
85
+ pointerHooks.movePointerTo(x + offset.x, y + offset.y)
86
+ await waitFor(0.3)
87
+ }
88
+
89
+ export async function clickPointer() {
90
+ pointerHooks?.clickPointer?.()
91
+ }
92
+
93
+ export async function enablePassThrough() {
94
+ pointerHooks?.enablePassThrough?.()
95
+ }
96
+
97
+ export async function disablePassThrough() {
98
+ pointerHooks?.disablePassThrough?.()
99
+ }
@@ -0,0 +1,169 @@
1
+ /**
2
+ * trusted-page-agent — a security-first rebuild of alibaba/page-agent.
3
+ *
4
+ * Kept from upstream (MIT, audited): the browser-use DOM extraction engine
5
+ * (complex layouts, occlusion hit-testing, iframes, shadow DOM, scrollable
6
+ * regions), the serialization format, and the React-aware action dispatch.
7
+ *
8
+ * Replaced/added: no eval tool of any kind, mandatory origin allowlist,
9
+ * fail-closed human confirmation for sensitive actions, PII redaction before
10
+ * page text reaches the LLM, audit log, rate limit, prompt-injection
11
+ * hardening, and an LLM connector that takes either a hosted API key or a
12
+ * local model URL (Ollama / LM Studio / vLLM / any OpenAI-compatible server).
13
+ *
14
+ * @example Hosted (Anthropic):
15
+ * const agent = new TrustedPageAgent({
16
+ * llm: { provider: 'anthropic', apiKey: '...', model: 'claude-sonnet-4-6' },
17
+ * allowedOrigins: [location.origin],
18
+ * onConfirm: async (p) => confirm(`Allow: ${p.reason}?`),
19
+ * })
20
+ * const result = await agent.run('Fill the signup form with test data')
21
+ *
22
+ * @example Local (Ollama):
23
+ * const agent = new TrustedPageAgent({
24
+ * llm: { provider: 'openai-compatible', baseURL: 'http://localhost:11434/v1', model: 'qwen3:14b' },
25
+ * allowedOrigins: [location.origin],
26
+ * onConfirm: async (p) => confirm(`Allow: ${p.reason}?`),
27
+ * })
28
+ */
29
+ import { AgentLoop } from './agent/AgentLoop.js'
30
+ import { GuardedController } from './controller.js'
31
+ import { createLLM } from './llm/connector.js'
32
+ import { PolicyGuard } from './policy/PolicyGuard.js'
33
+ // @handhold-patch H6 (VENDOR.md): whole-site support (reviewed manifest + navigate).
34
+ import { loadSiteManifest } from './site/manifest.js'
35
+
36
+ export { AgentLoop, parseAgentOutput } from './agent/AgentLoop.js'
37
+ export { TOOLS, validateToolInput } from './agent/tools.js'
38
+ export { GuardedController, domSnapshotProvider } from './controller.js'
39
+ export { createRedactor, redactText } from './dom/redact.js'
40
+ export { flatTreeToString, getFlatTree, getSelectorMap } from './dom/serializer.js'
41
+ export { setPointerHooks } from './dom/utils.js'
42
+ export { createLLM } from './llm/connector.js'
43
+ export { PolicyGuard } from './policy/PolicyGuard.js'
44
+ export { ChatWidget } from './ui/ChatWidget.js'
45
+
46
+ export class TrustedPageAgent {
47
+ #loop
48
+ #controller
49
+
50
+ /**
51
+ * @param {Object} cfg
52
+ * @param {Object} cfg.llm createLLM() config: {provider, baseURL?, apiKey?, model, ...}
53
+ * @param {string[]} cfg.allowedOrigins REQUIRED — origins the agent may act on
54
+ * @param {(pending:{name,input,reason}) => Promise<boolean>} [cfg.onConfirm]
55
+ * Human gate for sensitive actions. Without it, sensitive actions are blocked.
56
+ * @param {(question:string) => Promise<string>} [cfg.onAskUser]
57
+ * @param {(ev:Object) => void} [cfg.onActivity] UI feedback (thinking/executing/executed/done)
58
+ * @param {number} [cfg.maxSteps=25]
59
+ * @param {number} [cfg.maxActionsPerMinute=30]
60
+ * @param {string[]} [cfg.extraSensitiveKeywords]
61
+ * @param {Object} [cfg.domConfig] engine options: viewportExpansion, interactiveBlacklist,
62
+ * includeAttributes, redactor, keepSemanticTags...
63
+ * @param {Object} [cfg.siteManifest] @handhold-patch H6: RAW manifest JSON as served by
64
+ * GET /api/v1/manifest. Validated here — construction
65
+ * THROWS unless reviewed === true (client-side gate).
66
+ * Without it the agent stays single-page.
67
+ * @param {(url:string)=>void|Promise<void>} [cfg.navigateImpl] injectable navigation
68
+ */
69
+ constructor(cfg = {}) {
70
+ if (!cfg.llm) throw new Error('TrustedPageAgent: cfg.llm is required (see createLLM).')
71
+
72
+ const guard = new PolicyGuard({
73
+ allowedOrigins: cfg.allowedOrigins,
74
+ onConfirm: cfg.onConfirm,
75
+ extraSensitiveKeywords: cfg.extraSensitiveKeywords,
76
+ maxActionsPerMinute: cfg.maxActionsPerMinute,
77
+ })
78
+
79
+ // @handhold-patch H6: validate the manifest at the trust boundary. An
80
+ // unreviewed or malformed manifest must never configure the agent.
81
+ const siteManifest = cfg.siteManifest ? loadSiteManifest(cfg.siteManifest) : null
82
+
83
+ this.#controller = new GuardedController({
84
+ guard,
85
+ domConfig: cfg.domConfig,
86
+ siteManifest,
87
+ navigateImpl: cfg.navigateImpl,
88
+ })
89
+
90
+ this.#loop = new AgentLoop({
91
+ llm: createLLM(cfg.llm),
92
+ controller: this.#controller,
93
+ maxSteps: cfg.maxSteps,
94
+ onAskUser: cfg.onAskUser,
95
+ onActivity: cfg.onActivity,
96
+ siteManifest,
97
+ })
98
+ }
99
+
100
+ /** Run a natural-language task. Resolves with {success, text, steps}. */
101
+ run(task, opts) {
102
+ return this.#loop.run(task, opts)
103
+ }
104
+
105
+ /** Cooperative stop; the current step finishes, no further actions run. */
106
+ stop() {
107
+ this.#loop.stop()
108
+ }
109
+
110
+ /** Timestamped record of every action considered/allowed/blocked. */
111
+ getAuditLog() {
112
+ return this.#controller.getAuditLog()
113
+ }
114
+
115
+ /**
116
+ * @handhold-patch H7 (VENDOR.md): run snapshot for cross-page checkpoints —
117
+ * {history, stepsUsed} from the loop plus the guard's audit log.
118
+ */
119
+ getRunState() {
120
+ return { ...this.#loop.getRunState(), audit: this.#controller.getAuditLog() }
121
+ }
122
+ }
123
+
124
+ /**
125
+ * One-liner: mount the chat-bubble UI and wire it to an agent.
126
+ * The widget becomes the human gate (Allow/Deny buttons), the ask_user
127
+ * channel, and the activity feed. Any onConfirm/onAskUser/onActivity you
128
+ * pass in agentConfig still wins over the widget defaults.
129
+ *
130
+ * @example
131
+ * attachChatWidget({
132
+ * llm: { provider: 'deepseek', apiKey: 'sk-...', model: 'deepseek-chat' },
133
+ * allowedOrigins: [location.origin],
134
+ * })
135
+ *
136
+ * @returns {{ agent: TrustedPageAgent, widget: ChatWidget }}
137
+ */
138
+ export function attachChatWidget(agentConfig, widgetOptions = {}) {
139
+ let agent = null
140
+ const widget = new ChatWidgetClass({
141
+ ...widgetOptions,
142
+ onTask: async (task) => {
143
+ widget.setBusy(true)
144
+ try {
145
+ const result = await agent.run(task)
146
+ widget.addMessage('agent', `${result.success ? '✅' : '⚠️'} ${result.text}`)
147
+ } catch (e) {
148
+ widget.addMessage('agent', `⚠️ Error: ${e.message}`)
149
+ } finally {
150
+ widget.setBusy(false)
151
+ }
152
+ },
153
+ })
154
+ widget.mount()
155
+
156
+ agent = new TrustedPageAgent({
157
+ onConfirm: (p) => widget.requestConfirm(p),
158
+ onAskUser: (q) => widget.askUser(q),
159
+ onActivity: (ev) => widget.showActivity(ev),
160
+ ...agentConfig,
161
+ })
162
+
163
+ return { agent, widget }
164
+ }
165
+
166
+ // internal alias so attachChatWidget can construct without a circular re-import
167
+ import { ChatWidget as ChatWidgetClass } from './ui/ChatWidget.js'
168
+
169
+ export default TrustedPageAgent