handhold-sdk 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +427 -0
- package/dist/d44f1fe8001e32f8e74d.svg +3056 -0
- package/dist/fonts/Phosphor.ttf +0 -0
- package/dist/fonts/Phosphor.woff +0 -0
- package/dist/fonts/Phosphor.woff2 +0 -0
- package/dist/handhold.min.js +1 -0
- package/package.json +114 -0
- package/src/HandHold.js +1599 -0
- package/src/agent/VENDOR.md +203 -0
- package/src/agent/agent/AgentLoop.js +366 -0
- package/src/agent/agent/prompts.js +163 -0
- package/src/agent/agent/tools.js +148 -0
- package/src/agent/controller.js +235 -0
- package/src/agent/dom/actions.js +456 -0
- package/src/agent/dom/domTree.js +1761 -0
- package/src/agent/dom/redact.js +98 -0
- package/src/agent/dom/serializer.js +332 -0
- package/src/agent/dom/utils.js +99 -0
- package/src/agent/index.js +169 -0
- package/src/agent/llm/connector.js +143 -0
- package/src/agent/package.json +3 -0
- package/src/agent/policy/PolicyGuard.js +172 -0
- package/src/agent/site/manifest.js +145 -0
- package/src/agent/ui/ChatWidget.js +219 -0
- package/src/agent-mode/AgentMode.js +429 -0
- package/src/api/APIClient.js +258 -0
- package/src/core/Config.js +207 -0
- package/src/core/UiState.js +83 -0
- package/src/core/defaults.js +20 -0
- package/src/events.js +59 -0
- package/src/index.js +77 -0
- package/src/intent/ActionVerbMap.js +324 -0
- package/src/intent/IntentExtractor.js +317 -0
- package/src/observers/ElementScanner.js +284 -0
- package/src/observers/NavigationObserver.js +182 -0
- package/src/observers/PageObserver.js +293 -0
- package/src/session/SessionManager.js +402 -0
- package/src/session/SessionStorage.js +148 -0
- package/src/ui/HelpWidget.js +1189 -0
- package/src/ui/UICoach.js +1725 -0
- package/src/ui/components/Button.css +137 -0
- package/src/ui/components/Button.js +102 -0
- package/src/ui/widget.css +599 -0
- package/src/utils/Logger.js +132 -0
- package/src/utils/MarkdownRenderer.js +39 -0
- package/src/utils/domUtils.js +350 -0
- package/src/utils/eventBus.js +102 -0
- package/src/utils/index.js +8 -0
- package/src/utils/stringUtils.js +202 -0
- package/types/index.d.ts +538 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* redact.js — PII redaction applied to page text/attributes BEFORE they are
|
|
3
|
+
* serialized into the LLM prompt.
|
|
4
|
+
*
|
|
5
|
+
* Upstream (alibaba/page-agent) left this as an open TODO in dom/index.ts
|
|
6
|
+
* ("@todo 数据脱敏过滤器" — data-masking filter) and their own
|
|
7
|
+
* terms-and-privacy.md admits HTML cleaning "does not guarantee removal of
|
|
8
|
+
* sensitive information". This module is the missing piece.
|
|
9
|
+
*
|
|
10
|
+
* Design notes:
|
|
11
|
+
* - Redaction is ON by default. Secure by default; opt out per category.
|
|
12
|
+
* - Card numbers are only redacted when Luhn-valid, so 16-digit order ids
|
|
13
|
+
* and tracking numbers survive (a naive \d{16} rule destroys e-commerce
|
|
14
|
+
* pages and makes the agent useless on them).
|
|
15
|
+
* - This is a best-effort net, not a guarantee. Do not point the agent at
|
|
16
|
+
* pages containing data you are not allowed to send to your LLM provider.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
function luhnValid(digits) {
|
|
20
|
+
let sum = 0
|
|
21
|
+
let alt = false
|
|
22
|
+
for (let i = digits.length - 1; i >= 0; i--) {
|
|
23
|
+
let d = digits.charCodeAt(i) - 48
|
|
24
|
+
if (alt) {
|
|
25
|
+
d *= 2
|
|
26
|
+
if (d > 9) d -= 9
|
|
27
|
+
}
|
|
28
|
+
sum += d
|
|
29
|
+
alt = !alt
|
|
30
|
+
}
|
|
31
|
+
return sum % 10 === 0
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g
|
|
35
|
+
|
|
36
|
+
// Phones must show phone-ish intent: a leading +country code, parentheses,
|
|
37
|
+
// or dash/dot separators. Plain space-separated digit groups (order numbers,
|
|
38
|
+
// non-Luhn ids) are deliberately NOT treated as phones.
|
|
39
|
+
const PHONE_RE = /(?:\+\d{1,3}(?:[ .-]?\d{2,4}){2,4}|(?:\(\d{2,4}\)[ .-]?)\d{3,4}[ .-]\d{3,4}|\d{3,4}[.-]\d{3,4}[.-]\d{3,4})/g
|
|
40
|
+
|
|
41
|
+
// Last char must be a digit so trailing whitespace is never consumed.
|
|
42
|
+
const CARD_CANDIDATE_RE = /\b\d(?:[ -]?\d){12,18}\b/g
|
|
43
|
+
|
|
44
|
+
const SSN_RE = /\b\d{3}-\d{2}-\d{4}\b/g
|
|
45
|
+
|
|
46
|
+
function countDigits(s) {
|
|
47
|
+
let n = 0
|
|
48
|
+
for (const ch of s) if (ch >= '0' && ch <= '9') n++
|
|
49
|
+
return n
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* @param {Object} [options]
|
|
54
|
+
* @param {boolean} [options.email=true]
|
|
55
|
+
* @param {boolean} [options.phone=true]
|
|
56
|
+
* @param {boolean} [options.card=true]
|
|
57
|
+
* @param {boolean} [options.ssn=true]
|
|
58
|
+
* @param {{pattern: RegExp, label: string}[]} [options.extraPatterns]
|
|
59
|
+
* @returns {(text: string) => string}
|
|
60
|
+
*/
|
|
61
|
+
export function createRedactor(options = {}) {
|
|
62
|
+
const { email = true, phone = true, card = true, ssn = true, extraPatterns = [] } = options
|
|
63
|
+
|
|
64
|
+
return function redact(text) {
|
|
65
|
+
if (typeof text !== 'string' || text.length === 0) return text
|
|
66
|
+
let out = text
|
|
67
|
+
|
|
68
|
+
if (email) out = out.replace(EMAIL_RE, '[EMAIL]')
|
|
69
|
+
if (ssn) out = out.replace(SSN_RE, '[SSN]')
|
|
70
|
+
|
|
71
|
+
if (card) {
|
|
72
|
+
out = out.replace(CARD_CANDIDATE_RE, (match) => {
|
|
73
|
+
const digits = match.replace(/[ -]/g, '')
|
|
74
|
+
if (digits.length >= 13 && digits.length <= 19 && luhnValid(digits)) return '[CARD]'
|
|
75
|
+
return match
|
|
76
|
+
})
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (phone) {
|
|
80
|
+
out = out.replace(PHONE_RE, (match) => {
|
|
81
|
+
// require enough digits to plausibly be a phone number,
|
|
82
|
+
// and at least one separator (so plain long numbers pass through
|
|
83
|
+
// to the card check instead)
|
|
84
|
+
if (countDigits(match) >= 9 && /[ .-]/.test(match)) return '[PHONE]'
|
|
85
|
+
return match
|
|
86
|
+
})
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
for (const { pattern, label } of extraPatterns) {
|
|
90
|
+
out = out.replace(pattern, label)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return out
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Default redactor with everything on. */
|
|
98
|
+
export const redactText = createRedactor()
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* serializer.js — turns the domTree engine's flat tree into (1) the indexed
|
|
3
|
+
* text snapshot the LLM reads and (2) a selector map of index → live element.
|
|
4
|
+
*
|
|
5
|
+
* Ported from alibaba/page-agent packages/page-controller/src/dom/index.ts (MIT),
|
|
6
|
+
* with two deliberate changes:
|
|
7
|
+
* 1. REDACTION (new): every text fragment and attribute value passes through
|
|
8
|
+
* a redactor before entering the snapshot. Upstream left this as an open
|
|
9
|
+
* TODO ("@todo 数据脱敏过滤器"). Default redactor strips emails, phones,
|
|
10
|
+
* Luhn-valid card numbers, SSNs. See dom/redact.js.
|
|
11
|
+
* 2. No import-time side effects: upstream registers window listeners at
|
|
12
|
+
* module load; here installNavigationCleanup() must be called explicitly.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import domTree from './domTree.js'
|
|
16
|
+
import { redactText } from './redact.js'
|
|
17
|
+
|
|
18
|
+
const DEFAULT_VIEWPORT_EXPANSION = -1
|
|
19
|
+
|
|
20
|
+
const SEMANTIC_TAGS = new Set(['nav', 'menu', 'header', 'footer', 'aside', 'dialog'])
|
|
21
|
+
|
|
22
|
+
const DEFAULT_INCLUDE_ATTRIBUTES = [
|
|
23
|
+
'title', 'type', 'checked', 'name', 'role', 'value', 'placeholder',
|
|
24
|
+
'data-date-format', 'alt', 'aria-label', 'aria-expanded', 'data-state',
|
|
25
|
+
'aria-checked',
|
|
26
|
+
'id', 'for',
|
|
27
|
+
'target',
|
|
28
|
+
'aria-haspopup', 'aria-controls', 'aria-owns',
|
|
29
|
+
'contenteditable',
|
|
30
|
+
// needed by PolicyGuard's credential/payment detection
|
|
31
|
+
'autocomplete',
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
const newElementsCache = new WeakMap()
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @handhold-patch H5 (see sdk/src/agent/VENDOR.md, patch #5)
|
|
38
|
+
* Resolve blacklist/whitelist entries. Upstream mapped fn->value 1:1, which
|
|
39
|
+
* only supports blacklisting exact elements. Handhold must hide its WHOLE
|
|
40
|
+
* widget subtree (#handhold-container and every descendant) from snapshots,
|
|
41
|
+
* so functions may now return element collections (array/NodeList/iterable):
|
|
42
|
+
* results are flattened and nullish results dropped. Entries are re-evaluated
|
|
43
|
+
* on every snapshot, catching dynamically rendered widget UI.
|
|
44
|
+
*/
|
|
45
|
+
export const resolveElementList = (list = []) =>
|
|
46
|
+
list.flatMap((item) => {
|
|
47
|
+
const v = typeof item === 'function' ? item() : item
|
|
48
|
+
if (v == null) return []
|
|
49
|
+
if (typeof v === 'object' && typeof v[Symbol.iterator] === 'function') return [...v]
|
|
50
|
+
return [v]
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
export function getFlatTree(config = {}) {
|
|
54
|
+
const viewportExpansion = config.viewportExpansion ?? DEFAULT_VIEWPORT_EXPANSION
|
|
55
|
+
|
|
56
|
+
const resolve = resolveElementList
|
|
57
|
+
|
|
58
|
+
const elements = domTree({
|
|
59
|
+
doHighlightElements: config.highlight ?? false,
|
|
60
|
+
debugMode: false,
|
|
61
|
+
focusHighlightIndex: -1,
|
|
62
|
+
viewportExpansion,
|
|
63
|
+
interactiveBlacklist: resolve(config.interactiveBlacklist),
|
|
64
|
+
interactiveWhitelist: resolve(config.interactiveWhitelist),
|
|
65
|
+
highlightOpacity: config.highlightOpacity ?? 0.0,
|
|
66
|
+
highlightLabelOpacity: config.highlightLabelOpacity ?? 0.1,
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
const currentUrl = window.location.href
|
|
70
|
+
for (const nodeId in elements.map) {
|
|
71
|
+
const node = elements.map[nodeId]
|
|
72
|
+
if (node.isInteractive && node.ref) {
|
|
73
|
+
if (!newElementsCache.has(node.ref)) {
|
|
74
|
+
newElementsCache.set(node.ref, currentUrl)
|
|
75
|
+
node.isNew = true
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return elements
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const globRegexCache = new Map()
|
|
84
|
+
function globToRegex(pattern) {
|
|
85
|
+
let regex = globRegexCache.get(pattern)
|
|
86
|
+
if (!regex) {
|
|
87
|
+
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
|
88
|
+
regex = new RegExp(`^${escaped.replace(/\*/g, '.*')}$`)
|
|
89
|
+
globRegexCache.set(pattern, regex)
|
|
90
|
+
}
|
|
91
|
+
return regex
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function matchAttributes(attrs, patterns) {
|
|
95
|
+
const result = {}
|
|
96
|
+
for (const pattern of patterns) {
|
|
97
|
+
if (pattern.includes('*')) {
|
|
98
|
+
const regex = globToRegex(pattern)
|
|
99
|
+
for (const key of Object.keys(attrs)) {
|
|
100
|
+
if (regex.test(key) && attrs[key].trim()) result[key] = attrs[key].trim()
|
|
101
|
+
}
|
|
102
|
+
} else {
|
|
103
|
+
const value = attrs[pattern]
|
|
104
|
+
if (value && value.trim()) result[pattern] = value.trim()
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return result
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* @param {Object} flatTree output of getFlatTree / domTree
|
|
112
|
+
* @param {Object} [options]
|
|
113
|
+
* @param {string[]} [options.includeAttributes]
|
|
114
|
+
* @param {boolean} [options.keepSemanticTags=false]
|
|
115
|
+
* @param {(s: string) => string} [options.redactor] defaults to full redaction; pass (s)=>s to disable (NOT recommended)
|
|
116
|
+
* @returns {string}
|
|
117
|
+
*/
|
|
118
|
+
export function flatTreeToString(flatTree, options = {}) {
|
|
119
|
+
const {
|
|
120
|
+
includeAttributes = [],
|
|
121
|
+
keepSemanticTags = false,
|
|
122
|
+
redactor = redactText,
|
|
123
|
+
} = options
|
|
124
|
+
|
|
125
|
+
const includeAttrs = [...includeAttributes, ...DEFAULT_INCLUDE_ATTRIBUTES]
|
|
126
|
+
|
|
127
|
+
const capTextLength = (text, maxLength) =>
|
|
128
|
+
text.length > maxLength ? text.substring(0, maxLength) + '...' : text
|
|
129
|
+
|
|
130
|
+
const buildTreeNode = (nodeId) => {
|
|
131
|
+
const node = flatTree.map[nodeId]
|
|
132
|
+
if (!node) return null
|
|
133
|
+
|
|
134
|
+
if (node.type === 'TEXT_NODE') {
|
|
135
|
+
return {
|
|
136
|
+
type: 'text',
|
|
137
|
+
text: redactor(node.text ?? ''),
|
|
138
|
+
isVisible: node.isVisible,
|
|
139
|
+
parent: null,
|
|
140
|
+
children: [],
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const children = []
|
|
145
|
+
if (node.children) {
|
|
146
|
+
for (const childId of node.children) {
|
|
147
|
+
const child = buildTreeNode(childId)
|
|
148
|
+
if (child) children.push(child)
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const attributes = {}
|
|
153
|
+
for (const [k, v] of Object.entries(node.attributes ?? {})) {
|
|
154
|
+
attributes[k] = redactor(String(v))
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
type: 'element',
|
|
159
|
+
tagName: node.tagName,
|
|
160
|
+
attributes,
|
|
161
|
+
isVisible: node.isVisible ?? false,
|
|
162
|
+
isInteractive: node.isInteractive ?? false,
|
|
163
|
+
isTopElement: node.isTopElement ?? false,
|
|
164
|
+
isNew: node.isNew ?? false,
|
|
165
|
+
highlightIndex: node.highlightIndex,
|
|
166
|
+
parent: null,
|
|
167
|
+
children,
|
|
168
|
+
extra: node.extra ?? {},
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const setParentReferences = (node, parent = null) => {
|
|
173
|
+
node.parent = parent
|
|
174
|
+
for (const child of node.children) setParentReferences(child, node)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const rootNode = buildTreeNode(flatTree.rootId)
|
|
178
|
+
if (!rootNode) return ''
|
|
179
|
+
setParentReferences(rootNode)
|
|
180
|
+
|
|
181
|
+
const hasParentWithHighlightIndex = (node) => {
|
|
182
|
+
let current = node.parent
|
|
183
|
+
while (current) {
|
|
184
|
+
if (current.type === 'element' && current.highlightIndex !== undefined) return true
|
|
185
|
+
current = current.parent
|
|
186
|
+
}
|
|
187
|
+
return false
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const processNode = (node, depth, result) => {
|
|
191
|
+
let nextDepth = depth
|
|
192
|
+
const depthStr = '\t'.repeat(depth)
|
|
193
|
+
|
|
194
|
+
if (node.type === 'element') {
|
|
195
|
+
const isSemantic = keepSemanticTags && node.tagName && SEMANTIC_TAGS.has(node.tagName)
|
|
196
|
+
|
|
197
|
+
if (node.highlightIndex !== undefined) {
|
|
198
|
+
nextDepth += 1
|
|
199
|
+
|
|
200
|
+
const text = getAllTextTillNextClickableElement(node)
|
|
201
|
+
let attributesHtmlStr = ''
|
|
202
|
+
|
|
203
|
+
if (includeAttrs.length > 0 && node.attributes) {
|
|
204
|
+
const attributesToInclude = matchAttributes(node.attributes, includeAttrs)
|
|
205
|
+
|
|
206
|
+
const keys = Object.keys(attributesToInclude)
|
|
207
|
+
if (keys.length > 1) {
|
|
208
|
+
const keysToRemove = new Set()
|
|
209
|
+
const seenValues = {}
|
|
210
|
+
for (const key of keys) {
|
|
211
|
+
const value = attributesToInclude[key]
|
|
212
|
+
if (value.length > 5) {
|
|
213
|
+
if (value in seenValues) keysToRemove.add(key)
|
|
214
|
+
else seenValues[value] = key
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
for (const key of keysToRemove) delete attributesToInclude[key]
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
if (attributesToInclude.role === node.tagName) delete attributesToInclude.role
|
|
221
|
+
|
|
222
|
+
for (const attr of ['aria-label', 'placeholder', 'title']) {
|
|
223
|
+
if (
|
|
224
|
+
attributesToInclude[attr] &&
|
|
225
|
+
attributesToInclude[attr].toLowerCase().trim() === text.toLowerCase().trim()
|
|
226
|
+
) {
|
|
227
|
+
delete attributesToInclude[attr]
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
if (Object.keys(attributesToInclude).length > 0) {
|
|
232
|
+
attributesHtmlStr = Object.entries(attributesToInclude)
|
|
233
|
+
.map(([key, value]) => `${key}=${capTextLength(value, 20)}`)
|
|
234
|
+
.join(' ')
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const highlightIndicator = node.isNew ? `*[${node.highlightIndex}]` : `[${node.highlightIndex}]`
|
|
239
|
+
let line = `${depthStr}${highlightIndicator}<${node.tagName ?? ''}`
|
|
240
|
+
if (attributesHtmlStr) line += ` ${attributesHtmlStr}`
|
|
241
|
+
|
|
242
|
+
if (node.extra && node.extra.scrollable) {
|
|
243
|
+
let scrollDataText = ''
|
|
244
|
+
if (node.extra.scrollData?.left) scrollDataText += `left=${node.extra.scrollData.left}, `
|
|
245
|
+
if (node.extra.scrollData?.top) scrollDataText += `top=${node.extra.scrollData.top}, `
|
|
246
|
+
if (node.extra.scrollData?.right) scrollDataText += `right=${node.extra.scrollData.right}, `
|
|
247
|
+
if (node.extra.scrollData?.bottom) scrollDataText += `bottom=${node.extra.scrollData.bottom}`
|
|
248
|
+
line += ` data-scrollable="${scrollDataText}"`
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
if (text) {
|
|
252
|
+
const trimmedText = text.trim()
|
|
253
|
+
if (!attributesHtmlStr) line += ' '
|
|
254
|
+
line += `>${trimmedText}`
|
|
255
|
+
} else if (!attributesHtmlStr) {
|
|
256
|
+
line += ' '
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
line += ' />'
|
|
260
|
+
result.push(line)
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const emitSemantic = isSemantic && node.highlightIndex === undefined
|
|
264
|
+
const mark = emitSemantic ? result.length : -1
|
|
265
|
+
if (emitSemantic) {
|
|
266
|
+
result.push(`${depthStr}<${node.tagName}>`)
|
|
267
|
+
nextDepth += 1
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
for (const child of node.children) processNode(child, nextDepth, result)
|
|
271
|
+
|
|
272
|
+
if (emitSemantic) {
|
|
273
|
+
if (result.length === mark + 1) result.pop()
|
|
274
|
+
else result.push(`${depthStr}</${node.tagName}>`)
|
|
275
|
+
}
|
|
276
|
+
} else if (node.type === 'text') {
|
|
277
|
+
if (hasParentWithHighlightIndex(node)) return
|
|
278
|
+
if (node.parent && node.parent.type === 'element' && node.parent.isVisible && node.parent.isTopElement) {
|
|
279
|
+
result.push(`${depthStr}${node.text ?? ''}`)
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
const result = []
|
|
285
|
+
processNode(rootNode, 0, result)
|
|
286
|
+
return result.join('\n')
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
export const getAllTextTillNextClickableElement = (node, maxDepth = -1) => {
|
|
290
|
+
const textParts = []
|
|
291
|
+
const collectText = (currentNode, currentDepth) => {
|
|
292
|
+
if (maxDepth !== -1 && currentDepth > maxDepth) return
|
|
293
|
+
if (currentNode.type === 'element' && currentNode !== node && currentNode.highlightIndex !== undefined) return
|
|
294
|
+
if (currentNode.type === 'text' && currentNode.text) {
|
|
295
|
+
textParts.push(currentNode.text)
|
|
296
|
+
} else if (currentNode.type === 'element') {
|
|
297
|
+
for (const child of currentNode.children) collectText(child, currentDepth + 1)
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
collectText(node, 0)
|
|
301
|
+
return textParts.join('\n').trim()
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
export function getSelectorMap(flatTree) {
|
|
305
|
+
const selectorMap = new Map()
|
|
306
|
+
for (const key of Object.keys(flatTree.map)) {
|
|
307
|
+
const node = flatTree.map[key]
|
|
308
|
+
if (node.isInteractive && typeof node.highlightIndex === 'number') {
|
|
309
|
+
selectorMap.set(node.highlightIndex, node)
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
return selectorMap
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
export function cleanUpHighlights() {
|
|
316
|
+
const cleanupFunctions = window._highlightCleanupFunctions || []
|
|
317
|
+
for (const cleanup of cleanupFunctions) {
|
|
318
|
+
if (typeof cleanup === 'function') cleanup()
|
|
319
|
+
}
|
|
320
|
+
window._highlightCleanupFunctions = []
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/** Call once from the controller. No import-time listeners (unlike upstream). */
|
|
324
|
+
export function installNavigationCleanup() {
|
|
325
|
+
window.addEventListener('popstate', cleanUpHighlights)
|
|
326
|
+
window.addEventListener('hashchange', cleanUpHighlights)
|
|
327
|
+
window.addEventListener('beforeunload', cleanUpHighlights)
|
|
328
|
+
const navigation = window.navigation
|
|
329
|
+
if (navigation && typeof navigation.addEventListener === 'function') {
|
|
330
|
+
navigation.addEventListener('navigate', cleanUpHighlights)
|
|
331
|
+
}
|
|
332
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* utils.js — ported from alibaba/page-agent packages/page-controller/src/utils/index.ts (MIT).
|
|
3
|
+
* Changes vs upstream:
|
|
4
|
+
* - Pointer/mask CustomEvents removed. Upstream dispatched `PageAgent::MovePointerTo`
|
|
5
|
+
* etc. to drive a visual cursor overlay; here those are optional hooks the host
|
|
6
|
+
* app can register (see setPointerHooks). No hook → no-op, no artificial wait.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
// ======= type guards =======
|
|
10
|
+
// @note instanceof fails for elements inside iframes, hence nodeType/tagName checks.
|
|
11
|
+
|
|
12
|
+
export function isHTMLElement(el) {
|
|
13
|
+
return !!el && el.nodeType === 1
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function isInputElement(el) {
|
|
17
|
+
return el?.nodeType === 1 && el.tagName === 'INPUT'
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function isTextAreaElement(el) {
|
|
21
|
+
return el?.nodeType === 1 && el.tagName === 'TEXTAREA'
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function isSelectElement(el) {
|
|
25
|
+
return el?.nodeType === 1 && el.tagName === 'SELECT'
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function isAnchorElement(el) {
|
|
29
|
+
return el?.nodeType === 1 && el.tagName === 'A'
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// ======= iframe helpers =======
|
|
33
|
+
|
|
34
|
+
/** Iframe offset for translating element coordinates to top-frame viewport. */
|
|
35
|
+
export function getIframeOffset(element) {
|
|
36
|
+
const frame = element.ownerDocument.defaultView?.frameElement
|
|
37
|
+
if (!frame) return { x: 0, y: 0 }
|
|
38
|
+
const rect = frame.getBoundingClientRect()
|
|
39
|
+
return { x: rect.left, y: rect.top }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Get native value setter from the element's own prototype (iframe-safe).
|
|
44
|
+
* Required so React's value-tracking sees the change (React patches the
|
|
45
|
+
* instance property; the prototype setter bypasses the patch, then the
|
|
46
|
+
* dispatched 'input' event makes React re-read the value).
|
|
47
|
+
*/
|
|
48
|
+
export function getNativeValueSetter(element) {
|
|
49
|
+
return Object.getOwnPropertyDescriptor(Object.getPrototypeOf(element), 'value').set
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ======= general utils =======
|
|
53
|
+
|
|
54
|
+
export async function waitFor(seconds, signal) {
|
|
55
|
+
if (signal?.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError')
|
|
56
|
+
await new Promise((resolve, reject) => {
|
|
57
|
+
const t = setTimeout(done, seconds * 1000)
|
|
58
|
+
function done() {
|
|
59
|
+
signal?.removeEventListener('abort', onAbort)
|
|
60
|
+
resolve()
|
|
61
|
+
}
|
|
62
|
+
function onAbort() {
|
|
63
|
+
clearTimeout(t)
|
|
64
|
+
reject(signal.reason ?? new DOMException('Aborted', 'AbortError'))
|
|
65
|
+
}
|
|
66
|
+
signal?.addEventListener('abort', onAbort, { once: true })
|
|
67
|
+
})
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// ======= optional pointer-visual hooks (replaces upstream mask CustomEvents) =======
|
|
71
|
+
|
|
72
|
+
let pointerHooks = null
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Register visual hooks: { movePointerTo(x,y), clickPointer(), enablePassThrough(), disablePassThrough() }
|
|
76
|
+
* All optional. Used purely for UX (showing a fake cursor); never load-bearing.
|
|
77
|
+
*/
|
|
78
|
+
export function setPointerHooks(hooks) {
|
|
79
|
+
pointerHooks = hooks
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export async function movePointerToElement(element, x, y) {
|
|
83
|
+
if (!pointerHooks?.movePointerTo) return
|
|
84
|
+
const offset = getIframeOffset(element)
|
|
85
|
+
pointerHooks.movePointerTo(x + offset.x, y + offset.y)
|
|
86
|
+
await waitFor(0.3)
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export async function clickPointer() {
|
|
90
|
+
pointerHooks?.clickPointer?.()
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export async function enablePassThrough() {
|
|
94
|
+
pointerHooks?.enablePassThrough?.()
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export async function disablePassThrough() {
|
|
98
|
+
pointerHooks?.disablePassThrough?.()
|
|
99
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* trusted-page-agent — a security-first rebuild of alibaba/page-agent.
|
|
3
|
+
*
|
|
4
|
+
* Kept from upstream (MIT, audited): the browser-use DOM extraction engine
|
|
5
|
+
* (complex layouts, occlusion hit-testing, iframes, shadow DOM, scrollable
|
|
6
|
+
* regions), the serialization format, and the React-aware action dispatch.
|
|
7
|
+
*
|
|
8
|
+
* Replaced/added: no eval tool of any kind, mandatory origin allowlist,
|
|
9
|
+
* fail-closed human confirmation for sensitive actions, PII redaction before
|
|
10
|
+
* page text reaches the LLM, audit log, rate limit, prompt-injection
|
|
11
|
+
* hardening, and an LLM connector that takes either a hosted API key or a
|
|
12
|
+
* local model URL (Ollama / LM Studio / vLLM / any OpenAI-compatible server).
|
|
13
|
+
*
|
|
14
|
+
* @example Hosted (Anthropic):
|
|
15
|
+
* const agent = new TrustedPageAgent({
|
|
16
|
+
* llm: { provider: 'anthropic', apiKey: '...', model: 'claude-sonnet-4-6' },
|
|
17
|
+
* allowedOrigins: [location.origin],
|
|
18
|
+
* onConfirm: async (p) => confirm(`Allow: ${p.reason}?`),
|
|
19
|
+
* })
|
|
20
|
+
* const result = await agent.run('Fill the signup form with test data')
|
|
21
|
+
*
|
|
22
|
+
* @example Local (Ollama):
|
|
23
|
+
* const agent = new TrustedPageAgent({
|
|
24
|
+
* llm: { provider: 'openai-compatible', baseURL: 'http://localhost:11434/v1', model: 'qwen3:14b' },
|
|
25
|
+
* allowedOrigins: [location.origin],
|
|
26
|
+
* onConfirm: async (p) => confirm(`Allow: ${p.reason}?`),
|
|
27
|
+
* })
|
|
28
|
+
*/
|
|
29
|
+
import { AgentLoop } from './agent/AgentLoop.js'
|
|
30
|
+
import { GuardedController } from './controller.js'
|
|
31
|
+
import { createLLM } from './llm/connector.js'
|
|
32
|
+
import { PolicyGuard } from './policy/PolicyGuard.js'
|
|
33
|
+
// @handhold-patch H6 (VENDOR.md): whole-site support (reviewed manifest + navigate).
|
|
34
|
+
import { loadSiteManifest } from './site/manifest.js'
|
|
35
|
+
|
|
36
|
+
export { AgentLoop, parseAgentOutput } from './agent/AgentLoop.js'
|
|
37
|
+
export { TOOLS, validateToolInput } from './agent/tools.js'
|
|
38
|
+
export { GuardedController, domSnapshotProvider } from './controller.js'
|
|
39
|
+
export { createRedactor, redactText } from './dom/redact.js'
|
|
40
|
+
export { flatTreeToString, getFlatTree, getSelectorMap } from './dom/serializer.js'
|
|
41
|
+
export { setPointerHooks } from './dom/utils.js'
|
|
42
|
+
export { createLLM } from './llm/connector.js'
|
|
43
|
+
export { PolicyGuard } from './policy/PolicyGuard.js'
|
|
44
|
+
export { ChatWidget } from './ui/ChatWidget.js'
|
|
45
|
+
|
|
46
|
+
export class TrustedPageAgent {
|
|
47
|
+
#loop
|
|
48
|
+
#controller
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* @param {Object} cfg
|
|
52
|
+
* @param {Object} cfg.llm createLLM() config: {provider, baseURL?, apiKey?, model, ...}
|
|
53
|
+
* @param {string[]} cfg.allowedOrigins REQUIRED — origins the agent may act on
|
|
54
|
+
* @param {(pending:{name,input,reason}) => Promise<boolean>} [cfg.onConfirm]
|
|
55
|
+
* Human gate for sensitive actions. Without it, sensitive actions are blocked.
|
|
56
|
+
* @param {(question:string) => Promise<string>} [cfg.onAskUser]
|
|
57
|
+
* @param {(ev:Object) => void} [cfg.onActivity] UI feedback (thinking/executing/executed/done)
|
|
58
|
+
* @param {number} [cfg.maxSteps=25]
|
|
59
|
+
* @param {number} [cfg.maxActionsPerMinute=30]
|
|
60
|
+
* @param {string[]} [cfg.extraSensitiveKeywords]
|
|
61
|
+
* @param {Object} [cfg.domConfig] engine options: viewportExpansion, interactiveBlacklist,
|
|
62
|
+
* includeAttributes, redactor, keepSemanticTags...
|
|
63
|
+
* @param {Object} [cfg.siteManifest] @handhold-patch H6: RAW manifest JSON as served by
|
|
64
|
+
* GET /api/v1/manifest. Validated here — construction
|
|
65
|
+
* THROWS unless reviewed === true (client-side gate).
|
|
66
|
+
* Without it the agent stays single-page.
|
|
67
|
+
* @param {(url:string)=>void|Promise<void>} [cfg.navigateImpl] injectable navigation
|
|
68
|
+
*/
|
|
69
|
+
constructor(cfg = {}) {
|
|
70
|
+
if (!cfg.llm) throw new Error('TrustedPageAgent: cfg.llm is required (see createLLM).')
|
|
71
|
+
|
|
72
|
+
const guard = new PolicyGuard({
|
|
73
|
+
allowedOrigins: cfg.allowedOrigins,
|
|
74
|
+
onConfirm: cfg.onConfirm,
|
|
75
|
+
extraSensitiveKeywords: cfg.extraSensitiveKeywords,
|
|
76
|
+
maxActionsPerMinute: cfg.maxActionsPerMinute,
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
// @handhold-patch H6: validate the manifest at the trust boundary. An
|
|
80
|
+
// unreviewed or malformed manifest must never configure the agent.
|
|
81
|
+
const siteManifest = cfg.siteManifest ? loadSiteManifest(cfg.siteManifest) : null
|
|
82
|
+
|
|
83
|
+
this.#controller = new GuardedController({
|
|
84
|
+
guard,
|
|
85
|
+
domConfig: cfg.domConfig,
|
|
86
|
+
siteManifest,
|
|
87
|
+
navigateImpl: cfg.navigateImpl,
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
this.#loop = new AgentLoop({
|
|
91
|
+
llm: createLLM(cfg.llm),
|
|
92
|
+
controller: this.#controller,
|
|
93
|
+
maxSteps: cfg.maxSteps,
|
|
94
|
+
onAskUser: cfg.onAskUser,
|
|
95
|
+
onActivity: cfg.onActivity,
|
|
96
|
+
siteManifest,
|
|
97
|
+
})
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Run a natural-language task. Resolves with {success, text, steps}. */
|
|
101
|
+
run(task, opts) {
|
|
102
|
+
return this.#loop.run(task, opts)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Cooperative stop; the current step finishes, no further actions run. */
|
|
106
|
+
stop() {
|
|
107
|
+
this.#loop.stop()
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Timestamped record of every action considered/allowed/blocked. */
|
|
111
|
+
getAuditLog() {
|
|
112
|
+
return this.#controller.getAuditLog()
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* @handhold-patch H7 (VENDOR.md): run snapshot for cross-page checkpoints —
|
|
117
|
+
* {history, stepsUsed} from the loop plus the guard's audit log.
|
|
118
|
+
*/
|
|
119
|
+
getRunState() {
|
|
120
|
+
return { ...this.#loop.getRunState(), audit: this.#controller.getAuditLog() }
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* One-liner: mount the chat-bubble UI and wire it to an agent.
|
|
126
|
+
* The widget becomes the human gate (Allow/Deny buttons), the ask_user
|
|
127
|
+
* channel, and the activity feed. Any onConfirm/onAskUser/onActivity you
|
|
128
|
+
* pass in agentConfig still wins over the widget defaults.
|
|
129
|
+
*
|
|
130
|
+
* @example
|
|
131
|
+
* attachChatWidget({
|
|
132
|
+
* llm: { provider: 'deepseek', apiKey: 'sk-...', model: 'deepseek-chat' },
|
|
133
|
+
* allowedOrigins: [location.origin],
|
|
134
|
+
* })
|
|
135
|
+
*
|
|
136
|
+
* @returns {{ agent: TrustedPageAgent, widget: ChatWidget }}
|
|
137
|
+
*/
|
|
138
|
+
export function attachChatWidget(agentConfig, widgetOptions = {}) {
|
|
139
|
+
let agent = null
|
|
140
|
+
const widget = new ChatWidgetClass({
|
|
141
|
+
...widgetOptions,
|
|
142
|
+
onTask: async (task) => {
|
|
143
|
+
widget.setBusy(true)
|
|
144
|
+
try {
|
|
145
|
+
const result = await agent.run(task)
|
|
146
|
+
widget.addMessage('agent', `${result.success ? '✅' : '⚠️'} ${result.text}`)
|
|
147
|
+
} catch (e) {
|
|
148
|
+
widget.addMessage('agent', `⚠️ Error: ${e.message}`)
|
|
149
|
+
} finally {
|
|
150
|
+
widget.setBusy(false)
|
|
151
|
+
}
|
|
152
|
+
},
|
|
153
|
+
})
|
|
154
|
+
widget.mount()
|
|
155
|
+
|
|
156
|
+
agent = new TrustedPageAgent({
|
|
157
|
+
onConfirm: (p) => widget.requestConfirm(p),
|
|
158
|
+
onAskUser: (q) => widget.askUser(q),
|
|
159
|
+
onActivity: (ev) => widget.showActivity(ev),
|
|
160
|
+
...agentConfig,
|
|
161
|
+
})
|
|
162
|
+
|
|
163
|
+
return { agent, widget }
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// internal alias so attachChatWidget can construct without a circular re-import
|
|
167
|
+
import { ChatWidget as ChatWidgetClass } from './ui/ChatWidget.js'
|
|
168
|
+
|
|
169
|
+
export default TrustedPageAgent
|