@wobuxcm/dsh-win-tools 0.0.0-stage → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/LICENSES/Apache-2.0.txt +202 -0
- package/README.md +88 -2
- package/THIRD-PARTY-NOTICES.md +36 -0
- package/bin/agent-desktop-win32-x64.exe +0 -0
- package/bin/package.json +3 -0
- package/bin/web.js +147 -0
- package/cordis.patch.yml +4 -0
- package/index.js +348 -0
- package/lib/core.js +583 -0
- package/lib/cua/engine.js +322 -0
- package/lib/cua/input.js +205 -0
- package/lib/cua/native.js +490 -0
- package/lib/cua/safety.js +162 -0
- package/lib/cua/screen.js +300 -0
- package/package.json +56 -4
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The in-process wait/macro engine — the reason this plugin exists.
|
|
3
|
+
*
|
|
4
|
+
* Everything here runs INSIDE the host process: poll -> condition -> act, with
|
|
5
|
+
* zero model round-trips per iteration. A 60s wait at 120ms polling is ~500
|
|
6
|
+
* comparisons; done in-process that is one tool call, done through the model
|
|
7
|
+
* it would be a thousand.
|
|
8
|
+
*
|
|
9
|
+
* @module dsh-cua-win/lib/engine
|
|
10
|
+
*/
|
|
11
|
+
import { capture, getShot, imageChangeRatio, parseColor, pixelMatches, regionRatio } from './screen.js'
|
|
12
|
+
import { click, keyPress, move, typeText } from './input.js'
|
|
13
|
+
import { blacklistHit, isStopped, logAction } from './safety.js'
|
|
14
|
+
import { foregroundInfo, sleep, windowAtPoint } from './native.js'
|
|
15
|
+
|
|
16
|
+
/** Cap on trace entries so a 60s poll storm cannot bloat the tool result. */
|
|
17
|
+
const TRACE_CAP = 240
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Validate and normalize one condition object.
|
|
21
|
+
* @param {object} raw
|
|
22
|
+
* @returns {{type:string, data:object}|{error:string}}
|
|
23
|
+
*/
|
|
24
|
+
export function normalizeCondition(raw) {
|
|
25
|
+
if (typeof raw !== 'object' || raw === null) return { error: 'condition must be an object' }
|
|
26
|
+
if (raw.pixel_matches !== undefined) {
|
|
27
|
+
const c = raw.pixel_matches
|
|
28
|
+
const rgb = parseColor(c?.color)
|
|
29
|
+
if (!Number.isFinite(c?.x) || !Number.isFinite(c?.y) || !rgb) return { error: 'pixel_matches needs numeric x,y and color "#rrggbb"' }
|
|
30
|
+
return { type: 'pixel_matches', data: { x: c.x, y: c.y, rgb, tolerance: Number.isFinite(c.tolerance) ? c.tolerance : 16 } }
|
|
31
|
+
}
|
|
32
|
+
if (raw.region_ratio !== undefined) {
|
|
33
|
+
const c = raw.region_ratio
|
|
34
|
+
const rgb = parseColor(c?.color)
|
|
35
|
+
const rect = Array.isArray(c?.rect) ? c.rect.map(Number) : null
|
|
36
|
+
if (!rgb || !rect || rect.length !== 4 || rect.some((v) => !Number.isFinite(v)) || rect[2] <= 0 || rect[3] <= 0) {
|
|
37
|
+
return { error: 'region_ratio needs rect [x,y,w,h] and color "#rrggbb"' }
|
|
38
|
+
}
|
|
39
|
+
if (!Number.isFinite(c?.minRatio) || c.minRatio < 0 || c.minRatio > 1) return { error: 'region_ratio needs minRatio in [0,1]' }
|
|
40
|
+
return { type: 'region_ratio', data: { rect, rgb, tolerance: Number.isFinite(c.tolerance) ? c.tolerance : 16, minRatio: c.minRatio } }
|
|
41
|
+
}
|
|
42
|
+
if (raw.image_changed !== undefined) {
|
|
43
|
+
const c = raw.image_changed
|
|
44
|
+
if (typeof c?.baselineShotId !== 'string' || !getShot(c.baselineShotId)) return { error: 'image_changed needs a valid baselineShotId from a previous cua_screen_capture' }
|
|
45
|
+
let rect
|
|
46
|
+
if (c.rect !== undefined) {
|
|
47
|
+
const r = Array.isArray(c.rect) ? c.rect.map(Number) : null
|
|
48
|
+
if (!r || r.length !== 4 || r.some((v) => !Number.isFinite(v)) || r[2] <= 0 || r[3] <= 0) {
|
|
49
|
+
return { error: 'image_changed rect must be [x,y,w,h]' }
|
|
50
|
+
}
|
|
51
|
+
rect = r
|
|
52
|
+
}
|
|
53
|
+
return { type: 'image_changed', data: { baselineShotId: c.baselineShotId, rect, minChangeRatio: Number.isFinite(c.minChangeRatio) ? c.minChangeRatio : 0.02 } }
|
|
54
|
+
}
|
|
55
|
+
if (raw.window_title_contains !== undefined) {
|
|
56
|
+
const c = raw.window_title_contains
|
|
57
|
+
if (typeof c?.pattern !== 'string' || c.pattern.trim() === '') return { error: 'window_title_contains needs a non-empty pattern' }
|
|
58
|
+
return { type: 'window_title_contains', data: { pattern: c.pattern } }
|
|
59
|
+
}
|
|
60
|
+
return { error: 'condition must be one of pixel_matches | region_ratio | image_changed | window_title_contains' }
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Evaluate one normalized condition against the current screen state.
|
|
65
|
+
*
|
|
66
|
+
* @param {{type:string,data:object}} cond
|
|
67
|
+
* @param {object|null} frame - latest capture frame (null for pure-title checks).
|
|
68
|
+
* @returns {{matched:boolean, observed?:number}} observed = ratio or 0/1.
|
|
69
|
+
*/
|
|
70
|
+
export function evaluateCondition(cond, frame) {
|
|
71
|
+
if (cond.type === 'window_title_contains') {
|
|
72
|
+
const fg = foregroundInfo()
|
|
73
|
+
const title = fg?.title ?? ''
|
|
74
|
+
return { matched: title.toLowerCase().includes(cond.data.pattern.toLowerCase()), observed: title === '' ? 0 : 1 }
|
|
75
|
+
}
|
|
76
|
+
if (!frame) return { matched: false, observed: 0 }
|
|
77
|
+
if (cond.type === 'pixel_matches') {
|
|
78
|
+
const matched = pixelMatches(frame, cond.data.x, cond.data.y, cond.data.rgb, cond.data.tolerance)
|
|
79
|
+
return { matched, observed: matched ? 1 : 0 }
|
|
80
|
+
}
|
|
81
|
+
if (cond.type === 'region_ratio') {
|
|
82
|
+
const ratio = regionRatio(frame, cond.data.rect, cond.data.rgb, cond.data.tolerance)
|
|
83
|
+
return { matched: ratio >= cond.data.minRatio && ratio >= 0, observed: ratio }
|
|
84
|
+
}
|
|
85
|
+
if (cond.type === 'image_changed') {
|
|
86
|
+
const baseline = getShot(cond.data.baselineShotId)
|
|
87
|
+
const ratio = imageChangeRatio(frame, baseline, cond.data.rect)
|
|
88
|
+
return { matched: ratio >= cond.data.minChangeRatio && ratio >= 0, observed: ratio }
|
|
89
|
+
}
|
|
90
|
+
return { matched: false, observed: 0 }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** One condition needs a screen capture to evaluate. */
|
|
94
|
+
function conditionNeedsFrame(cond) {
|
|
95
|
+
return cond.type !== 'window_title_contains'
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Build the act function used by both wait_and_click and macro steps.
|
|
100
|
+
*
|
|
101
|
+
* @param {object} config - effective settings.
|
|
102
|
+
* @param {object} gateState - {dryRun, reason} from the safety gate.
|
|
103
|
+
* @param {string} toolName - for the log line.
|
|
104
|
+
* @returns {Function} async (action: {type, x, y, text, keys, ...}) -> result object
|
|
105
|
+
*/
|
|
106
|
+
export function makeAct(config, gateState, toolName) {
|
|
107
|
+
return async (action) => {
|
|
108
|
+
const type = action?.type ?? 'click'
|
|
109
|
+
const started = Date.now()
|
|
110
|
+
const fgBefore = foregroundInfo()
|
|
111
|
+
|
|
112
|
+
if (type !== 'type_text' && type !== 'key_press') {
|
|
113
|
+
const x = Number(action?.x)
|
|
114
|
+
const y = Number(action?.y)
|
|
115
|
+
if (!Number.isFinite(x) || !Number.isFinite(y)) return { ok: false, code: 'INVALID_ARGS', message: `${type} needs numeric x,y` }
|
|
116
|
+
// window under the point: blacklist on its title AND the foreground title
|
|
117
|
+
const at = windowAtPoint(x, y)
|
|
118
|
+
const hit = blacklistHit(config, [at?.title, fgBefore?.title])
|
|
119
|
+
if (hit) {
|
|
120
|
+
logAction(config, { tool: toolName, action: type, x, y, window: at?.title, blocked: true, keyword: hit, gate: gateState.reason })
|
|
121
|
+
return {
|
|
122
|
+
ok: false, code: 'BLOCKED_KEYWORD', keyword: hit,
|
|
123
|
+
message: `refused to click: window title "${at?.title ?? fgBefore?.title ?? '?'}" contains blacklisted keyword "${hit}"`,
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (gateState.dryRun) {
|
|
129
|
+
const what = type === 'type_text' ? `type_text(${(action?.text ?? '').slice(0, 60)}…)`
|
|
130
|
+
: type === 'key_press' ? `key_press(${action?.keys})`
|
|
131
|
+
: type === 'double_click' ? `double_click(${action?.x},${action?.y})`
|
|
132
|
+
: `${type}(${action?.x},${action?.y})`
|
|
133
|
+
logAction(config, { tool: toolName, action: type, x: action?.x, y: action?.y, dryRun: true, gate: gateState.reason, window: fgBefore?.title })
|
|
134
|
+
return { ok: true, dryRun: true, would: what, gate: gateState.reason, foreground: fgBefore?.title ?? null }
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
let result
|
|
138
|
+
if (type === 'click' || type === 'double_click') {
|
|
139
|
+
result = click(action.x, action.y, action.button === 'right' || action.button === 'middle' ? action.button : 'left', type === 'double_click' ? 2 : 1)
|
|
140
|
+
} else if (type === 'move') {
|
|
141
|
+
result = move(action.x, action.y)
|
|
142
|
+
} else if (type === 'type_text') {
|
|
143
|
+
result = await typeText(String(action?.text ?? ''))
|
|
144
|
+
} else if (type === 'key_press') {
|
|
145
|
+
result = await keyPress(String(action?.keys ?? ''), Number(action?.times) || 1)
|
|
146
|
+
} else {
|
|
147
|
+
return { ok: false, code: 'INVALID_ARGS', message: `unknown action type "${type}"` }
|
|
148
|
+
}
|
|
149
|
+
const fgAfter = foregroundInfo()
|
|
150
|
+
const entry = {
|
|
151
|
+
tool: toolName, action: type,
|
|
152
|
+
...(['click', 'double_click', 'move'].includes(type) ? { x: action.x, y: action.y, cursor: result?.cursor ?? null } : {}),
|
|
153
|
+
...(type === 'type_text' ? { chars: String(action?.text ?? '').length } : {}),
|
|
154
|
+
...(type === 'key_press' ? { keys: action?.keys, times: result?.times } : {}),
|
|
155
|
+
window: fgBefore?.title, windowAfter: fgAfter?.title, elapsedMs: Date.now() - started,
|
|
156
|
+
}
|
|
157
|
+
logAction(config, entry)
|
|
158
|
+
return { ok: true, dryRun: false, foreground: fgAfter?.title ?? null, ...(result ?? {}) }
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* cua_wait_and_click: poll conditions in-process, act on first match.
|
|
164
|
+
*
|
|
165
|
+
* @param {object} options
|
|
166
|
+
* @param {object} config - effective settings.
|
|
167
|
+
* @param {object} gateState
|
|
168
|
+
* @param {AbortSignal|undefined} signal
|
|
169
|
+
*/
|
|
170
|
+
export async function waitAndClick(options, config, gateState, signal) {
|
|
171
|
+
const timeoutMs = Math.min(Math.max(Number(options.timeoutMs) || config.timeoutMs, 500), 600000)
|
|
172
|
+
const pollMs = Math.min(Math.max(Number(options.pollMs) || config.pollMs, 30), 2000)
|
|
173
|
+
|
|
174
|
+
const conditions = []
|
|
175
|
+
const conditionSources = Array.isArray(options.conditions) ? options.conditions : options.condition !== undefined ? [options.condition] : []
|
|
176
|
+
for (const raw of conditionSources) {
|
|
177
|
+
const norm = normalizeCondition(raw)
|
|
178
|
+
if (norm.error) return { ok: false, code: 'INVALID_ARGS', message: norm.error }
|
|
179
|
+
conditions.push(norm)
|
|
180
|
+
}
|
|
181
|
+
if (conditions.length === 0) return { ok: false, code: 'INVALID_ARGS', message: 'provide at least one condition (pixel_matches | region_ratio | image_changed | window_title_contains)' }
|
|
182
|
+
|
|
183
|
+
const actions = Array.isArray(options.actions) ? options.actions : options.action !== undefined ? [options.action] : []
|
|
184
|
+
if (actions.length === 0) return { ok: false, code: 'INVALID_ARGS', message: 'provide at least one action (click/double_click/type_text/move/key_press)' }
|
|
185
|
+
if (actions.length > 12) return { ok: false, code: 'INVALID_ARGS', message: 'at most 12 actions per wait call' }
|
|
186
|
+
|
|
187
|
+
let abortCond = null
|
|
188
|
+
if (options.abortIf !== undefined) {
|
|
189
|
+
const norm = normalizeCondition(options.abortIf)
|
|
190
|
+
if (norm.error) return { ok: false, code: 'INVALID_ARGS', message: `abortIf: ${norm.error}` }
|
|
191
|
+
abortCond = norm
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const act = makeAct(config, gateState, 'cua_wait_and_click')
|
|
195
|
+
const trace = []
|
|
196
|
+
const started = Date.now()
|
|
197
|
+
let attempts = 0
|
|
198
|
+
let lastFrame = null
|
|
199
|
+
let lastShotId = null
|
|
200
|
+
|
|
201
|
+
while (true) {
|
|
202
|
+
if (signal?.aborted) return { ok: false, code: 'ABORTED', message: 'session aborted', elapsedMs: Date.now() - started, attempts }
|
|
203
|
+
if (isStopped()) return { ok: false, code: 'STOPPED', message: 'cua.stop file present; macro loop terminated', elapsedMs: Date.now() - started, attempts }
|
|
204
|
+
|
|
205
|
+
attempts += 1
|
|
206
|
+
const needFrame = conditions.some(conditionNeedsFrame) || (abortCond !== null && conditionNeedsFrame(abortCond))
|
|
207
|
+
if (needFrame) {
|
|
208
|
+
const shot = capture({ target: 'screen' })
|
|
209
|
+
if (shot.ok) {
|
|
210
|
+
lastFrame = getShot(shot.screenshotId)
|
|
211
|
+
lastShotId = shot.screenshotId
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
if (abortCond !== null) {
|
|
216
|
+
const { matched } = evaluateCondition(abortCond, lastFrame)
|
|
217
|
+
if (matched) {
|
|
218
|
+
return { matched: false, aborted: true, attempts, elapsedMs: Date.now() - started, trace: trace.slice(-TRACE_CAP), screenshotId: lastShotId }
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
for (const cond of conditions) {
|
|
223
|
+
const { matched, observed } = evaluateCondition(cond, lastFrame)
|
|
224
|
+
if (trace.length < TRACE_CAP && (attempts <= 5 || matched || attempts % 20 === 0)) {
|
|
225
|
+
trace.push({ t: Date.now() - started, cond: cond.type, observed: round(observed, 4) })
|
|
226
|
+
}
|
|
227
|
+
if (matched) {
|
|
228
|
+
const results = []
|
|
229
|
+
for (const action of actions) {
|
|
230
|
+
const result = await act(action)
|
|
231
|
+
results.push(result)
|
|
232
|
+
if (!result.ok) {
|
|
233
|
+
return { matched: true, ok: false, code: result.code, message: result.message, attempts, elapsedMs: Date.now() - started, trace: trace.slice(-TRACE_CAP), screenshotId: lastShotId, actionResults: results }
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return { matched: true, ok: true, attempts, elapsedMs: Date.now() - started, actions: results.length, actionResults: results, trace: trace.slice(-TRACE_CAP), screenshotId: lastShotId }
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
if (Date.now() - started >= timeoutMs) {
|
|
241
|
+
return { matched: false, ok: true, timedOut: true, attempts, elapsedMs: Date.now() - started, trace: trace.slice(-TRACE_CAP), screenshotId: lastShotId }
|
|
242
|
+
}
|
|
243
|
+
await sleep(pollMs)
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* cua_macro: ordered steps, each optionally waiting on a condition first.
|
|
249
|
+
* @param {object} options
|
|
250
|
+
* @param {object} config
|
|
251
|
+
* @param {object} gateState
|
|
252
|
+
* @param {AbortSignal|undefined} signal
|
|
253
|
+
*/
|
|
254
|
+
export async function runMacro(options, config, gateState, signal) {
|
|
255
|
+
const steps = Array.isArray(options.steps) ? options.steps : []
|
|
256
|
+
if (steps.length === 0) return { ok: false, code: 'INVALID_ARGS', message: 'steps must be a non-empty array' }
|
|
257
|
+
if (steps.length > 50) return { ok: false, code: 'INVALID_ARGS', message: 'at most 50 steps per macro' }
|
|
258
|
+
const timeoutMs = Math.min(Math.max(Number(options.timeoutMs) || config.timeoutMs, 1000), 600000)
|
|
259
|
+
const stepTimeoutMs = Math.min(Math.max(Number(options.stepTimeoutMs) || 20000, 1000), 120000)
|
|
260
|
+
|
|
261
|
+
const act = makeAct(config, gateState, 'cua_macro')
|
|
262
|
+
const started = Date.now()
|
|
263
|
+
const results = []
|
|
264
|
+
|
|
265
|
+
for (let index = 0; index < steps.length; index += 1) {
|
|
266
|
+
if (signal?.aborted) return { ok: false, code: 'ABORTED', message: `aborted at step ${index}`, steps: results }
|
|
267
|
+
if (isStopped()) return { ok: false, code: 'STOPPED', message: `cua.stop present; terminated at step ${index}`, steps: results }
|
|
268
|
+
if (Date.now() - started >= timeoutMs) {
|
|
269
|
+
return { ok: false, code: 'TIMEOUT', message: `macro exceeded ${timeoutMs}ms at step ${index}`, steps: results, elapsedMs: Date.now() - started }
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
const step = steps[index]
|
|
273
|
+
const actions = Array.isArray(step.actions) ? step.actions : step.action !== undefined ? [step.action] : []
|
|
274
|
+
if (actions.length === 0) return { ok: false, code: 'INVALID_ARGS', message: `step ${index}: no action(s)` }
|
|
275
|
+
|
|
276
|
+
// optional wait inside the step
|
|
277
|
+
let waitInfo = null
|
|
278
|
+
if (step.wait !== undefined) {
|
|
279
|
+
const norm = normalizeCondition(step.wait)
|
|
280
|
+
if (norm.error) return { ok: false, code: 'INVALID_ARGS', message: `step ${index} wait: ${norm.error}` }
|
|
281
|
+
const stepStart = Date.now()
|
|
282
|
+
let attempts = 0
|
|
283
|
+
let frame = null
|
|
284
|
+
while (true) {
|
|
285
|
+
if (signal?.aborted || isStopped()) return { ok: false, code: 'ABORTED', message: `aborted while waiting in step ${index}`, steps: results }
|
|
286
|
+
attempts += 1
|
|
287
|
+
if (conditionNeedsFrame(norm)) {
|
|
288
|
+
const shot = capture({ target: 'screen' })
|
|
289
|
+
if (shot.ok) frame = getShot(shot.screenshotId)
|
|
290
|
+
}
|
|
291
|
+
const { matched, observed } = evaluateCondition(norm, frame)
|
|
292
|
+
if (matched) { waitInfo = { attempts, elapsedMs: Date.now() - stepStart, observed: round(observed, 4) }; break }
|
|
293
|
+
if (Date.now() - stepStart >= stepTimeoutMs) {
|
|
294
|
+
return { ok: false, code: 'WAIT_TIMEOUT', message: `step ${index} wait condition not met within ${stepTimeoutMs}ms`, steps: results }
|
|
295
|
+
}
|
|
296
|
+
if (Date.now() - started >= timeoutMs) {
|
|
297
|
+
return { ok: false, code: 'TIMEOUT', message: `macro exceeded ${timeoutMs}ms while waiting in step ${index}`, steps: results }
|
|
298
|
+
}
|
|
299
|
+
await sleep(Math.min(Math.max(config.pollMs, 30), 500))
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const actionResults = []
|
|
304
|
+
for (const action of actions) {
|
|
305
|
+
const result = await act(action)
|
|
306
|
+
actionResults.push(result)
|
|
307
|
+
if (!result.ok) {
|
|
308
|
+
return { ok: false, code: result.code, message: `step ${index}: ${result.message}`, steps: results, failedStep: index }
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
results.push({ step: index, wait: waitInfo, actions: actionResults })
|
|
312
|
+
if (typeof step.delayMs === 'number' && step.delayMs > 0) await sleep(Math.min(step.delayMs, 2000))
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
return { ok: true, steps: results, elapsedMs: Date.now() - started }
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function round(value, digits) {
|
|
319
|
+
if (!Number.isFinite(value)) return value
|
|
320
|
+
const f = 10 ** digits
|
|
321
|
+
return Math.round(value * f) / f
|
|
322
|
+
}
|
package/lib/cua/input.js
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Input injection for dsh-cua-win: SendInput only, absolute coordinates.
|
|
3
|
+
*
|
|
4
|
+
* SetCursorPos + relative mouse_event were tried against WeChatAppEx.exe
|
|
5
|
+
* (Chromium borderless, Chrome_WidgetWin_0): the cursor moves but the click
|
|
6
|
+
* never lands. Absolute SendInput (MOUSEEVENTF_ABSOLUTE | MOVE, coordinates
|
|
7
|
+
* normalized to the virtual desktop) hits the same point first try. This
|
|
8
|
+
* file contains nothing else on purpose.
|
|
9
|
+
*
|
|
10
|
+
* @module dsh-cua-win/lib/input
|
|
11
|
+
*/
|
|
12
|
+
import {
|
|
13
|
+
CONST,
|
|
14
|
+
cursorPos,
|
|
15
|
+
keyInput,
|
|
16
|
+
mouseButtonInput,
|
|
17
|
+
mouseMoveInput,
|
|
18
|
+
native,
|
|
19
|
+
sendInput,
|
|
20
|
+
sleep,
|
|
21
|
+
unicodeInput,
|
|
22
|
+
} from './native.js'
|
|
23
|
+
|
|
24
|
+
/** Map of key names to virtual-key codes (the combos a desktop agent needs). */
|
|
25
|
+
const VK = {
|
|
26
|
+
backspace: 0x08, tab: 0x09, enter: 0x0d, return: 0x0d, shift: 0x10, ctrl: 0x11, control: 0x11,
|
|
27
|
+
alt: 0x12, menu: 0x12, pause: 0x13, capslock: 0x14, esc: 0x1b, escape: 0x1b, space: 0x20,
|
|
28
|
+
pageup: 0x21, pagedown: 0x22, end: 0x23, home: 0x24, left: 0x25, up: 0x26, right: 0x27, down: 0x28,
|
|
29
|
+
printscreen: 0x2c, insert: 0x2d, delete: 0x2e, win: 0x5b, meta: 0x5b, apps: 0x5d,
|
|
30
|
+
f1: 0x70, f2: 0x71, f3: 0x72, f4: 0x73, f5: 0x74, f6: 0x75, f7: 0x76, f8: 0x77, f9: 0x78,
|
|
31
|
+
f10: 0x79, f11: 0x7a, f12: 0x7b, f13: 0x7c, f14: 0x7d, f15: 0x7e,
|
|
32
|
+
numlock: 0x90, scrolllock: 0x91,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const BUTTONS = {
|
|
36
|
+
left: { down: CONST.MOUSEEVENTF_LEFTDOWN, up: CONST.MOUSEEVENTF_LEFTUP },
|
|
37
|
+
right: { down: CONST.MOUSEEVENTF_RIGHTDOWN, up: CONST.MOUSEEVENTF_RIGHTUP },
|
|
38
|
+
middle: { down: CONST.MOUSEEVENTF_MIDDLEDOWN, up: CONST.MOUSEEVENTF_MIDDLEUP },
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Parse a key combo string ("ctrl+shift+tab", "enter", "a") to VK codes.
|
|
43
|
+
* @param {string} keys
|
|
44
|
+
* @returns {Array<number>} in modifier-first order.
|
|
45
|
+
* @throws on an unknown key name.
|
|
46
|
+
*/
|
|
47
|
+
export function parseKeys(keys) {
|
|
48
|
+
if (typeof keys !== 'string' || keys.trim() === '') throw new Error('keys must be a non-empty string')
|
|
49
|
+
const parts = keys.split('+').map((part) => part.trim().toLowerCase()).filter((part) => part !== '')
|
|
50
|
+
if (parts.length === 0) throw new Error('keys parsed to nothing')
|
|
51
|
+
const out = []
|
|
52
|
+
for (const part of parts) {
|
|
53
|
+
if (part.length === 1) {
|
|
54
|
+
out.push(part.toUpperCase().charCodeAt(0))
|
|
55
|
+
} else if (VK[part] !== undefined) {
|
|
56
|
+
out.push(VK[part])
|
|
57
|
+
} else {
|
|
58
|
+
throw new Error(`unknown key "${part}"`)
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return out
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Move the cursor to a physical point (absolute SendInput).
|
|
66
|
+
* @param {number} x
|
|
67
|
+
* @param {number} y
|
|
68
|
+
*/
|
|
69
|
+
export function move(x, y) {
|
|
70
|
+
sendInput([mouseMoveInput(x, y)])
|
|
71
|
+
return cursorPos()
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Click at a physical point. Moves first (absolute), then the button events.
|
|
76
|
+
* @param {number} x
|
|
77
|
+
* @param {number} y
|
|
78
|
+
* @param {string} button - left | right | middle.
|
|
79
|
+
* @param {number} clickCount - 1 or 2 (2 = double click).
|
|
80
|
+
* @param {number} [delayMs] - between events.
|
|
81
|
+
* @returns {{cursor:{x,y}|null, sent:number, afterWindow:null}}
|
|
82
|
+
*/
|
|
83
|
+
export function click(x, y, button = 'left', clickCount = 1, delayMs = 40) {
|
|
84
|
+
const b = BUTTONS[button] ?? BUTTONS.left
|
|
85
|
+
move(x, y)
|
|
86
|
+
const events = []
|
|
87
|
+
const count = clickCount === 2 ? 2 : 1
|
|
88
|
+
for (let i = 0; i < count; i += 1) {
|
|
89
|
+
events.push(mouseButtonInput(b.down))
|
|
90
|
+
events.push(mouseButtonInput(b.up))
|
|
91
|
+
}
|
|
92
|
+
// Interleave small waits inside one batch is not possible; send per pair.
|
|
93
|
+
let sent = 0
|
|
94
|
+
for (let i = 0; i < events.length; i += 2) {
|
|
95
|
+
sent += sendInput([events[i], events[i + 1]])
|
|
96
|
+
if (i + 2 < events.length && delayMs > 0) {
|
|
97
|
+
// busy-free pause between double-click halves
|
|
98
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delayMs)
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return { cursor: cursorPos(), sent }
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Drag from one point to another with interpolated motion.
|
|
106
|
+
* @param {number[]} from - [x,y].
|
|
107
|
+
* @param {number[]} to - [x,y].
|
|
108
|
+
* @param {number} durationMs - total drag duration.
|
|
109
|
+
* @param {string} button
|
|
110
|
+
*/
|
|
111
|
+
export async function drag(from, to, durationMs = 300, button = 'left') {
|
|
112
|
+
const b = BUTTONS[button] ?? BUTTONS.left
|
|
113
|
+
const fx = from[0], fy = from[1], tx = to[0], ty = to[1]
|
|
114
|
+
move(fx, fy)
|
|
115
|
+
await sleep(60)
|
|
116
|
+
sendInput([mouseButtonInput(b.down)])
|
|
117
|
+
const steps = Math.max(2, Math.min(60, Math.round(durationMs / 10)))
|
|
118
|
+
const dt = durationMs / steps
|
|
119
|
+
for (let i = 1; i <= steps; i += 1) {
|
|
120
|
+
const t = i / steps
|
|
121
|
+
// ease-in-out so Chromium sees a plausible human-ish swipe
|
|
122
|
+
const eased = t < 0.5 ? 2 * t * t : 1 - 2 * (1 - t) * (1 - t)
|
|
123
|
+
move(fx + (tx - fx) * eased, fy + (ty - fy) * eased)
|
|
124
|
+
await sleep(Math.max(1, Math.round(dt)))
|
|
125
|
+
}
|
|
126
|
+
move(tx, ty)
|
|
127
|
+
await sleep(40)
|
|
128
|
+
sendInput([mouseButtonInput(b.up)])
|
|
129
|
+
return { cursor: cursorPos() }
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Press a key combo n times.
|
|
134
|
+
* @param {string} keys
|
|
135
|
+
* @param {number} times
|
|
136
|
+
*/
|
|
137
|
+
export async function keyPress(keys, times = 1) {
|
|
138
|
+
const codes = parseKeys(keys)
|
|
139
|
+
const count = Math.min(20, Math.max(1, Math.round(times) || 1))
|
|
140
|
+
for (let n = 0; n < count; n += 1) {
|
|
141
|
+
for (const vk of codes) sendInput([keyInput(vk, false)])
|
|
142
|
+
// release in reverse (modifiers last)
|
|
143
|
+
for (const vk of [...codes].reverse()) sendInput([keyInput(vk, true)])
|
|
144
|
+
if (n + 1 < count) await sleep(60)
|
|
145
|
+
}
|
|
146
|
+
return { keys, times: count }
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Type text through KEYEVENTF_UNICODE (layout-independent).
|
|
151
|
+
*
|
|
152
|
+
* A newline cannot travel as a unicode character: most edit controls ignore a
|
|
153
|
+
* U+000A / U+000D arrival, so a multi-line string used to land on one line.
|
|
154
|
+
* "\n" (and "\r\n") is therefore emitted as a real VK_RETURN keystroke.
|
|
155
|
+
*
|
|
156
|
+
* @param {string} text
|
|
157
|
+
*/
|
|
158
|
+
export async function typeText(text) {
|
|
159
|
+
const value = typeof text === 'string' ? text : String(text)
|
|
160
|
+
if (value.length === 0) return { typed: 0 }
|
|
161
|
+
const VK_RETURN = 0x0d
|
|
162
|
+
// Batch so a single sync call does not stall the host for long text.
|
|
163
|
+
const CHUNK = 64
|
|
164
|
+
let events = []
|
|
165
|
+
let typed = 0
|
|
166
|
+
const flush = () => {
|
|
167
|
+
if (events.length > 0) {
|
|
168
|
+
sendInput(events)
|
|
169
|
+
events = []
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
for (const ch of value) {
|
|
173
|
+
if (ch === '\r') continue // CRLF collapses to the single Enter below
|
|
174
|
+
if (ch === '\n') {
|
|
175
|
+
flush()
|
|
176
|
+
sendInput([keyInput(VK_RETURN, false)])
|
|
177
|
+
sendInput([keyInput(VK_RETURN, true)])
|
|
178
|
+
await sleep(25)
|
|
179
|
+
typed += 1
|
|
180
|
+
continue
|
|
181
|
+
}
|
|
182
|
+
const code = ch.codePointAt(0)
|
|
183
|
+
if (code > 0xffff) {
|
|
184
|
+
const lead = 0xd800 + Math.floor((code - 0x10000) / 0x400)
|
|
185
|
+
const trail = 0xdc00 + ((code - 0x10000) % 0x400)
|
|
186
|
+
events.push(unicodeInput(lead, false), unicodeInput(lead, true), unicodeInput(trail, false), unicodeInput(trail, true))
|
|
187
|
+
} else {
|
|
188
|
+
events.push(unicodeInput(code, false), unicodeInput(code, true))
|
|
189
|
+
}
|
|
190
|
+
typed += 1
|
|
191
|
+
if (events.length >= CHUNK * 2) {
|
|
192
|
+
flush()
|
|
193
|
+
await sleep(8)
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
flush()
|
|
197
|
+
return { typed }
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** Scroll helper (not a first-class tool yet; used by future cua_scroll). */
|
|
201
|
+
export async function scroll(dx, dy) {
|
|
202
|
+
if (dy) sendInput([mouseButtonInput(CONST.MOUSEEVENTF_WHEEL, -Math.round(dy) * CONST.WHEEL_DELTA)])
|
|
203
|
+
if (dx) sendInput([mouseButtonInput(CONST.MOUSEEVENTF_HWHEEL, Math.round(dx) * CONST.WHEEL_DELTA)])
|
|
204
|
+
return { dx, dy }
|
|
205
|
+
}
|