@ycheung5/pi-computer-use 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -33,7 +33,7 @@ Use your existing Pi installation. Another computer-use extension is not require
33
33
 
34
34
  ### From npm
35
35
 
36
- Install the published package with:
36
+ Install [@ycheung5/pi-computer-use](https://www.npmjs.com/package/@ycheung5/pi-computer-use) from npm:
37
37
 
38
38
  ```sh
39
39
  pi install npm:@ycheung5/pi-computer-use
@@ -57,11 +57,11 @@ To retain the built checkout as a local Pi package, run `pi install .` from that
57
57
 
58
58
  ### Install a built archive
59
59
 
60
- For a prebuilt `ycheung5-pi-computer-use-0.1.0.tgz` produced by `npm pack`:
60
+ For a prebuilt `ycheung5-pi-computer-use-0.1.1.tgz` produced by `npm pack`:
61
61
 
62
62
  ```sh
63
63
  mkdir pi-computer-use
64
- tar -xzf ycheung5-pi-computer-use-0.1.0.tgz -C pi-computer-use --strip-components=1
64
+ tar -xzf ycheung5-pi-computer-use-0.1.1.tgz -C pi-computer-use --strip-components=1
65
65
  cd pi-computer-use
66
66
  npm install --omit=dev --omit=peer --ignore-scripts
67
67
  pi install .
@@ -128,12 +128,12 @@ Settings are shared across projects and native Windows/WSL2 Pi in `%LOCALAPPDATA
128
128
 
129
129
  ## Tested scope and known limits
130
130
 
131
- The release audit recorded **187 passing portable tests**, one opt-in historical-reference skip, and checks of the built distribution. Named native/browser agent workflows and human control/display checks have separate [acceptance receipts](docs/validation.md#capability-coverage). CI is not a claim that every application or desktop state works.
131
+ The input-timing update passed **214 portable tests**, with one opt-in historical-reference skip, plus native compilation and built-distribution checks. Named native/browser agent workflows and human control/display checks have separate [acceptance receipts](docs/validation.md#capability-coverage). CI is not a claim that every application or desktop state works.
132
132
 
133
133
  - Second-computer WSL installation and desktop actions were user-confirmed. Fresh native-Windows installation, broken-prerequisite handling and shutdown in that clean environment remain accepted, unverified first-release test limitations.
134
134
  - Display mapping was exercised at **100%, 150% and 175%**, including a changed negative-origin layout. Other configurations, including 200%, remain unverified.
135
135
  - Windows may refuse foreground activation or access to elevated/protected interfaces. The extension reports refusal rather than bypassing Windows protections.
136
- - Explicit Unicode SendInput retains a **50 ms per-unit delay**. Clipboard preservation has format-dependent limits; installed Firefox support is experimental.
136
+ - Explicit Unicode SendInput defaults to a **50 ms per-unit delay**; explicit native pacing and per-call deadlines are [configurable](docs/text.md#sendinput-pacing-and-deadlines). Clipboard preservation has format-dependent limits; installed Firefox support is experimental.
137
137
  - Arm64, genuine same-HWND/PID reuse, verified UIPI denial and additional recorder/remote/capture variants remain unverified.
138
138
 
139
139
  See the [release scope](docs/validation.md#first-release-preparation) for the accepted limits and their evidence.
@@ -0,0 +1,42 @@
1
+ using System;
2
+ using System.Collections.Generic;
3
+ using System.Diagnostics;
4
+ using System.Threading;
5
+
6
+ partial class WindowsBridge {
7
+ // One call's budget and guarded waits; no held-input state or scheduling policy.
8
+ sealed class InputTiming {
9
+ internal readonly long TimeoutMs;
10
+ internal readonly string DeadlinePolicy;
11
+ readonly Stopwatch elapsed=Stopwatch.StartNew();
12
+ internal static long Milliseconds(Dictionary<string,object> args,string name,long fallback) {
13
+ object raw;if(!args.TryGetValue(name,out raw))return fallback;
14
+ if(!(raw is byte || raw is short || raw is int || raw is long || raw is float || raw is double || raw is decimal))throw new InvalidOperationException("INVALID_ARGUMENTS: "+name+" must be non-negative safe whole milliseconds");
15
+ double value=Convert.ToDouble(raw);
16
+ if(Double.IsNaN(value) || Double.IsInfinity(value) || value<0 || value>9007199254740991D || Math.Floor(value)!=value)throw new InvalidOperationException("INVALID_ARGUMENTS: "+name+" must be non-negative safe whole milliseconds");
17
+ return (long)value;
18
+ }
19
+ internal InputTiming(Dictionary<string,object> args,long requestedDuration,bool explicitDuration,long ordinaryTimeoutMs) {
20
+ if(args.ContainsKey("timeoutMs")){TimeoutMs=Milliseconds(args,"timeoutMs",0);DeadlinePolicy=TimeoutMs==0?"disabled":"explicit";}
21
+ else {
22
+ if(requestedDuration>9007199254740991L-ordinaryTimeoutMs)throw new InvalidOperationException("INVALID_ARGUMENTS: derived execution duration is not representable");
23
+ TimeoutMs=ordinaryTimeoutMs+requestedDuration;DeadlinePolicy=explicitDuration?"derived":"default";
24
+ }
25
+ }
26
+ internal void Check() {
27
+ if(TimeoutMs!=0 && elapsed.ElapsedMilliseconds>=TimeoutMs)throw new InvalidOperationException("WORKER_TIMEOUT: input exceeded the execution deadline; input was not replayed");
28
+ }
29
+ internal void Wait(long milliseconds,EventWaitHandle cancel,Action check) {
30
+ Stopwatch wait=Stopwatch.StartNew();
31
+ for(;;) {
32
+ Check();check();
33
+ long remaining=milliseconds-wait.ElapsedMilliseconds;
34
+ if(remaining<=0)return;
35
+ long interval=Math.Min(25L,remaining);
36
+ if(TimeoutMs!=0)interval=Math.Min(interval,Math.Max(1L,TimeoutMs-elapsed.ElapsedMilliseconds));
37
+ if(cancel.WaitOne((int)interval))throw new InvalidOperationException("COMPUTER_USE_STOPPED");
38
+ }
39
+ }
40
+ internal Dictionary<string,object> Describe() {return new Dictionary<string,object>{{"timeoutMs",TimeoutMs},{"deadlinePolicy",DeadlinePolicy}};}
41
+ }
42
+ }
package/assets/pointer.cs CHANGED
@@ -118,11 +118,18 @@ partial class WindowsBridge {
118
118
  Dictionary<string,object> result=Result(true,action);result["cleanupConfirmed"]=true;
119
119
  IntPtr window=IntPtr.Zero,before=IntPtr.Zero;uint owner=0;
120
120
  ControlPoint original=new ControlPoint(),last=original;
121
- bool moved=false,held=false,releaseFailed=false,activated=false;uint up=4;
121
+ bool moved=false,held=false,releaseFailed=false,activated=false,inputDispatched=false;uint up=4;
122
+ long holdMs=0;InputTiming timing=null;
122
123
  AutomationElement element=null;HashSet<long> popups=new HashSet<long>();
123
124
  InteractionGuard interaction=null;
124
125
  EventWaitHandle cancel=null;
125
126
  try {
127
+ if(action=="click") {
128
+ holdMs=InputTiming.Milliseconds(args,"holdMs",0);
129
+ timing=new InputTiming(args,holdMs,args.ContainsKey("holdMs"),40000);
130
+ var policy=timing.Describe();policy["holdMs"]=holdMs;result["timing"]=policy;
131
+ if((args.ContainsKey("holdMs") || args.ContainsKey("timeoutMs")) && (TextOption(args,"dispatch","auto")=="background" || (args.ContainsKey("activate") && !Option(args,"activate",false))))throw new InvalidOperationException("TIMING_UNSUPPORTED: click timing requires native foreground dispatch without explicit activate: false");
132
+ }
126
133
  interaction=new InteractionGuard(TextOption(args,"_cancelEvent",""),NumberOption(args,"_stopProcessId",0));before=GetForegroundWindow();GetCursorPos(out original);last=original;
127
134
  // Resolve without activation: validate identity and point before any takeover.
128
135
  Dictionary<string,object> quiet=new Dictionary<string,object>(args);quiet["activate"]=false;
@@ -139,7 +146,7 @@ partial class WindowsBridge {
139
146
  string button=TextOption(args,"button","left"),dispatch=TextOption(args,"dispatch","auto");
140
147
  if(action=="click" || action=="double_click" || action=="drag")ForgetTextFocus();
141
148
  if(element!=null && !element.Current.IsEnabled)throw new InvalidOperationException("ELEMENT_DISABLED");
142
- bool useBackground=dispatch=="background" || (action=="click" && element!=null && button=="left" && dispatch=="auto" && !Option(args,"activate",false));
149
+ bool useBackground=dispatch=="background" || (action=="click" && !args.ContainsKey("holdMs") && !args.ContainsKey("timeoutMs") && element!=null && button=="left" && dispatch=="auto" && !Option(args,"activate",false));
143
150
  if(useBackground && element!=null) {
144
151
  if(action=="click" && button=="left") {
145
152
  Dictionary<string,object> controlArgs=new Dictionary<string,object>(args);controlArgs["fallbackClick"]=true;
@@ -181,7 +188,7 @@ partial class WindowsBridge {
181
188
  int repetitions=action=="double_click"?2:1;
182
189
  for(int iteration=0;iteration<repetitions;iteration++) {
183
190
  for(int i=0;i<path.Count;i++) {
184
- interaction.Check();
191
+ if(timing!=null)timing.Check();interaction.Check();
185
192
  ControlPoint p=path[i],current;GetCursorPos(out current);
186
193
  if(moved && (current.X!=last.X || current.Y!=last.Y))throw new InvalidOperationException("USER_INPUT_CHANGED: pointer moved during automation");
187
194
  if(action=="drag" && i>0) {
@@ -197,9 +204,13 @@ partial class WindowsBridge {
197
204
  } else PointerCheck(cancel,window,owner,p,true,action!="move" || Option(args,"activate",false));
198
205
  if(action=="click" || action=="double_click" || (action=="drag" && i==0)) {
199
206
  // Set held before the down attempt: uncertain dispatch still requires an up.
200
- held=true;PointerSend(down,0);
207
+ held=true;inputDispatched=true;PointerSend(down,0);
201
208
  if(cancel.WaitOne(0))throw new InvalidOperationException("COMPUTER_USE_STOPPED");
202
209
  }
210
+ if(action=="click" && holdMs>0)timing.Wait(holdMs,cancel,delegate {
211
+ interaction.Check();PointerCheck(cancel,window,owner,p,true);
212
+ ControlPoint currentPointer;if(!GetCursorPos(out currentPointer) || currentPointer.X!=last.X || currentPointer.Y!=last.Y)throw new InvalidOperationException("USER_INPUT_CHANGED: pointer moved during automation");
213
+ });
203
214
  if(action=="click" || action=="double_click") {PointerSend(up,0);held=false;}
204
215
  if(action=="scroll") {int y=NumberOption(args,"deltaY",480),x=NumberOption(args,"deltaX",0);if(y!=0)PointerSend(0x800,-y);if(x!=0){PointerCheck(cancel,window,owner,p,true,action!="move" || Option(args,"activate",false));PointerSend(0x1000,x);}}
205
216
  if(action=="drag" && i<path.Count-1 && cancel.WaitOne(10))throw new InvalidOperationException("COMPUTER_USE_STOPPED");
@@ -212,6 +223,7 @@ partial class WindowsBridge {
212
223
  finally {
213
224
  if(held)try {PointerSend(up,0);held=false;}catch(Exception error){releaseFailed=true;result["ok"]=false;result["error"]="INPUT_CLEANUP_UNCONFIRMED: "+error.Message;}
214
225
  result["cleanupConfirmed"]=!releaseFailed;
226
+ if(action=="click"){result["inputDispatched"]=inputDispatched;result["interrupted"]=!Convert.ToBoolean(result["ok"]);}
215
227
  if(cancel!=null)cancel.Dispose();
216
228
  if(pointerListener!=null){pointerListener.Dispose();pointerListener=null;}
217
229
  ControlPoint now;bool same=CurrentReferenceIntact() && GetCursorPos(out now) && now.X==last.X && now.Y==last.Y;
package/assets/text.cs CHANGED
@@ -12,6 +12,7 @@ using System.Windows.Automation;
12
12
  partial class WindowsBridge {
13
13
  [StructLayout(LayoutKind.Sequential)] struct KeyboardInput {public ushort Key,Scan;public uint Flags,Time;public UIntPtr Extra;}
14
14
  const int UnicodeInputDelayMs=50;
15
+ static bool textInputDispatched;
15
16
  sealed class HeldKey {public ushort Key,Scan;public uint Flags;}
16
17
  [DllImport("user32.dll")] static extern IntPtr GetKeyboardLayout(uint thread);
17
18
  [DllImport("user32.dll",CharSet=CharSet.Unicode)] static extern short VkKeyScanEx(char value,IntPtr layout);
@@ -100,6 +101,7 @@ partial class WindowsBridge {
100
101
  if(!release){BeginWindowDispatch();NotifyPhysicalInput(true);}
101
102
  PointerInput input=new PointerInput{Type=1,Data=new InputUnion{Keyboard=new KeyboardInput{Key=key.Key,Scan=key.Scan,Flags=key.Flags | (release?2U:0U),Extra=OwnedInputMarker}}};
102
103
  if(SendInput(1,new PointerInput[]{input},Marshal.SizeOf(typeof(PointerInput)))!=1)throw new InvalidOperationException("KEY_DISPATCH_UNCONFIRMED: partial input must not be replayed");
104
+ if(!release)textInputDispatched=true;
103
105
  }
104
106
  static void Down(List<HeldKey> held,HeldKey key) {held.Add(key);KeyEvent(key,false);}
105
107
  static void Up(List<HeldKey> held,HeldKey key) {KeyEvent(key,true);held.Remove(key);}
@@ -107,22 +109,29 @@ partial class WindowsBridge {
107
109
  switch(key){case "enter":case "return":return 13;case "tab":return 9;case "esc":case "escape":return 27;case "backspace":return 8;case "delete":case "del":return 46;case "home":return 36;case "end":return 35;case "pageup":case "pgup":return 33;case "pagedown":case "pgdn":return 34;case "up":return 38;case "down":return 40;case "left":return 37;case "right":return 39;case "space":return 32;case "insert":case "ins":return 45;case "pause":return 19;case "capslock":return 20;case "numlock":return 144;case "scrolllock":return 145;}
108
110
  int number;if(key.StartsWith("f") && Int32.TryParse(key.Substring(1),out number) && number>=1 && number<=24)return (ushort)(111+number);return 0;
109
111
  }
110
- static void Chord(IEnumerable values,EventWaitHandle cancel,Process listener,IntPtr window,uint owner,List<HeldKey> held,InteractionGuard interaction) {
112
+ static void Chord(IEnumerable values,EventWaitHandle cancel,Process listener,IntPtr window,uint owner,List<HeldKey> held,InteractionGuard interaction,long holdMs=0,InputTiming timing=null) {
113
+ Action holdCheck=delegate {
114
+ TextCheck(cancel,listener,window,owner,IntPtr.Zero,false,interaction);
115
+ };
116
+ if(timing!=null)timing.Check();
111
117
  List<ushort> modifiers=new List<ushort>();string main=null;
112
118
  foreach(object item in values) {string key=Convert.ToString(item),name=key.Trim().ToLowerInvariant();if(name=="ctrl" || name=="control")modifiers.Add(17);else if(name=="alt" || name=="option")modifiers.Add(18);else if(name=="shift")modifiers.Add(16);else if(name=="win" || name=="windows" || name=="cmd" || name=="meta")throw new InvalidOperationException("KEY_BLOCKED");else if(main!=null)throw new InvalidOperationException("INVALID_ARGUMENTS: exactly one non-modifier key required");else main=key;}
113
- if(main==null)throw new InvalidOperationException("INVALID_ARGUMENTS: non-modifier key required");
114
- ushort code=SpecialKey(main.Trim().ToLowerInvariant());
115
- if(code==0) {
119
+ if(main==null && modifiers.Count!=1)throw new InvalidOperationException("INVALID_ARGUMENTS: one main key or standalone modifier required");
120
+ ushort code=main==null?(ushort)0:SpecialKey(main.Trim().ToLowerInvariant());
121
+ if(main!=null && code==0) {
116
122
  if(main.Length!=1)throw new InvalidOperationException("INVALID_ARGUMENTS: unsupported key");
117
123
  uint ignored;uint thread=GetWindowThreadProcessId(window,out ignored);short mapping=VkKeyScanEx(main[0],GetKeyboardLayout(thread));
118
- if(mapping==-1) {if(modifiers.Count>0)throw new InvalidOperationException("KEY_LAYOUT_UNSUPPORTED: chord cannot represent this character");TextCheck(cancel,listener,window,owner,IntPtr.Zero,false,interaction);HeldKey unicode=new HeldKey{Scan=main[0],Flags=4};Down(held,unicode);Up(held,unicode);return;}
124
+ if(mapping==-1) {if(modifiers.Count>0)throw new InvalidOperationException("KEY_LAYOUT_UNSUPPORTED: chord cannot represent this character");TextCheck(cancel,listener,window,owner,IntPtr.Zero,false,interaction);HeldKey unicode=new HeldKey{Scan=main[0],Flags=4};Down(held,unicode);if(timing!=null && holdMs>0)timing.Wait(holdMs,cancel,holdCheck);Up(held,unicode);return;}
119
125
  code=(ushort)(mapping & 255);
120
126
  // A named shortcut letter is layout-mapped without adding implicit Shift.
121
127
  if(modifiers.Count==0){if((mapping & 0x100)!=0)modifiers.Add(16);if((mapping & 0x200)!=0)modifiers.Add(17);if((mapping & 0x400)!=0)modifiers.Add(18);}
122
128
  }
123
129
  foreach(ushort modifier in modifiers){TextCheck(cancel,listener,window,owner,IntPtr.Zero,false,interaction);Down(held,new HeldKey{Key=modifier});}
124
130
  TextCheck(cancel,listener,window,owner,IntPtr.Zero,false,interaction);
125
- HeldKey primary=new HeldKey{Key=code,Flags=(code>=33 && code<=46)?1U:0U};Down(held,primary);Up(held,primary);
131
+ if(main!=null){HeldKey primary=new HeldKey{Key=code,Flags=(code>=33 && code<=46)?1U:0U};Down(held,primary);}
132
+ if(timing!=null && holdMs>0)timing.Wait(holdMs,cancel,holdCheck);
133
+ // Immediate presses release without post-dispatch checks: their effect may move focus.
134
+ // Positive holds retain the existing keypress target/protection checks during the wait.
126
135
  // Release modifiers regardless of target/focus changes caused by the chord.
127
136
  for(int i=held.Count-1;i>=0;i--){HeldKey key=held[i];Up(held,key);}
128
137
  }
@@ -148,13 +157,34 @@ partial class WindowsBridge {
148
157
  throw new InvalidOperationException("CLIPBOARD_RESTORE_UNCONFIRMED");
149
158
  }
150
159
  static object ExecuteText(string action,Dictionary<string,object> args) {
151
- Dictionary<string,object> result=Result(true,action);result["cleanupConfirmed"]=true;
160
+ Dictionary<string,object> result=Result(true,action);result["cleanupConfirmed"]=true;textInputDispatched=false;
152
161
  IntPtr before=IntPtr.Zero,window=IntPtr.Zero;ControlPoint pointer=new ControlPoint();
153
162
  List<HeldKey> held=new List<HeldKey>();HashSet<long> popups=new HashSet<long>();
154
163
  EventWaitHandle cancel=null;Process listener=null,clipboard=null;bool clipboardMayChange=false,clipboardReplaced=false;
155
164
  InteractionGuard interaction=null;
156
165
  AutomationElement main=null;bool cleanup=true;
157
166
  try {
167
+ long holdMs=0,inputDelayMs=UnicodeInputDelayMs;InputTiming timing=null;
168
+ string text=TextOption(args,"text",""),method=TextOption(args,"method","auto");
169
+ if(text.IndexOf('\0')>=0)throw new InvalidOperationException("INVALID_ARGUMENTS: NUL text cannot be entered safely");
170
+ if(action=="type_text") {
171
+ inputDelayMs=InputTiming.Milliseconds(args,"inputDelayMs",UnicodeInputDelayMs);
172
+ if((args.ContainsKey("inputDelayMs") || args.ContainsKey("timeoutMs")) && (method!="sendinput" || args.ContainsKey("browserTab")))throw new InvalidOperationException("TIMING_UNSUPPORTED: text timing requires explicit native sendinput");
173
+ if(method=="sendinput") {
174
+ long units=text.Replace("\r\n","\n").Length;
175
+ if(inputDelayMs!=0 && units>9007199254740991L/inputDelayMs)throw new InvalidOperationException("INVALID_ARGUMENTS: requested pacing duration is not representable");
176
+ long duration=args.ContainsKey("inputDelayMs")?units*inputDelayMs:0;
177
+ timing=new InputTiming(args,duration,args.ContainsKey("inputDelayMs"),40000);
178
+ var policy=timing.Describe();policy["inputDelayMs"]=inputDelayMs;result["timing"]=policy;
179
+ result["inputDelayMs"]=inputDelayMs;result["inputUnitsDispatched"]=0;
180
+ }
181
+ }
182
+ if(action=="keypress") {
183
+ holdMs=InputTiming.Milliseconds(args,"holdMs",0);
184
+ timing=new InputTiming(args,holdMs,args.ContainsKey("holdMs"),40000);
185
+ var policy=timing.Describe();policy["holdMs"]=holdMs;result["timing"]=policy;
186
+ if((args.ContainsKey("holdMs") || args.ContainsKey("timeoutMs")) && TextOption(args,"dispatch","foreground")=="background")throw new InvalidOperationException("TIMING_UNSUPPORTED: keypress timing requires native foreground dispatch");
187
+ }
158
188
  interaction=new InteractionGuard(TextOption(args,"_cancelEvent",""),NumberOption(args,"_stopProcessId",0));before=GetForegroundWindow();GetCursorPos(out pointer);
159
189
  Dictionary<string,object> quiet=new Dictionary<string,object>(args);quiet["activate"]=false;
160
190
  if(args.ContainsKey("_elementId")) {
@@ -165,15 +195,13 @@ partial class WindowsBridge {
165
195
  ApplyVisibleIntent(args,window);
166
196
  if(textFocus.Count>0 && !textFocus.ContainsKey(PointerOriginKey(window)))ForgetTextFocus();
167
197
  uint owner=WindowOwner(window);popups=ControlPopups(owner,window);
168
- string text=TextOption(args,"text",""),method=TextOption(args,"method","auto");
169
- if(text.IndexOf('\0')>=0)throw new InvalidOperationException("INVALID_ARGUMENTS: NUL text cannot be entered safely");
170
- if(action=="type_text" && method=="auto" && TextOption(args,"focusMode","preserve")=="preserve" && !Option(args,"activate",false) && BackgroundEdit(window,text,result)) {result["inputTargetHandle"]=window.ToInt64();result["inputTargetProcessId"]=owner;return result;}
198
+ if(action=="type_text" && method=="auto" && TextOption(args,"focusMode","preserve")=="preserve" && !Option(args,"activate",false) && BackgroundEdit(window,text,result)) {textInputDispatched=true;result["inputTargetHandle"]=window.ToInt64();result["inputTargetProcessId"]=owner;return result;}
171
199
  bool background=action=="type_text"?method=="background":TextOption(args,"dispatch","foreground")=="background";
172
200
  if(background) {
173
201
  if(!args.ContainsKey("windowRef") && !args.ContainsKey("windowTitle") && !args.ContainsKey("processId") && !args.ContainsKey("nativeWindowHandle"))throw new InvalidOperationException("BACKGROUND_TARGET_REQUIRED");
174
202
  string characters=action=="type_text"?text:Convert.ToString(((IList)args["keys"])[0]);
175
203
  IntPtr recipient=TextFocus(window);if(recipient==IntPtr.Zero)recipient=window;AssertWritableText(recipient);
176
- foreach(char character in characters) {CheckWindowReference(window);if(!IsWindow(window) || WindowOwner(window)!=owner || WindowOwner(recipient)!=owner)throw new InvalidOperationException("TARGET_IDENTITY_CHANGED");BeginWindowDispatch();if(!ControlPost(recipient,0x102,new IntPtr(character),new IntPtr(1)))throw new InvalidOperationException("TEXT_DISPATCH_UNCONFIRMED: some WM_CHAR messages may have been delivered; never replay");}
204
+ foreach(char character in characters) {CheckWindowReference(window);if(!IsWindow(window) || WindowOwner(window)!=owner || WindowOwner(recipient)!=owner)throw new InvalidOperationException("TARGET_IDENTITY_CHANGED");BeginWindowDispatch();if(!ControlPost(recipient,0x102,new IntPtr(character),new IntPtr(1)))throw new InvalidOperationException("TEXT_DISPATCH_UNCONFIRMED: some WM_CHAR messages may have been delivered; never replay");textInputDispatched=true;}
177
205
  result["method"]="postmessage-wmchar";result["verification"]="unconfirmed";result["verified"]=false;result["inputTargetHandle"]=window.ToInt64();result["inputTargetProcessId"]=owner;return result;
178
206
  }
179
207
  if(TextOption(args,"_cancelEvent","").Length==0)throw new InvalidOperationException("WCU_NEEDS_INPUT: physical text/key input requires independent stopping");
@@ -192,19 +220,20 @@ partial class WindowsBridge {
192
220
  IntPtr expectedFocus=main!=null?ConfirmTextFocus(main,cancel,listener,window,owner,interaction):TextFocus(window,false);if(main!=null){result["focusConfirmed"]=true;result["inputFocusHandle"]=expectedFocus.ToInt64();}if(action=="type_text")AssertWritableText(expectedFocus);
193
221
  TextCheck(cancel,listener,window,owner,expectedFocus,action=="type_text",interaction);
194
222
  if(action=="type_text")RememberTextFocus(window,expectedFocus);
195
- if(action=="keypress") {Chord((IEnumerable)args["keys"],cancel,listener,window,owner,held,interaction);result["method"]="sendinput-keychord";}
223
+ if(action=="keypress") {Chord((IEnumerable)args["keys"],cancel,listener,window,owner,held,interaction,holdMs,timing);result["method"]="sendinput-keychord";}
196
224
  else if(method=="sendinput") {
197
- // Modern Notepad also corrupts seeded/appended text with a 10 ms gap.
198
- // Keep complete pairs paced and cancellable; dispatch is still not verification.
225
+ // Preserve complete UTF-16 pairs and newline/tab handling at the selected pace.
226
+ // Dispatch is still not application verification.
199
227
  for(int index=0;index<text.Length;index++) {
200
228
  char character=text[index];
201
229
  if(character=='\r' && index+1<text.Length && text[index+1]=='\n')index++;
202
- TextCheck(cancel,listener,window,owner,expectedFocus,true,interaction);
230
+ timing.Check();TextCheck(cancel,listener,window,owner,expectedFocus,true,interaction);
203
231
  HeldKey key=character=='\r' || character=='\n'?new HeldKey{Key=13}:character=='\t'?new HeldKey{Key=9}:new HeldKey{Scan=character,Flags=4};
204
232
  Down(held,key);Up(held,key);
205
- if(cancel.WaitOne(UnicodeInputDelayMs))throw new InvalidOperationException("COMPUTER_USE_STOPPED");
233
+ result["inputUnitsDispatched"]=Convert.ToInt32(result["inputUnitsDispatched"])+1;
234
+ timing.Wait(inputDelayMs,cancel,delegate {TextCheck(cancel,listener,window,owner,expectedFocus,true,interaction);});
206
235
  }
207
- result["method"]="sendinput-unicode";result["inputDelayMs"]=UnicodeInputDelayMs;
236
+ result["method"]="sendinput-unicode";
208
237
  }else {
209
238
  // All UIA focus/provider calls finish BEFORE clipboard replacement.
210
239
  ProcessStartInfo info=new ProcessStartInfo{FileName=TextOption(args,"_clipboardExecutable",""),UseShellExecute=false,CreateNoWindow=true,RedirectStandardInput=true,RedirectStandardOutput=true,RedirectStandardError=true,StandardOutputEncoding=Encoding.UTF8,StandardErrorEncoding=Encoding.UTF8};
@@ -239,6 +268,7 @@ partial class WindowsBridge {
239
268
  }
240
269
  if(cancel!=null)cancel.Dispose();if(listener!=null)listener.Dispose();
241
270
  result["cleanupConfirmed"]=cleanup;
271
+ if(action=="keypress" || action=="type_text"){result["inputDispatched"]=textInputDispatched;result["interrupted"]=!Convert.ToBoolean(result["ok"]); }
242
272
  // No potentially blocked UIA verification is permitted before clipboard cleanup.
243
273
  if(cleanup)RestoreControlState(result,before,pointer,window,popups,Option(args,"activate",false),interaction);
244
274
  else {result["pointer"]="left: cleanup unconfirmed";result["foreground"]="left: cleanup unconfirmed";}
@@ -17,5 +17,6 @@ export declare const browserActionTools: {
17
17
  additionalProperties: boolean;
18
18
  };
19
19
  }[];
20
+ export declare function browserActionRefusal(action: string, args: Record<string, unknown>): string | undefined;
20
21
  export declare function normalizeBrowserAction(action: string, args: Record<string, unknown>): Record<string, unknown> | undefined;
21
22
  export {};
@@ -1,3 +1,4 @@
1
+ import { timingMilliseconds } from "./input-timing.js";
1
2
  // Shared reference-compatible tool schemas; browser execution stays in browser.ts.
2
3
  const target = {
3
4
  windowRef: { type: "string", minLength: 1 },
@@ -29,7 +30,16 @@ const dispatch = {
29
30
  };
30
31
  const definitions = {
31
32
  click: {
32
- properties: { ...target, ...view, ...element, ...point, button, dispatch },
33
+ properties: {
34
+ ...target,
35
+ ...view,
36
+ ...element,
37
+ ...point,
38
+ button,
39
+ dispatch,
40
+ holdMs: timingMilliseconds,
41
+ timeoutMs: timingMilliseconds,
42
+ },
33
43
  },
34
44
  double_click: {
35
45
  properties: { ...target, ...view, ...element, ...point, button, dispatch },
@@ -63,11 +73,15 @@ const definitions = {
63
73
  default: "preserve",
64
74
  },
65
75
  restoreClipboard: { type: "boolean", default: true },
76
+ inputDelayMs: timingMilliseconds,
77
+ timeoutMs: timingMilliseconds,
66
78
  },
67
79
  },
68
80
  keypress: {
69
81
  required: ["keys"],
70
82
  properties: {
83
+ holdMs: timingMilliseconds,
84
+ timeoutMs: timingMilliseconds,
71
85
  ...target,
72
86
  ...tab,
73
87
  keys: {
@@ -153,7 +167,26 @@ export const browserActionTools = Object.entries(definitions).map(([action, defi
153
167
  action,
154
168
  parameters: { type: "object", additionalProperties: false, ...definition },
155
169
  }));
170
+ export function browserActionRefusal(action, args) {
171
+ if (action === "click" &&
172
+ (args.holdMs !== undefined || args.timeoutMs !== undefined))
173
+ return "TIMING_UNSUPPORTED: click timing requires native foreground dispatch";
174
+ if (action === "keypress" &&
175
+ (args.holdMs !== undefined ||
176
+ args.timeoutMs !== undefined ||
177
+ (Array.isArray(args.keys) &&
178
+ args.keys.length === 1 &&
179
+ /^(ctrl|control|alt|option|shift)$/i.test(String(args.keys[0]).trim()))))
180
+ return "TIMING_UNSUPPORTED: keypress timing and standalone modifiers require native foreground dispatch";
181
+ if (action === "type_text" &&
182
+ (args.inputDelayMs !== undefined || args.timeoutMs !== undefined))
183
+ return "TIMING_UNSUPPORTED: text timing requires explicit native sendinput";
184
+ return undefined;
185
+ }
156
186
  export function normalizeBrowserAction(action, args) {
187
+ const refusal = browserActionRefusal(action, args);
188
+ if (refusal)
189
+ throw new Error(refusal);
157
190
  const definition = definitions[action];
158
191
  if (!definition)
159
192
  return undefined;
package/dist/client.d.ts CHANGED
@@ -27,6 +27,7 @@ export declare class WorkerClient {
27
27
  private shutdownChanged;
28
28
  private readonly lifetime;
29
29
  private queue;
30
+ private readonly deadlineCleanup;
30
31
  constructor(options: WorkerClientOptions);
31
32
  private start;
32
33
  private fail;
package/dist/client.js CHANGED
@@ -1,7 +1,11 @@
1
1
  import { spawn } from "node:child_process";
2
- function abortable(operation, signal) {
2
+ import { executionDeadline, pressTiming, textTiming } from "./input-timing.js";
3
+ function abortable(operation, signal, retain) {
3
4
  return new Promise((resolve, reject) => {
4
- const abort = () => reject(signal.reason);
5
+ const abort = () => {
6
+ if (!retain?.())
7
+ reject(signal.reason);
8
+ };
5
9
  signal.addEventListener("abort", abort, { once: true });
6
10
  if (signal.aborted)
7
11
  abort();
@@ -24,6 +28,7 @@ export class WorkerClient {
24
28
  shutdownChanged;
25
29
  lifetime = new AbortController();
26
30
  queue = Promise.resolve();
31
+ deadlineCleanup = new Set();
27
32
  constructor(options) {
28
33
  this.options = options;
29
34
  }
@@ -180,31 +185,63 @@ export class WorkerClient {
180
185
  return this.executeNow(action, args, callSignal, operationId);
181
186
  });
182
187
  this.queue = result.catch(() => undefined);
183
- return abortable(result, callSignal);
188
+ return abortable(result, callSignal, () => this.deadlineCleanup.has(callSignal));
184
189
  }
185
190
  async executeNow(action, args, signal, operationId) {
191
+ let inputPolicy;
192
+ if (action === "keypress" || action === "click" || action === "type_text") {
193
+ try {
194
+ inputPolicy = (action === "type_text" ? textTiming : pressTiming)(args, this.options.timeoutMs ?? 30_000);
195
+ }
196
+ catch (error) {
197
+ return {
198
+ ok: false,
199
+ action,
200
+ error: error instanceof Error ? error.message : String(error),
201
+ };
202
+ }
203
+ }
186
204
  const child = await this.start(signal);
187
205
  if (this.disposed)
188
206
  throw new Error("WORKER_DISPOSED: session has closed");
189
207
  signal?.throwIfAborted();
190
208
  const id = ++this.nextId;
191
209
  return new Promise((resolve, reject) => {
192
- const timer = setTimeout(() => {
210
+ let expired = false;
211
+ let cleanupTimer;
212
+ const timeoutError = () => new Error(`WORKER_TIMEOUT: ${action} exceeded the execution deadline; input was not replayed`);
213
+ const timer = executionDeadline(inputPolicy?.timeoutMs ??
214
+ Math.max(1, this.options.timeoutMs ?? 30_000, action === "wait" &&
215
+ Number.isSafeInteger(args.milliseconds ?? 500) &&
216
+ Number(args.milliseconds ?? 500) >= 1 &&
217
+ Number(args.milliseconds ?? 500) <= 30000
218
+ ? Number(args.milliseconds ?? 500) + 2000
219
+ : 0, action === "wait_for" &&
220
+ Number.isSafeInteger(args.timeoutMs ?? 5000) &&
221
+ Number(args.timeoutMs ?? 5000) >= 100 &&
222
+ Number(args.timeoutMs ?? 5000) <= 60000
223
+ ? Number(args.timeoutMs ?? 5000) + 10000
224
+ : 0), () => {
225
+ if (inputPolicy) {
226
+ expired = true;
227
+ if (signal)
228
+ this.deadlineCleanup.add(signal);
229
+ // Keep the request and its owner until cooperative release is acknowledged.
230
+ child.stdin.write(`${JSON.stringify({ cancel: id })}\n`);
231
+ cleanupTimer = setTimeout(() => {
232
+ this.fail(child, timeoutError());
233
+ if (!this.inputActive && this.options.canTerminate?.() !== false)
234
+ child.kill();
235
+ }, 5000);
236
+ return;
237
+ }
193
238
  this.fail(child, new Error(`WORKER_TIMEOUT: ${action} exceeded the execution deadline; input was not replayed`));
194
239
  if (!this.inputActive && this.options.canTerminate?.() !== false)
195
240
  child.kill();
196
- }, Math.max(this.options.timeoutMs ?? 30_000, action === "wait" &&
197
- Number.isSafeInteger(args.milliseconds ?? 500) &&
198
- Number(args.milliseconds ?? 500) >= 1 &&
199
- Number(args.milliseconds ?? 500) <= 30000
200
- ? Number(args.milliseconds ?? 500) + 2000
201
- : 0, action === "wait_for" &&
202
- Number.isSafeInteger(args.timeoutMs ?? 5000) &&
203
- Number(args.timeoutMs ?? 5000) >= 100 &&
204
- Number(args.timeoutMs ?? 5000) <= 60000
205
- ? Number(args.timeoutMs ?? 5000) + 10000
206
- : 0));
241
+ });
207
242
  const abort = () => {
243
+ if (expired)
244
+ return;
208
245
  const pending = this.pending.get(id);
209
246
  if (!pending)
210
247
  return;
@@ -216,10 +253,26 @@ export class WorkerClient {
216
253
  };
217
254
  this.pending.set(id, {
218
255
  operationId,
219
- resolve,
256
+ resolve: (result) => resolve(expired
257
+ ? {
258
+ ...result,
259
+ ok: false,
260
+ error: result.cleanupConfirmed === false ||
261
+ /CLEANUP_UNCONFIRMED/.test(String(result.error))
262
+ ? (result.error ??
263
+ "INPUT_CLEANUP_UNCONFIRMED: deadline expired before release was confirmed")
264
+ : timeoutError().message,
265
+ executionDeadlineExpired: true,
266
+ interrupted: true,
267
+ timing: inputPolicy,
268
+ }
269
+ : result),
220
270
  reject,
221
271
  cleanup: () => {
222
- clearTimeout(timer);
272
+ timer();
273
+ if (signal)
274
+ this.deadlineCleanup.delete(signal);
275
+ clearTimeout(cleanupTimer);
223
276
  signal?.removeEventListener("abort", abort);
224
277
  },
225
278
  });
package/dist/extension.js CHANGED
@@ -18,13 +18,13 @@ const actionDescriptions = {
18
18
  accessibility_tree: "Read a bounded UIA tree or browser page outline with element IDs. Partial trees do not establish absence.",
19
19
  find: "Find UIA or page controls; sparse UIA searches may add OCR. OCR hits use physical screen coordinates, not element IDs.",
20
20
  element_info: "Inspect a fresh UIA or page element ID, or a physical screen point.",
21
- click: "Click a control or physical screen point; background delivery is unverified. Browser page IDs act inside the page. Observe effects before retrying.",
21
+ click: "Click a control or physical screen point. Native foreground holdMs presses and releases left/right/middle in this call; timeoutMs: 0 disables only its execution deadline. Explicit timing refuses browser/background and activate: false routes. Omitted timing preserves defaults; background delivery is unverified. Observe effects before retrying.",
22
22
  double_click: "Double-click a control or physical screen point; background delivery is unverified. Browser page IDs act inside the page. Observe effects before retrying.",
23
23
  move: "Move the pointer to a control or physical screen point; browser page IDs act inside the page. Observe hover effects.",
24
24
  drag: "Drag a path of physical screen coordinates using guarded foreground input. Observe the drop and cleanup before retrying.",
25
25
  scroll: "Scroll a control or physical screen point; native element scrolling tries background first. Positive deltaY scrolls down. Browser tabs scroll inside the page.",
26
- type_text: "Type into the focused control. Native auto tries a background edit then clipboard paste; clipboard restoration is requested by default. Background WM_CHAR is unverified.",
27
- keypress: 'Send a key or chord, for example ["ctrl","a"]. Windows keys are blocked; native background dispatch supports one printable character only.',
26
+ type_text: "Type into the focused control. Native auto tries a background edit then clipboard paste; clipboard restoration is requested by default. Explicit native sendinput supports inputDelayMs (default 50; zero adds no delay) and timeoutMs (zero disables execution deadline). Other routes refuse timing fields. Partial input is never replayed; observe text to verify it. Background WM_CHAR is unverified.",
27
+ keypress: "Send a key/chord or standalone Ctrl, Alt or Shift. Native foreground holdMs completes and releases in this call; timeoutMs: 0 disables only its execution deadline. Omitted timing preserves defaults. Timing and modifier taps refuse browser/background routes; Windows keys remain blocked. Observe effects before retrying.",
28
28
  focus: "Focus a fresh UIA or browser page element; observe the intended focus before typing.",
29
29
  invoke: "Invoke, toggle or select a control, or request expand, collapse or scroll-into-view. Optional native click fallback is guarded; verify its effect.",
30
30
  set_value: 'Set a native or browser field value with guarded fallback. Browser "new-tab" opens a dedicated automation tab only when that workspace is explicitly requested; "<tab>:address" navigates it.',
@@ -0,0 +1,24 @@
1
+ /** Shared per-call timing policy; durations are milliseconds, zero disables only a deadline. */
2
+ export declare const timingMilliseconds: {
3
+ type: string;
4
+ minimum: number;
5
+ maximum: number;
6
+ };
7
+ export declare function milliseconds(value: unknown, name: string): number;
8
+ export declare function inputTiming(args: Record<string, unknown>, requestedDurationMs: number, ordinaryTimeoutMs: number): {
9
+ timeoutMs: number;
10
+ deadlinePolicy: "disabled" | "explicit" | "derived" | "default";
11
+ };
12
+ export declare function pressTiming(args: Record<string, unknown>, ordinaryTimeoutMs: number): {
13
+ timeoutMs: number;
14
+ deadlinePolicy: "disabled" | "explicit" | "derived" | "default";
15
+ holdMs: number;
16
+ };
17
+ /** Count dispatched UTF-16 units, treating CRLF as the existing single Enter. */
18
+ export declare function textTiming(args: Record<string, unknown>, ordinaryTimeoutMs: number): {
19
+ timeoutMs: number;
20
+ deadlinePolicy: "disabled" | "explicit" | "derived" | "default";
21
+ inputDelayMs: number;
22
+ };
23
+ /** Node timers accept signed 32-bit intervals; chain monotonic chunks without wrapping. */
24
+ export declare function executionDeadline(milliseconds: number, expired: () => void): () => void;
@@ -0,0 +1,70 @@
1
+ /** Shared per-call timing policy; durations are milliseconds, zero disables only a deadline. */
2
+ export const timingMilliseconds = {
3
+ type: "integer",
4
+ minimum: 0,
5
+ maximum: Number.MAX_SAFE_INTEGER,
6
+ };
7
+ export function milliseconds(value, name) {
8
+ if (!Number.isSafeInteger(value) || Number(value) < 0)
9
+ throw new Error(`INVALID_ARGUMENTS: ${name} must be non-negative safe whole milliseconds`);
10
+ return Number(value);
11
+ }
12
+ export function inputTiming(args, requestedDurationMs, ordinaryTimeoutMs) {
13
+ milliseconds(requestedDurationMs, "requested duration");
14
+ if (args.timeoutMs !== undefined) {
15
+ const timeoutMs = milliseconds(args.timeoutMs, "timeoutMs");
16
+ return {
17
+ timeoutMs,
18
+ deadlinePolicy: timeoutMs === 0 ? "disabled" : "explicit",
19
+ };
20
+ }
21
+ const timeoutMs = ordinaryTimeoutMs + requestedDurationMs;
22
+ if (!Number.isSafeInteger(timeoutMs))
23
+ throw new Error("INVALID_ARGUMENTS: derived execution duration is not representable");
24
+ return {
25
+ timeoutMs,
26
+ deadlinePolicy: args.holdMs !== undefined || args.inputDelayMs !== undefined
27
+ ? "derived"
28
+ : "default",
29
+ };
30
+ }
31
+ export function pressTiming(args, ordinaryTimeoutMs) {
32
+ const holdMs = args.holdMs === undefined ? 0 : milliseconds(args.holdMs, "holdMs");
33
+ return { holdMs, ...inputTiming(args, holdMs, ordinaryTimeoutMs) };
34
+ }
35
+ /** Count dispatched UTF-16 units, treating CRLF as the existing single Enter. */
36
+ export function textTiming(args, ordinaryTimeoutMs) {
37
+ const inputDelayMs = args.inputDelayMs === undefined
38
+ ? 50
39
+ : milliseconds(args.inputDelayMs, "inputDelayMs");
40
+ if (args.inputDelayMs !== undefined || args.timeoutMs !== undefined) {
41
+ if (args.method !== "sendinput" || args.browserTab !== undefined)
42
+ throw new Error("TIMING_UNSUPPORTED: text timing requires explicit native sendinput");
43
+ }
44
+ const units = typeof args.text === "string" ? args.text.replace(/\r\n/g, "\n").length : 0;
45
+ const requestedDuration = args.inputDelayMs === undefined ? 0 : units * inputDelayMs;
46
+ if (!Number.isSafeInteger(requestedDuration))
47
+ throw new Error("INVALID_ARGUMENTS: requested pacing duration is not representable");
48
+ return {
49
+ inputDelayMs,
50
+ ...inputTiming(args, requestedDuration, ordinaryTimeoutMs),
51
+ };
52
+ }
53
+ /** Node timers accept signed 32-bit intervals; chain monotonic chunks without wrapping. */
54
+ export function executionDeadline(milliseconds, expired) {
55
+ let timer;
56
+ let remaining = milliseconds;
57
+ let previous = performance.now();
58
+ const arm = () => {
59
+ const now = performance.now();
60
+ remaining -= now - previous;
61
+ previous = now;
62
+ if (remaining <= 0)
63
+ expired();
64
+ else
65
+ timer = setTimeout(arm, Math.min(remaining, 2_147_483_647));
66
+ };
67
+ if (milliseconds !== 0)
68
+ arm();
69
+ return () => clearTimeout(timer);
70
+ }
package/dist/native.js CHANGED
@@ -127,6 +127,7 @@ export const nativeDesktop = new NativeDesktopClient({
127
127
  "pointer.cs",
128
128
  "browser-indicators.cs",
129
129
  "text.cs",
130
+ "input-timing.cs",
130
131
  ];
131
132
  const framework = join(process.env.WINDIR ?? "", "Microsoft.NET", "Framework64", "v4.0.30319");
132
133
  return compileNative(names, [