dsh-codex-style-computer-use 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -15
- package/README.zh.md +23 -15
- package/icon.svg +29 -0
- package/lib/bin/attention-worker.exe +0 -0
- package/lib/bin/attention-worker.exe.json +1 -1
- package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
- package/lib/core/bin/dsh-computer-use-worker.exe.json +2 -2
- package/lib/core/compile-worker.ps1 +5 -1
- package/lib/core/index.js +98 -11
- package/lib/core/native-runtime.js +6 -1
- package/lib/core/winmd/Windows.winmd +0 -0
- package/lib/core/worker.cs +653 -155
- package/lib/index.js +17 -0
- package/lib/policy.js +23 -3
- package/lib/tools.js +40 -2
- package/package.json +4 -2
- package/scripts/render-promo.py +81 -0
- package/skills/computer-use/SKILL.md +17 -2
- package/skills/computer-use/references/lifecycle.md +2 -2
- package/skills/computer-use/references/zero-uia-apps.md +54 -0
package/lib/core/worker.cs
CHANGED
|
@@ -24,11 +24,15 @@ using System.Drawing.Imaging;
|
|
|
24
24
|
using System.IO;
|
|
25
25
|
using System.Linq;
|
|
26
26
|
using System.Runtime.InteropServices;
|
|
27
|
+
using System.Runtime.InteropServices.WindowsRuntime; // AsBuffer: byte[] -> WinRT IBuffer
|
|
27
28
|
using System.Text;
|
|
28
29
|
using System.Threading;
|
|
29
30
|
using System.Web.Script.Serialization;
|
|
30
31
|
using System.Windows.Automation;
|
|
31
32
|
using System.Windows.Forms; // Clipboard (watch out: bare Point stays our own struct)
|
|
33
|
+
// Inbox OCR (zero external dependencies): the union Windows.winmd ships in lib/core/winmd.
|
|
34
|
+
using Windows.Media.Ocr;
|
|
35
|
+
using Windows.Graphics.Imaging;
|
|
32
36
|
|
|
33
37
|
namespace DshComputerUse.Worker
|
|
34
38
|
{
|
|
@@ -81,7 +85,14 @@ namespace DshComputerUse.Worker
|
|
|
81
85
|
}
|
|
82
86
|
}
|
|
83
87
|
void Start(long now) { phase = "yielding"; first = now; if (!seen) last = now; burst++; }
|
|
84
|
-
public void Record(int key, bool down, bool up, long now)
|
|
88
|
+
public void Record(int key, bool down, bool up, long now) { Record(key, down, up, now, false); }
|
|
89
|
+
/// A DELIBERATE pointer move (the 16-px tolerance in PointerHandoffTolerance has already
|
|
90
|
+
/// filtered jitter and agent motion) yields even BETWEEN actuations. Human spec
|
|
91
|
+
/// 2026-11-04: while a cycle is live, the human reaching for the mouse is a takeover no
|
|
92
|
+
/// matter whether an injection happens to be in flight at that instant — the old gate
|
|
93
|
+
/// (`idle && !acting`) only lit the yield during the few dozen milliseconds of each
|
|
94
|
+
/// click, so an interruption in the gaps showed no light at all.
|
|
95
|
+
public void Record(int key, bool down, bool up, long now, bool deliberate)
|
|
85
96
|
{
|
|
86
97
|
lock (sync)
|
|
87
98
|
{
|
|
@@ -89,7 +100,7 @@ namespace DshComputerUse.Worker
|
|
|
89
100
|
lastKind = key == 0 ? "pointer-or-wheel" : key <= 6 ? "button" : "keyboard";
|
|
90
101
|
if (key != 0) { if (down) held.Add(key); if (up) held.Remove(key); }
|
|
91
102
|
seen = true; last = now;
|
|
92
|
-
if (!enabled || (phase == "idle" && !acting)) return;
|
|
103
|
+
if (!enabled || (phase == "idle" && !acting && !deliberate)) return;
|
|
93
104
|
revision++;
|
|
94
105
|
if (phase == "idle" || !continued) Start(now);
|
|
95
106
|
if (now - first >= SustainedMs) phase = "waiting";
|
|
@@ -113,6 +124,39 @@ namespace DshComputerUse.Worker
|
|
|
113
124
|
}
|
|
114
125
|
}
|
|
115
126
|
|
|
127
|
+
// The UAC secure desktop (and the lock screen) is the ONE surface automation must never
|
|
128
|
+
// reach — Windows enforces that itself. The honest addition (human request 2026-11-04)
|
|
129
|
+
// is to NOTICE the switch, refuse to dispatch while it stands, tell the human exactly
|
|
130
|
+
// what to do, and auto-continue the moment the normal desktop is back. Detection is an
|
|
131
|
+
// OpenInputDesktop probe: while the secure desktop is up, an unprivileged thread cannot
|
|
132
|
+
// open the input desktop at all. Transitions are confirmed over consecutive misses so a
|
|
133
|
+
// single hiccup never pauses dispatch, and one success clears the state immediately.
|
|
134
|
+
public sealed class SecureDesktopWatch
|
|
135
|
+
{
|
|
136
|
+
public const int ConfirmMisses = 3;
|
|
137
|
+
readonly object sync = new object();
|
|
138
|
+
bool active; int misses;
|
|
139
|
+
public bool Active { get { lock (sync) { return active; } } }
|
|
140
|
+
/// Returns true only on a STATE TRANSITION; the caller audits and notifies those alone.
|
|
141
|
+
public bool Observe(bool reachable)
|
|
142
|
+
{
|
|
143
|
+
lock (sync)
|
|
144
|
+
{
|
|
145
|
+
if (reachable)
|
|
146
|
+
{
|
|
147
|
+
misses = 0;
|
|
148
|
+
if (!active) return false;
|
|
149
|
+
active = false;
|
|
150
|
+
return true; // secured -> normal: dispatch may resume
|
|
151
|
+
}
|
|
152
|
+
if (active) return false;
|
|
153
|
+
if (++misses < ConfirmMisses) return false;
|
|
154
|
+
active = true;
|
|
155
|
+
return true; // normal -> secured
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
116
160
|
// Sum only physical displacement, rebasing every injected cursor event. Agent
|
|
117
161
|
// motion cannot turn a 1-pixel hand tremor into a takeover. Back-and-forth jitter
|
|
118
162
|
// cancels rather than accumulating path length; a deliberate 16-pixel move yields.
|
|
@@ -216,7 +260,12 @@ namespace DshComputerUse.Worker
|
|
|
216
260
|
if (!active || paused || exited) return null;
|
|
217
261
|
lock (sync)
|
|
218
262
|
{
|
|
219
|
-
if (hwnd == 0 || pid == 0)
|
|
263
|
+
if (hwnd == 0 || pid == 0)
|
|
264
|
+
{
|
|
265
|
+
const string missing = "this worker has no confirmed target for the current cycle";
|
|
266
|
+
if (host || foreground == 0 || process == 0) return missing;
|
|
267
|
+
return missing + ". Observed foreground hwnd=" + foreground + ", pid=" + process;
|
|
268
|
+
}
|
|
220
269
|
if (!targetExists) return "the activated target window no longer exists";
|
|
221
270
|
if (host) return "the agent host is in front of the target window";
|
|
222
271
|
if (hwnd != foreground || pid != process) return "the foreground no longer matches the activated target";
|
|
@@ -428,6 +477,13 @@ namespace DshComputerUse.Worker
|
|
|
428
477
|
public static extern bool DrawIconEx(IntPtr hdc, int xLeft, int yTop, IntPtr hIcon, int cxWidth, int cyHeight, int istepIfAniCur, IntPtr hbrFlickerFreeDraw, uint diFlags);
|
|
429
478
|
[DllImport("user32.dll")]
|
|
430
479
|
public static extern uint SendNotifyMessage(IntPtr hWnd, uint msg, IntPtr w, IntPtr l);
|
|
480
|
+
// Secure-desktop probe: while the UAC prompt (or the lock screen) is up, an unprivileged
|
|
481
|
+
// thread CANNOT open the input desktop — that refusal is the only reliable, permission-free
|
|
482
|
+
// witness that the machine switched to the surface automation must never touch.
|
|
483
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
484
|
+
public static extern IntPtr OpenInputDesktop(uint flags, bool inherit, uint access);
|
|
485
|
+
[DllImport("user32.dll")]
|
|
486
|
+
public static extern bool CloseDesktop(IntPtr h);
|
|
431
487
|
|
|
432
488
|
// ---- low-level keyboard hook: the human's emergency brake ----
|
|
433
489
|
public delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam);
|
|
@@ -454,7 +510,7 @@ namespace DshComputerUse.Worker
|
|
|
454
510
|
[StructLayout(LayoutKind.Sequential)]
|
|
455
511
|
public struct KBDLLHOOKSTRUCT { public uint vkCode, scanCode, flags, time; public IntPtr dwExtraInfo; }
|
|
456
512
|
[StructLayout(LayoutKind.Sequential)]
|
|
457
|
-
public struct NativeMsg { public IntPtr hwnd; public uint message; public IntPtr wParam, lParam; public uint time; public int ptX, ptY; }
|
|
513
|
+
public struct NativeMsg { public IntPtr hwnd; public uint message; public IntPtr wParam, lParam; public uint time; public int ptX, ptY; public uint lPrivate; }
|
|
458
514
|
public const int WH_KEYBOARD_LL = 13, WM_KEYDOWN_ = 0x0100, WM_SYSKEYDOWN_ = 0x0104,
|
|
459
515
|
WM_KEYUP_ = 0x0101, WM_SYSKEYUP_ = 0x0105;
|
|
460
516
|
// KBDLLHOOKSTRUCT.flags bit 4: the key did NOT come from the physical keyboard — some
|
|
@@ -494,7 +550,6 @@ namespace DshComputerUse.Worker
|
|
|
494
550
|
[StructLayout(LayoutKind.Sequential)] struct SIZEL { public int cx, cy; }
|
|
495
551
|
[StructLayout(LayoutKind.Sequential, Pack = 1)]
|
|
496
552
|
struct BLENDFUNCTION { public byte BlendOp, BlendFlags, SourceConstantAlpha, AlphaFormat; }
|
|
497
|
-
[StructLayout(LayoutKind.Sequential)] struct MSG { public IntPtr hwnd; public uint message; public IntPtr wParam, lParam; public uint time; public int ptX, ptY; }
|
|
498
553
|
|
|
499
554
|
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
|
500
555
|
static extern IntPtr CreateWindowEx(int exStyle, string cls, string name, int style,
|
|
@@ -504,13 +559,14 @@ namespace DshComputerUse.Worker
|
|
|
504
559
|
[DllImport("user32.dll")] static extern bool SetWindowPos(IntPtr h, IntPtr after, int x, int y, int cx, int cy, uint flags);
|
|
505
560
|
[DllImport("user32.dll")] static extern IntPtr GetDC(IntPtr h);
|
|
506
561
|
[DllImport("user32.dll")] static extern int ReleaseDC(IntPtr h, IntPtr dc);
|
|
507
|
-
[DllImport("user32.dll")] static extern bool PeekMessage(out
|
|
562
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern bool PeekMessage(out Native.NativeMsg m, IntPtr h, uint min, uint max, uint remove);
|
|
563
|
+
[DllImport("user32.dll")] static extern bool ValidateRect(IntPtr h, IntPtr rect);
|
|
508
564
|
[DllImport("gdi32.dll")] static extern IntPtr CreateCompatibleDC(IntPtr dc);
|
|
509
565
|
[DllImport("gdi32.dll")] static extern bool DeleteDC(IntPtr dc);
|
|
510
566
|
[DllImport("gdi32.dll")] static extern IntPtr SelectObject(IntPtr dc, IntPtr o);
|
|
511
567
|
[DllImport("gdi32.dll")] static extern bool DeleteObject(IntPtr o);
|
|
512
568
|
[DllImport("gdi32.dll")] static extern IntPtr CreateDIBSection(IntPtr dc, ref Native.BITMAPINFO bmi, uint usage, out IntPtr bits, IntPtr section, uint offset);
|
|
513
|
-
[DllImport("user32.dll")] static extern bool UpdateLayeredWindow(IntPtr h, IntPtr dst, ref POINTL pptDst, ref SIZEL psize,
|
|
569
|
+
[DllImport("user32.dll", SetLastError = true)] static extern bool UpdateLayeredWindow(IntPtr h, IntPtr dst, ref POINTL pptDst, ref SIZEL psize,
|
|
514
570
|
IntPtr src, ref POINTL pptSrc, int colorKey, ref BLENDFUNCTION blend, int flags);
|
|
515
571
|
[DllImport("user32.dll")] static extern int GetSystemMetrics(int i);
|
|
516
572
|
[DllImport("user32.dll")] static extern bool SetWindowDisplayAffinity(IntPtr hwnd, uint affinity);
|
|
@@ -519,8 +575,29 @@ namespace DshComputerUse.Worker
|
|
|
519
575
|
static volatile bool _acting;
|
|
520
576
|
static volatile bool _preparing;
|
|
521
577
|
static volatile int _preparationStart;
|
|
522
|
-
static int _preparationSerial, _presentedPreparation;
|
|
523
|
-
|
|
578
|
+
static int _preparationSerial, _presentedPreparation, _completedPreparation;
|
|
579
|
+
static volatile string _rendererStage = "not-started", _rendererFault;
|
|
580
|
+
static volatile int _lastLoopTick, _lastPresentTick, _lastFailedError;
|
|
581
|
+
static int _frames, _presentAttempts, _presentSuccesses, _lastAttemptWant, _lastAttemptSerial;
|
|
582
|
+
static int _lastAttemptedPreparationSerial = -1, _lastFailedSerial = -1, _lastFailedWant, _rebuiltSerial = -1;
|
|
583
|
+
public static Dictionary<string, object> Health()
|
|
584
|
+
{
|
|
585
|
+
var thread = _thread;
|
|
586
|
+
int now = Environment.TickCount;
|
|
587
|
+
return Program.Dict("workerPid", System.Diagnostics.Process.GetCurrentProcess().Id,
|
|
588
|
+
"threadAlive", thread != null && thread.IsAlive, "stage", _rendererStage, "fault", _rendererFault,
|
|
589
|
+
"want", Want, "preparationSerial", Volatile.Read(ref _preparationSerial), "presentedSerial", Volatile.Read(ref _presentedPreparation),
|
|
590
|
+
"completedSerial", Volatile.Read(ref _completedPreparation),
|
|
591
|
+
"loopAgeMs", Volatile.Read(ref _frames) == 0 ? (object)null : unchecked((uint)(now - _lastLoopTick)),
|
|
592
|
+
"presentAttempts", Volatile.Read(ref _presentAttempts), "lastAttemptWant", Volatile.Read(ref _lastAttemptWant),
|
|
593
|
+
"lastAttemptSerial", Volatile.Read(ref _lastAttemptSerial), "presentSuccesses", Volatile.Read(ref _presentSuccesses),
|
|
594
|
+
"lastAttemptedPreparationSerial", Volatile.Read(ref _lastAttemptedPreparationSerial),
|
|
595
|
+
"lastSuccessAgeMs", Volatile.Read(ref _presentSuccesses) == 0 ? (object)null : unchecked((uint)(now - _lastPresentTick)),
|
|
596
|
+
"lastFailedSerial", Volatile.Read(ref _lastFailedSerial), "lastFailedWant", Volatile.Read(ref _lastFailedWant),
|
|
597
|
+
"lastWin32Error", Volatile.Read(ref _lastFailedSerial) < 0 ? (object)null : _lastFailedError,
|
|
598
|
+
"rebuiltSerial", Volatile.Read(ref _rebuiltSerial));
|
|
599
|
+
}
|
|
600
|
+
public const int PreparationMs = 1200; // human spec 2026-11-04: shortened from 1500 after review
|
|
524
601
|
static volatile bool _enabled = true;
|
|
525
602
|
static volatile string _label = "AI 正在操控此电脑 · 监听中";
|
|
526
603
|
|
|
@@ -535,6 +612,7 @@ namespace DshComputerUse.Worker
|
|
|
535
612
|
public static bool Acting { get { return _acting; } }
|
|
536
613
|
public static bool Preparing { get { return _preparing; } }
|
|
537
614
|
public static bool PreparationVisible { get { return _preparing && Volatile.Read(ref _presentedPreparation) == Volatile.Read(ref _preparationSerial); } }
|
|
615
|
+
public static bool PreparationComplete { get { return _preparing && Volatile.Read(ref _completedPreparation) == Volatile.Read(ref _preparationSerial); } }
|
|
538
616
|
public static void ActuationBegin() { _acting = true; EnsureStarted(); }
|
|
539
617
|
public static void ActuationEnd() { _acting = false; }
|
|
540
618
|
public static void Prepare(bool on)
|
|
@@ -619,7 +697,7 @@ namespace DshComputerUse.Worker
|
|
|
619
697
|
{
|
|
620
698
|
if (Panic.Exited) return "AI 已停止操控 · 未监听";
|
|
621
699
|
return _askMode ? "AI 正在提问 · 在 DSH 聊天里回答 · 无倒计时(Ctrl+Alt+R 撤销)"
|
|
622
|
-
: (_stoppedMode ? "已暂停 · Ctrl+Alt+R 继续 · Ctrl+Alt+Q 退出" : Panic.HumanActive ? "已让出键鼠 · 等待人操作 · 静默 3 秒后继续" : _preparing ? "即将开始操作,请暂停键鼠 · 1.
|
|
700
|
+
: (_stoppedMode ? "已暂停 · Ctrl+Alt+R 继续 · Ctrl+Alt+Q 退出" : Panic.HumanActive ? "已让出键鼠 · 等待人操作 · 静默 3 秒后继续" : _preparing ? "即将开始操作,请暂停键鼠 · 1.2 秒准备" : label);
|
|
623
701
|
}
|
|
624
702
|
static volatile string _shownTitle = null;
|
|
625
703
|
// What the indicator is ACTUALLY rendering right now, readable from outside via `probe`.
|
|
@@ -633,14 +711,9 @@ namespace DshComputerUse.Worker
|
|
|
633
711
|
static extern bool SetWindowText(IntPtr h, string text);
|
|
634
712
|
public static void Stopped(bool on)
|
|
635
713
|
{
|
|
714
|
+
// Only the window owner sets the title (in Run). A physical brake must
|
|
715
|
+
// never wait for WM_SETTEXT to be serviced by the overlay thread.
|
|
636
716
|
_stoppedMode = on;
|
|
637
|
-
// The overlay's WINDOW TITLE is part of the text channel: renaming it means any
|
|
638
|
-
// window list — the agent's own computer_windows, or a human's Task Manager —
|
|
639
|
-
// states plainly that the session is stopped, with no screenshot required.
|
|
640
|
-
if (_hwnd != IntPtr.Zero)
|
|
641
|
-
{
|
|
642
|
-
try { SetWindowText(_hwnd, on ? "已暂停 · Ctrl+Alt+R 继续 · Ctrl+Alt+Q 退出" : _label); } catch { }
|
|
643
|
-
}
|
|
644
717
|
}
|
|
645
718
|
|
|
646
719
|
/// Mark "the agent just captured the screen": a yellow border flashes around the
|
|
@@ -707,38 +780,90 @@ namespace DshComputerUse.Worker
|
|
|
707
780
|
|
|
708
781
|
static void ThreadProc()
|
|
709
782
|
{
|
|
710
|
-
try { Run(); }
|
|
783
|
+
try { Run(); }
|
|
784
|
+
catch (Exception ex)
|
|
785
|
+
{
|
|
786
|
+
_rendererFault = _rendererStage + ": " + ex.Message;
|
|
787
|
+
Program.AuditRecord("GLOW-RENDER-ERROR", _rendererFault);
|
|
788
|
+
}
|
|
789
|
+
finally
|
|
790
|
+
{
|
|
791
|
+
_rendererStage = "stopped";
|
|
792
|
+
lock (typeof(Glow)) { _thread = null; }
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
static void PumpOverlayMessages(IntPtr hwnd)
|
|
797
|
+
{
|
|
798
|
+
_rendererStage = "message-pump";
|
|
799
|
+
const uint WM_PAINT = 0x000F;
|
|
800
|
+
Native.NativeMsg msg;
|
|
801
|
+
for (int i = 0; i < 32 && PeekMessage(out msg, hwnd, 0, 0, PM_REMOVE); i++)
|
|
802
|
+
{
|
|
803
|
+
// UpdateLayeredWindow provides all pixels. WM_PAINT stays pending
|
|
804
|
+
// until its update region is validated, even with PM_REMOVE.
|
|
805
|
+
if (msg.message == WM_PAINT) ValidateRect(msg.hwnd, IntPtr.Zero);
|
|
806
|
+
else Native.DispatchMessage(ref msg);
|
|
807
|
+
}
|
|
711
808
|
}
|
|
712
809
|
|
|
713
810
|
static void Run()
|
|
714
811
|
{
|
|
812
|
+
while (RenderOverlay()) { } // Rebuild at most once per unpresented preparation.
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
static bool RenderOverlay()
|
|
816
|
+
{
|
|
817
|
+
_rendererFault = null;
|
|
818
|
+
_rendererStage = "display-metrics";
|
|
715
819
|
int vx = GetSystemMetrics(Native.SM_XVIRTUALSCREEN);
|
|
716
820
|
int vy = GetSystemMetrics(Native.SM_YVIRTUALSCREEN);
|
|
717
821
|
int vw = GetSystemMetrics(Native.SM_CXVIRTUALSCREEN);
|
|
718
822
|
int vh = GetSystemMetrics(Native.SM_CYVIRTUALSCREEN);
|
|
719
|
-
if (vw <= 0 || vh <= 0)
|
|
823
|
+
if (vw <= 0 || vh <= 0) throw new InvalidOperationException("empty virtual desktop");
|
|
720
824
|
|
|
825
|
+
_rendererStage = "create-window";
|
|
721
826
|
IntPtr hwnd = CreateWindowEx(
|
|
722
827
|
WS_EX_LAYERED | WS_EX_TRANSPARENT | WS_EX_TOOLWINDOW | WS_EX_NOACTIVATE | WS_EX_TOPMOST,
|
|
723
828
|
"STATIC", _label, WS_POPUP, vx, vy, vw, vh, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
|
|
724
|
-
if (hwnd == IntPtr.Zero)
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
829
|
+
if (hwnd == IntPtr.Zero) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
|
830
|
+
IntPtr screenDc = IntPtr.Zero, memDc = IntPtr.Zero, dib = IntPtr.Zero, oldBmp = IntPtr.Zero;
|
|
831
|
+
try
|
|
832
|
+
{
|
|
833
|
+
TryExcludeOverlayFromCapture(hwnd);
|
|
834
|
+
_hwnd = hwnd;
|
|
835
|
+
_rendererStage = "create-surface";
|
|
836
|
+
screenDc = GetDC(IntPtr.Zero);
|
|
837
|
+
if (screenDc == IntPtr.Zero) throw new InvalidOperationException("GetDC failed");
|
|
838
|
+
memDc = CreateCompatibleDC(screenDc);
|
|
839
|
+
if (memDc == IntPtr.Zero) throw new InvalidOperationException("CreateCompatibleDC failed");
|
|
840
|
+
Native.BITMAPINFO bmi = new Native.BITMAPINFO();
|
|
841
|
+
bmi.bmiHeader.biSize = (uint)Marshal.SizeOf(typeof(Native.BITMAPINFOHEADER));
|
|
842
|
+
bmi.bmiHeader.biWidth = vw;
|
|
843
|
+
bmi.bmiHeader.biHeight = -vh; // top-down
|
|
844
|
+
bmi.bmiHeader.biPlanes = 1;
|
|
845
|
+
bmi.bmiHeader.biBitCount = 32;
|
|
846
|
+
IntPtr bits;
|
|
847
|
+
dib = CreateDIBSection(screenDc, ref bmi, 0, out bits, IntPtr.Zero, 0);
|
|
848
|
+
if (dib == IntPtr.Zero) throw new InvalidOperationException("CreateDIBSection failed");
|
|
849
|
+
oldBmp = SelectObject(memDc, dib);
|
|
850
|
+
if (oldBmp == IntPtr.Zero || oldBmp == new IntPtr(-1)) throw new InvalidOperationException("SelectObject failed");
|
|
851
|
+
_rendererStage = "rendering";
|
|
852
|
+
return RenderFrames(hwnd, screenDc, memDc, bits, vx, vy, vw, vh);
|
|
853
|
+
}
|
|
854
|
+
finally
|
|
855
|
+
{
|
|
856
|
+
_hwnd = IntPtr.Zero; _shownTitle = null;
|
|
857
|
+
if (oldBmp != IntPtr.Zero && oldBmp != new IntPtr(-1)) SelectObject(memDc, oldBmp);
|
|
858
|
+
if (dib != IntPtr.Zero) DeleteObject(dib);
|
|
859
|
+
if (memDc != IntPtr.Zero) DeleteDC(memDc);
|
|
860
|
+
if (screenDc != IntPtr.Zero) ReleaseDC(IntPtr.Zero, screenDc);
|
|
861
|
+
DestroyWindow(hwnd);
|
|
862
|
+
}
|
|
863
|
+
}
|
|
741
864
|
|
|
865
|
+
static bool RenderFrames(IntPtr hwnd, IntPtr screenDc, IntPtr memDc, IntPtr bits, int vx, int vy, int vw, int vh)
|
|
866
|
+
{
|
|
742
867
|
POINTL dst = new POINTL(); dst.x = vx; dst.y = vy;
|
|
743
868
|
POINTL src = new POINTL(); src.x = 0; src.y = 0;
|
|
744
869
|
SIZEL size = new SIZEL(); size.cx = vw; size.cy = vh;
|
|
@@ -746,11 +871,15 @@ namespace DshComputerUse.Worker
|
|
|
746
871
|
// ---- state machine with dwell hysteresis (no strobing) ----
|
|
747
872
|
int curBitmap = 0; // 0 = none, 1 = blue border, 2 = yellow flash
|
|
748
873
|
int paintedFlash = 0;
|
|
874
|
+
int lastPreparationPaint = 0;
|
|
749
875
|
int alpha = 0;
|
|
750
876
|
bool visible = false;
|
|
751
877
|
int committed = 0;
|
|
752
878
|
while (true)
|
|
753
879
|
{
|
|
880
|
+
_lastLoopTick = Environment.TickCount;
|
|
881
|
+
_rendererStage = "rendering";
|
|
882
|
+
Interlocked.Increment(ref _frames);
|
|
754
883
|
// Ownership and deep blue use the same actual action scope. There is no
|
|
755
884
|
// delayed onset or lingering tail during reading, thinking or preparation.
|
|
756
885
|
bool acting = _enabled && Acting;
|
|
@@ -761,12 +890,13 @@ namespace DshComputerUse.Worker
|
|
|
761
890
|
|
|
762
891
|
// ONE derivation (Glow.DeriveWant): the exit flag outranks ask/stop/flash/committed,
|
|
763
892
|
// so no later state change on another thread can paint a light on an ended session.
|
|
893
|
+
int preparationSerial = Volatile.Read(ref _preparationSerial);
|
|
764
894
|
int want = DeriveWant(flashOn, committed);
|
|
765
895
|
_want = want; // publish it: the state the light is showing, as a fact
|
|
766
896
|
|
|
767
897
|
// Keep the overlay's WINDOW TITLE in sync with the state, from the render loop.
|
|
768
|
-
//
|
|
769
|
-
//
|
|
898
|
+
// When the title was only written by Stopped(), a process that INHERITED a stop
|
|
899
|
+
// before this thread created the window (SyncFromFile at startup)
|
|
770
900
|
// showed a red border with the title "AI 正在操控此电脑". The title is the text
|
|
771
901
|
// channel the agent and the human both read, so a lying title is worse than none.
|
|
772
902
|
// It is derived from the same flags (Glow.DeriveTitle) and a hidden window still has
|
|
@@ -785,7 +915,7 @@ namespace DshComputerUse.Worker
|
|
|
785
915
|
alpha = 0;
|
|
786
916
|
if (visible) { ShowWindow(hwnd, SW_HIDE); visible = false; }
|
|
787
917
|
curBitmap = 0; committed = 0; paintedFlash = 0;
|
|
788
|
-
|
|
918
|
+
PumpOverlayMessages(hwnd);
|
|
789
919
|
Thread.Sleep(12);
|
|
790
920
|
continue;
|
|
791
921
|
}
|
|
@@ -802,7 +932,14 @@ namespace DshComputerUse.Worker
|
|
|
802
932
|
// RED frame when the border happened to be at full alpha already — whether the
|
|
803
933
|
// brake was visible was luck. Present when EITHER the bitmap or the alpha changed.
|
|
804
934
|
bool repaint = false;
|
|
805
|
-
|
|
935
|
+
int preparationElapsed = -1;
|
|
936
|
+
if (want == 7 && (curBitmap != 6 || !PreparationVisible || unchecked(Environment.TickCount - lastPreparationPaint) >= 50))
|
|
937
|
+
{
|
|
938
|
+
lastPreparationPaint = Environment.TickCount;
|
|
939
|
+
preparationElapsed = PreparationVisible ? unchecked(lastPreparationPaint - _preparationStart) : 0;
|
|
940
|
+
PaintPreparation(bits, vw, vh, preparationElapsed);
|
|
941
|
+
curBitmap = 6; repaint = true;
|
|
942
|
+
}
|
|
806
943
|
else if (needBitmap == 2 && paintedFlash != _flashStart) { PaintFlash(bits, vw, vh); paintedFlash = _flashStart; curBitmap = 2; alpha = 204; repaint = true; /* snap ON at 80%: a flash is an event, not a fade */ }
|
|
807
944
|
else if (needBitmap == 5 && curBitmap != 5) { PaintHumanYield(bits, vw, vh); curBitmap = 5; repaint = true; }
|
|
808
945
|
else if (needBitmap == 4 && curBitmap != 4) { PaintStop(bits, vw, vh); curBitmap = 4; repaint = true; }
|
|
@@ -834,22 +971,23 @@ namespace DshComputerUse.Worker
|
|
|
834
971
|
else if (want == 5)
|
|
835
972
|
{
|
|
836
973
|
// ASK: red like a stop, but at TWICE the cyan breath rate (1250 ms vs 2500 ms)
|
|
837
|
-
// and
|
|
974
|
+
// and a full swing — so the pulse is unmistakable at a glance. Human spec
|
|
975
|
+
// 2026-11-04, raised the same day after review: the first floor (25) still
|
|
976
|
+
// read as too dim at the trough; the whole band moved up to 50..140.
|
|
838
977
|
double askPhase = (Environment.TickCount % 1250) / 1250.0;
|
|
839
|
-
target =
|
|
978
|
+
target = 95 + (int)(Math.Sin(askPhase * 2 * Math.PI) * 45);
|
|
840
979
|
breathFrame = true;
|
|
841
980
|
}
|
|
842
981
|
else
|
|
843
982
|
{
|
|
844
983
|
// Faint blue "thinking", breathing so a still screen still shows life.
|
|
845
984
|
// 2.5 s cycle = 20% faster than the original 3.0 s.
|
|
846
|
-
//
|
|
847
|
-
//
|
|
848
|
-
//
|
|
849
|
-
//
|
|
850
|
-
// point: the ring INHALES and goes dark, then comes back.
|
|
985
|
+
// Human spec 2026-11-04, raised the same day after review: the first floor
|
|
986
|
+
// (25) still read as "going dark" at the trough; the whole band moved up to
|
|
987
|
+
// 50..140 — the halo stays clearly visible at its dimmest while the breathing
|
|
988
|
+
// contrast (swing 90) is preserved.
|
|
851
989
|
double phase = (Environment.TickCount % 2500) / 2500.0;
|
|
852
|
-
target =
|
|
990
|
+
target = 95 + (int)(Math.Sin(phase * 2 * Math.PI) * 45);
|
|
853
991
|
breathFrame = true;
|
|
854
992
|
}
|
|
855
993
|
|
|
@@ -886,37 +1024,94 @@ namespace DshComputerUse.Worker
|
|
|
886
1024
|
SetWindowPos(hwnd, new IntPtr(-1), vx, vy, vw, vh, 0x0004 | 0x0010 | 0x0040);
|
|
887
1025
|
visible = true;
|
|
888
1026
|
}
|
|
889
|
-
|
|
1027
|
+
_rendererStage = "presenting";
|
|
1028
|
+
Volatile.Write(ref _lastAttemptWant, want);
|
|
1029
|
+
Volatile.Write(ref _lastAttemptSerial, preparationSerial);
|
|
1030
|
+
if (want == 7) Volatile.Write(ref _lastAttemptedPreparationSerial, preparationSerial);
|
|
1031
|
+
Interlocked.Increment(ref _presentAttempts);
|
|
1032
|
+
bool presented = UpdateLayeredWindow(hwnd, screenDc, ref dst, ref size, memDc, ref src, 0, ref bf, ULW_ALPHA);
|
|
1033
|
+
int nativeError = presented ? 0 : Marshal.GetLastWin32Error();
|
|
1034
|
+
if (presented)
|
|
1035
|
+
{
|
|
1036
|
+
_lastPresentTick = Environment.TickCount;
|
|
1037
|
+
Interlocked.Increment(ref _presentSuccesses);
|
|
1038
|
+
}
|
|
1039
|
+
else
|
|
1040
|
+
{
|
|
1041
|
+
bool firstFailure = Volatile.Read(ref _lastFailedSerial) != preparationSerial;
|
|
1042
|
+
_lastFailedError = nativeError;
|
|
1043
|
+
Volatile.Write(ref _lastFailedWant, want);
|
|
1044
|
+
Volatile.Write(ref _lastFailedSerial, preparationSerial);
|
|
1045
|
+
if (firstFailure) Program.AuditRecord("GLOW-PRESENT-FAILED", "serial=" + preparationSerial + "; want=" + want + "; win32=" + nativeError);
|
|
1046
|
+
if (want == 7 && !PreparationVisible && Volatile.Read(ref _rebuiltSerial) != preparationSerial)
|
|
1047
|
+
{
|
|
1048
|
+
Volatile.Write(ref _rebuiltSerial, preparationSerial);
|
|
1049
|
+
return true;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
if (presented && want == 7 && !PreparationVisible)
|
|
890
1053
|
{
|
|
891
1054
|
_preparationStart = Environment.TickCount;
|
|
892
|
-
Volatile.Write(ref _presentedPreparation,
|
|
1055
|
+
Volatile.Write(ref _presentedPreparation, preparationSerial);
|
|
893
1056
|
}
|
|
1057
|
+
if (presented && want == 7 && preparationElapsed >= PreparationMs)
|
|
1058
|
+
Volatile.Write(ref _completedPreparation, preparationSerial);
|
|
894
1059
|
}
|
|
895
1060
|
else if (visible) { ShowWindow(hwnd, SW_HIDE); visible = false; }
|
|
896
1061
|
}
|
|
897
|
-
|
|
898
|
-
while (PeekMessage(out m, hwnd, 0, 0, PM_REMOVE)) { }
|
|
1062
|
+
PumpOverlayMessages(hwnd);
|
|
899
1063
|
Thread.Sleep(16);
|
|
900
1064
|
}
|
|
901
1065
|
}
|
|
902
1066
|
|
|
903
1067
|
// Thin, square screen-edge rails leave the application itself completely clear.
|
|
904
1068
|
// Each pixel belongs to exactly one inset ring: corners meet without gaps or
|
|
905
|
-
// overlapping hot spots,
|
|
1069
|
+
// overlapping hot spots. Human request 2026-11-04, doubled the same day after
|
|
1070
|
+
// review: the bloom must span 1.6 cm of physical screen for every colour, not
|
|
1071
|
+
// the old fixed eight pixels (which read as a hairline on a 2560x1600 panel at
|
|
1072
|
+
// 175%), so the ring count below is derived from the overlay window's own DPI:
|
|
1073
|
+
// ~60 rings at 96 dpi, ~106 at 168.
|
|
1074
|
+
static int EdgeRailThickness()
|
|
1075
|
+
{
|
|
1076
|
+
// Offscreen painting (visual fixtures) has no window; GetDpiForWindow then
|
|
1077
|
+
// fails and the 96-dpi fallback keeps the fixture geometry deterministic.
|
|
1078
|
+
uint dpi = 96;
|
|
1079
|
+
IntPtr window = _hwnd;
|
|
1080
|
+
if (window != IntPtr.Zero)
|
|
1081
|
+
{
|
|
1082
|
+
uint live = Native.GetDpiForWindow(window);
|
|
1083
|
+
if (live > 0) dpi = live;
|
|
1084
|
+
}
|
|
1085
|
+
int rings = (int)Math.Round(1.6 / 2.54 * dpi); // 1.6 cm in this panel's pixels
|
|
1086
|
+
return Math.Max(8, Math.Min(rings, 192));
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
// The fixed eight-ring ramp was {244,216,94,53,30,16,8,3}: two bright core
|
|
1090
|
+
// rings, then a falloff to near zero. Same profile here, stretched over the
|
|
1091
|
+
// DPI-derived width: ring 0-1 keep their core alpha, rings 2..n-1 decay
|
|
1092
|
+
// geometrically from 94 down to ~3 at the innermost ring.
|
|
1093
|
+
static int EdgeRailAlpha(int ring, int rings)
|
|
1094
|
+
{
|
|
1095
|
+
if (ring == 0) return 244;
|
|
1096
|
+
if (ring == 1) return 216;
|
|
1097
|
+
return (int)Math.Round(94 * Math.Pow(3.0 / 94.0, (double)(ring - 2) / (rings - 3)));
|
|
1098
|
+
}
|
|
1099
|
+
|
|
906
1100
|
static void PaintEdgeRail(System.Drawing.Graphics g, Rectangle bounds, Color tint)
|
|
907
1101
|
{
|
|
908
|
-
int
|
|
1102
|
+
int rings = EdgeRailThickness();
|
|
909
1103
|
var smoothing = g.SmoothingMode;
|
|
910
1104
|
g.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.None;
|
|
911
1105
|
try
|
|
912
1106
|
{
|
|
913
|
-
for (int i = 0; i <
|
|
1107
|
+
for (int i = 0; i < rings; i++)
|
|
914
1108
|
{
|
|
1109
|
+
int alpha = EdgeRailAlpha(i, rings);
|
|
915
1110
|
int x = bounds.Left + i, y = bounds.Top + i;
|
|
916
1111
|
int width = bounds.Width - 2 * i, height = bounds.Height - 2 * i;
|
|
917
1112
|
if (width <= 0 || height <= 0) break;
|
|
918
|
-
Color start = Color.FromArgb(alpha
|
|
919
|
-
Color end = Color.FromArgb(alpha
|
|
1113
|
+
Color start = Color.FromArgb(alpha, Math.Min(255, tint.R + 18), Math.Min(255, tint.G + 14), Math.Min(255, tint.B + 8));
|
|
1114
|
+
Color end = Color.FromArgb(alpha, tint.R, tint.G, tint.B);
|
|
920
1115
|
using (var brush = new System.Drawing.Drawing2D.LinearGradientBrush(
|
|
921
1116
|
new RectangleF(bounds.X, bounds.Y, Math.Max(1, bounds.Width), Math.Max(1, bounds.Height)), start, end, 32f))
|
|
922
1117
|
{
|
|
@@ -968,7 +1163,7 @@ namespace DshComputerUse.Worker
|
|
|
968
1163
|
}
|
|
969
1164
|
|
|
970
1165
|
public static int HumanYieldAlpha(int tick)
|
|
971
|
-
{ return
|
|
1166
|
+
{ return 90 + (int)Math.Round(35 * (1 - Math.Cos((unchecked((uint)tick) % 2800) * 2 * Math.PI / 2800.0))); }
|
|
972
1167
|
static void PaintHumanYield(IntPtr bits, int w, int h)
|
|
973
1168
|
{
|
|
974
1169
|
WithGraphics(bits, w, h, delegate(System.Drawing.Graphics g) {
|
|
@@ -1252,6 +1447,7 @@ namespace DshComputerUse.Worker
|
|
|
1252
1447
|
_engaged = false;
|
|
1253
1448
|
_stopped = false;
|
|
1254
1449
|
CycleLit = false; // R2: the cycle is dead and must not be revived by anyone
|
|
1450
|
+
_controlsReleased = true;
|
|
1255
1451
|
Disarm(); // ... and its monitors die WITH it, not at the next turn
|
|
1256
1452
|
try { Glow.Kill(); } catch { }
|
|
1257
1453
|
try { Glow.Ask(false); } catch { }
|
|
@@ -1303,7 +1499,7 @@ namespace DshComputerUse.Worker
|
|
|
1303
1499
|
SyncExitedFromFile(); SyncFromFile();
|
|
1304
1500
|
lock (SyncLock)
|
|
1305
1501
|
{
|
|
1306
|
-
if (_engaged || _stopped || PauseRequested) return false;
|
|
1502
|
+
if (_engaged || _stopped || PauseRequested || ExitRequested) return false;
|
|
1307
1503
|
_controlsReleased = true;
|
|
1308
1504
|
CycleLit = false;
|
|
1309
1505
|
Disarm(); Glow.Kill();
|
|
@@ -1481,6 +1677,9 @@ namespace DshComputerUse.Worker
|
|
|
1481
1677
|
static IntPtr _hook = IntPtr.Zero;
|
|
1482
1678
|
static Native.LowLevelKeyboardProc _proc; // keep alive: the GC must not collect it
|
|
1483
1679
|
static IntPtr _mouseHook = IntPtr.Zero;
|
|
1680
|
+
static readonly uint CurrentPid = (uint)Process.GetCurrentProcess().Id;
|
|
1681
|
+
static readonly ManualResetEvent _hooksInitialized = new ManualResetEvent(false);
|
|
1682
|
+
static volatile bool _hooksReady;
|
|
1484
1683
|
static Native.LowLevelMouseProc _mouseProc; // same: the GC must not collect it
|
|
1485
1684
|
static Thread _thread;
|
|
1486
1685
|
|
|
@@ -1750,9 +1949,14 @@ namespace DshComputerUse.Worker
|
|
|
1750
1949
|
_proc = HookCallback;
|
|
1751
1950
|
IntPtr hmod = Native.GetModuleHandle(null);
|
|
1752
1951
|
_hook = Native.SetWindowsHookEx(Native.WH_KEYBOARD_LL, _proc, hmod, 0);
|
|
1952
|
+
int keyboardError = _hook == IntPtr.Zero ? Marshal.GetLastWin32Error() : 0;
|
|
1753
1953
|
// Same thread, same message loop: a second low-level hook for the wheel.
|
|
1754
1954
|
_mouseProc = MouseHookCallback;
|
|
1755
1955
|
_mouseHook = Native.SetWindowsHookEx(Native.WH_MOUSE_LL, _mouseProc, hmod, 0);
|
|
1956
|
+
int mouseError = _mouseHook == IntPtr.Zero ? Marshal.GetLastWin32Error() : 0;
|
|
1957
|
+
_hooksReady = _hook != IntPtr.Zero && _mouseHook != IntPtr.Zero;
|
|
1958
|
+
_hooksInitialized.Set();
|
|
1959
|
+
if (!_hooksReady) Program.AuditRecord("CONTROL-HOOK-ERROR", "keyboard=" + keyboardError + "; mouse=" + mouseError);
|
|
1756
1960
|
Native.NativeMsg msg;
|
|
1757
1961
|
while (Native.GetMessage(out msg, IntPtr.Zero, 0, 0) > 0)
|
|
1758
1962
|
{
|
|
@@ -1760,13 +1964,33 @@ namespace DshComputerUse.Worker
|
|
|
1760
1964
|
Native.DispatchMessage(ref msg);
|
|
1761
1965
|
}
|
|
1762
1966
|
}
|
|
1763
|
-
catch
|
|
1967
|
+
catch (Exception ex) { Program.AuditRecord("CONTROL-HOOK-ERROR", ex.GetType().Name); }
|
|
1968
|
+
finally { _hooksReady = false; _hooksInitialized.Set(); }
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
public static void RequireControlHooks()
|
|
1972
|
+
{
|
|
1973
|
+
if (!_hooksInitialized.WaitOne(1500) || !_hooksReady)
|
|
1974
|
+
{
|
|
1975
|
+
var error = new Exception("CONTROL_HOOK_UNAVAILABLE: physical input monitoring did not start or has stopped; no input was dispatched.");
|
|
1976
|
+
error.Data["code"] = "CONTROL_HOOK_UNAVAILABLE";
|
|
1977
|
+
error.Data["outcome"] = "not-dispatched";
|
|
1978
|
+
throw error;
|
|
1979
|
+
}
|
|
1764
1980
|
}
|
|
1765
1981
|
|
|
1766
1982
|
// Hook callbacks only record ownership and enqueue control work. Serial execution
|
|
1767
1983
|
// preserves Ctrl+Esc / resume / exit ordering without blocking the OS hook on I/O.
|
|
1768
|
-
static int _pendingPauses;
|
|
1984
|
+
static int _pendingPauses, _pendingExits;
|
|
1769
1985
|
public static bool PauseRequested { get { return Volatile.Read(ref _pendingPauses) > 0; } }
|
|
1986
|
+
public static bool ExitRequested { get { return Volatile.Read(ref _pendingExits) > 0; } }
|
|
1987
|
+
public static bool FinishOneShotControls()
|
|
1988
|
+
{
|
|
1989
|
+
if (!PauseRequested && !ExitRequested) return true;
|
|
1990
|
+
// A one-shot process must give the recognised chord a chance to persist
|
|
1991
|
+
// STOP/EXITED before Main returns and its background threads disappear.
|
|
1992
|
+
return SpinWait.SpinUntil(delegate { return !PauseRequested && !ExitRequested; }, 5000);
|
|
1993
|
+
}
|
|
1770
1994
|
static readonly ConcurrentQueue<Action> ControlQueue = new ConcurrentQueue<Action>();
|
|
1771
1995
|
static int _controlWorkQueued;
|
|
1772
1996
|
internal static void QueueControl(Action action)
|
|
@@ -1781,20 +2005,31 @@ namespace DshComputerUse.Worker
|
|
|
1781
2005
|
} while (!ControlQueue.IsEmpty && Interlocked.CompareExchange(ref _controlWorkQueued, 1, 0) == 0);
|
|
1782
2006
|
});
|
|
1783
2007
|
}
|
|
1784
|
-
static void QueueDesktopControl(Action action, Action
|
|
2008
|
+
static void QueueDesktopControl(Action action, Action finished = null)
|
|
1785
2009
|
{
|
|
1786
2010
|
QueueControl(delegate {
|
|
1787
2011
|
// Serialize with graceful release: a queued physical hotkey may not
|
|
1788
2012
|
// reopen a completed cycle or steer an exit into its continuing chat.
|
|
1789
|
-
|
|
2013
|
+
try
|
|
1790
2014
|
{
|
|
1791
|
-
|
|
1792
|
-
{
|
|
1793
|
-
|
|
1794
|
-
|
|
2015
|
+
lock (SyncLock)
|
|
2016
|
+
{
|
|
2017
|
+
if (!ControlListenersEnabled || !Program.TryClaimDesktop("physical control"))
|
|
2018
|
+
{ Program.AuditRecord("CONTROL-SKIPPED", ControlListenersEnabled ? "desktop owned by another worker" : "cycle released"); return; }
|
|
2019
|
+
try { action(); }
|
|
2020
|
+
finally { if (!CycleLit) Program.ReleaseDesktopOwnership(); }
|
|
2021
|
+
}
|
|
1795
2022
|
}
|
|
2023
|
+
catch (Exception ex) { Program.AuditRecord("CONTROL-ERROR", ex.GetType().Name); }
|
|
2024
|
+
finally { if (finished != null) finished(); }
|
|
1796
2025
|
});
|
|
1797
2026
|
}
|
|
2027
|
+
|
|
2028
|
+
static void QueueExit(Action exit)
|
|
2029
|
+
{
|
|
2030
|
+
Interlocked.Increment(ref _pendingExits);
|
|
2031
|
+
QueueDesktopControl(exit, delegate { Interlocked.Decrement(ref _pendingExits); });
|
|
2032
|
+
}
|
|
1798
2033
|
public static string ControlHotkey(uint vk, bool down, bool human, bool ctrl, bool alt, bool shift, bool win, bool live)
|
|
1799
2034
|
{
|
|
1800
2035
|
if (!down || !human) return "";
|
|
@@ -1824,10 +2059,10 @@ namespace DshComputerUse.Worker
|
|
|
1824
2059
|
if (physical)
|
|
1825
2060
|
{
|
|
1826
2061
|
int key = 0;
|
|
1827
|
-
bool down = false, up = false, activity = true;
|
|
2062
|
+
bool down = false, up = false, activity = true, deliberate = false;
|
|
1828
2063
|
switch (message)
|
|
1829
2064
|
{
|
|
1830
|
-
case 0x200: activity = movement; break;
|
|
2065
|
+
case 0x200: activity = movement; deliberate = movement; break;
|
|
1831
2066
|
case 0x201: key = 1; down = true; break;
|
|
1832
2067
|
case 0x202: key = 1; up = true; break;
|
|
1833
2068
|
case 0x204: key = 2; down = true; break;
|
|
@@ -1839,7 +2074,7 @@ namespace DshComputerUse.Worker
|
|
|
1839
2074
|
case 0x20A: case 0x20E: activity = unchecked((short)(m.mouseData >> 16)) != 0; break;
|
|
1840
2075
|
default: activity = false; break;
|
|
1841
2076
|
}
|
|
1842
|
-
if (activity) RecordHumanInput(key, down, up);
|
|
2077
|
+
if (activity) RecordHumanInput(key, down, up, deliberate);
|
|
1843
2078
|
}
|
|
1844
2079
|
}
|
|
1845
2080
|
}
|
|
@@ -1871,20 +2106,18 @@ namespace DshComputerUse.Worker
|
|
|
1871
2106
|
if (hotkey == "pause")
|
|
1872
2107
|
{
|
|
1873
2108
|
_suppressPauseEsc = true;
|
|
1874
|
-
Interlocked.Increment(ref _pendingPauses);
|
|
1875
2109
|
// The hook marks ownership immediately; do not perform SendInput or
|
|
1876
2110
|
// filesystem work on this OS hook thread.
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
}, delegate { Interlocked.Decrement(ref _pendingPauses); });
|
|
2111
|
+
var evidence = ChordEvidence(k, ctrl, alt);
|
|
2112
|
+
Interlocked.Increment(ref _pendingPauses);
|
|
2113
|
+
QueueDesktopControl(delegate { Engage("Ctrl+Esc chord detected", false, "hook", evidence()); },
|
|
2114
|
+
delegate { Interlocked.Decrement(ref _pendingPauses); });
|
|
1882
2115
|
return new IntPtr(1); // Ctrl+Esc must not also open the Windows Start menu.
|
|
1883
2116
|
}
|
|
1884
2117
|
if (hotkey == "exit")
|
|
1885
2118
|
{
|
|
1886
|
-
|
|
1887
|
-
|
|
2119
|
+
var evidence = ChordEvidence(k, ctrl, alt);
|
|
2120
|
+
QueueExit(delegate { Exit("Ctrl+Alt+Q chord detected", "hook", evidence()); });
|
|
1888
2121
|
}
|
|
1889
2122
|
else if (hotkey == "resume")
|
|
1890
2123
|
QueueDesktopControl(delegate { Clear(); });
|
|
@@ -1940,7 +2173,7 @@ namespace DshComputerUse.Worker
|
|
|
1940
2173
|
/// The chord as it was SEEN: the fields someone can re-check, including whose Ctrl/Alt the
|
|
1941
2174
|
/// hook recorded and which window was in front. Deliberately verbose — this line is the
|
|
1942
2175
|
/// difference between "the human pressed it" and "we cannot know who pressed it".
|
|
1943
|
-
static string ChordEvidence(Native.KBDLLHOOKSTRUCT k, bool ctrl, bool alt)
|
|
2176
|
+
static Func<string> ChordEvidence(Native.KBDLLHOOKSTRUCT k, bool ctrl, bool alt)
|
|
1944
2177
|
{
|
|
1945
2178
|
StringBuilder sb = new StringBuilder();
|
|
1946
2179
|
sb.Append("vk=0x").Append(k.vkCode.ToString("X2"));
|
|
@@ -1954,31 +2187,37 @@ namespace DshComputerUse.Worker
|
|
|
1954
2187
|
// later whether "the human pressed it" was ever measurable — and note that OURSELVES
|
|
1955
2188
|
// (extra=OUR_MAGIC) is a different, stronger answer than "some injector".
|
|
1956
2189
|
sb.Append(" injected=").Append((k.flags & Native.LLKHF_INJECTED) != 0 ? "1" : "0");
|
|
1957
|
-
|
|
2190
|
+
IntPtr fg = Native.GetForegroundWindow();
|
|
2191
|
+
uint fgPid;
|
|
2192
|
+
Native.GetWindowThreadProcessId(fg, out fgPid);
|
|
2193
|
+
sb.Append(" fgPid=").Append(fgPid);
|
|
2194
|
+
// Capture the foreground identity on the hook thread. Enrich it on the
|
|
2195
|
+
// control queue without sending a message to our own overlay window.
|
|
2196
|
+
return delegate
|
|
1958
2197
|
{
|
|
1959
|
-
|
|
1960
|
-
uint fgPid;
|
|
1961
|
-
Native.GetWindowThreadProcessId(fg, out fgPid);
|
|
1962
|
-
sb.Append(" fgPid=").Append(fgPid);
|
|
1963
|
-
try { sb.Append(" fg=").Append(Process.GetProcessById((int)fgPid).ProcessName); } catch { }
|
|
1964
|
-
StringBuilder t = new StringBuilder(256);
|
|
1965
|
-
if (Native.GetWindowText(fg, t, 256) > 0)
|
|
2198
|
+
try
|
|
1966
2199
|
{
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
2200
|
+
using (var proc = Process.GetProcessById((int)fgPid)) sb.Append(" fg=").Append(proc.ProcessName);
|
|
2201
|
+
uint currentPid;
|
|
2202
|
+
Native.GetWindowThreadProcessId(fg, out currentPid);
|
|
2203
|
+
StringBuilder t = new StringBuilder(256);
|
|
2204
|
+
if (currentPid == fgPid && fgPid != CurrentPid && Native.GetWindowText(fg, t, 256) > 0)
|
|
2205
|
+
{
|
|
2206
|
+
string title = t.ToString();
|
|
2207
|
+
if (title.Length > 60) title = title.Substring(0, 60);
|
|
2208
|
+
sb.Append(" fgTitle=\"").Append(title.Replace("\t", " ")).Append('"');
|
|
2209
|
+
}
|
|
1970
2210
|
}
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
2211
|
+
catch { }
|
|
2212
|
+
return sb.ToString();
|
|
2213
|
+
};
|
|
1974
2214
|
}
|
|
1975
2215
|
|
|
1976
2216
|
/// Polling fallback for the human's combos.
|
|
1977
2217
|
///
|
|
1978
2218
|
/// The low-level hook is the primary path, but it only reports what Windows hands it: a
|
|
1979
|
-
/// missed keydown would
|
|
1980
|
-
///
|
|
1981
|
-
/// missed when polled every 40 ms — so Q always exits and R always releases, hook or no hook.
|
|
2219
|
+
/// missed keydown would otherwise have no fallback. Poll the press bits as an additional
|
|
2220
|
+
/// witness for Q/R while the hook has confirmed human-held modifiers.
|
|
1982
2221
|
static void PollThread()
|
|
1983
2222
|
{
|
|
1984
2223
|
bool prevQ = false, prevR = false;
|
|
@@ -2020,11 +2259,9 @@ namespace DshComputerUse.Worker
|
|
|
2020
2259
|
{
|
|
2021
2260
|
Native.KBDLLHOOKSTRUCT kk = new Native.KBDLLHOOKSTRUCT();
|
|
2022
2261
|
kk.vkCode = Native.VK_Q;
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
ChordEvidence(kk, asyncCtrl, asyncAlt);
|
|
2027
|
-
QueueDesktopControl(delegate { Exit("Ctrl+Alt+Q chord detected", "poll", evidence); });
|
|
2262
|
+
var evidence = ChordEvidence(kk, asyncCtrl, asyncAlt);
|
|
2263
|
+
QueueExit(delegate { Exit("Ctrl+Alt+Q chord detected", "poll",
|
|
2264
|
+
"q-latch + hook-tracked modifiers + measured async key state; " + evidence()); });
|
|
2028
2265
|
}
|
|
2029
2266
|
}
|
|
2030
2267
|
prevQ = q; prevR = r;
|
|
@@ -2062,6 +2299,36 @@ namespace DshComputerUse.Worker
|
|
|
2062
2299
|
return state;
|
|
2063
2300
|
}
|
|
2064
2301
|
static void ResetMonitorRun() { /* Manual recovery must not erase physical input history. */ }
|
|
2302
|
+
// ---------- secure-desktop (UAC / lock screen) dispatch gate, human request 2026-11-04 ----------
|
|
2303
|
+
static readonly SecureDesktopWatch SecureWatch = new SecureDesktopWatch();
|
|
2304
|
+
public static bool SecureDesktopActive { get { return SecureWatch.Active; } }
|
|
2305
|
+
static bool InputDesktopReachable()
|
|
2306
|
+
{
|
|
2307
|
+
IntPtr desktop = Native.OpenInputDesktop(0, false, 0x0001); // DESKTOP_READOBJECT
|
|
2308
|
+
if (desktop != IntPtr.Zero) { Native.CloseDesktop(desktop); return true; }
|
|
2309
|
+
return false;
|
|
2310
|
+
}
|
|
2311
|
+
/// Polled from the monitor thread. Audits and notifies TRANSITIONS only.
|
|
2312
|
+
static void PollSecureDesktop()
|
|
2313
|
+
{
|
|
2314
|
+
if (!SecureWatch.Observe(InputDesktopReachable())) return;
|
|
2315
|
+
bool active = SecureWatch.Active;
|
|
2316
|
+
Program.AuditRecord(active ? "SECURE-DESKTOP-ENGAGED" : "SECURE-DESKTOP-CLEARED",
|
|
2317
|
+
active
|
|
2318
|
+
? "input desktop unreachable (UAC prompt or lock screen); input dispatch paused until it returns"
|
|
2319
|
+
: "input desktop reachable again; dispatch may continue");
|
|
2320
|
+
Notify("secure", active ? "active" : "cleared");
|
|
2321
|
+
}
|
|
2322
|
+
/// Input ops refuse to dispatch while the secure desktop stands: no injection can reach
|
|
2323
|
+
/// it anyway, and a clean refusal plus a notified human beats a blind timeout mid-cycle.
|
|
2324
|
+
public static void RequireNormalDesktop()
|
|
2325
|
+
{
|
|
2326
|
+
if (!SecureWatch.Active) return;
|
|
2327
|
+
var error = new Exception("SECURE_DESKTOP_ACTIVE: the Windows secure desktop (a UAC prompt or the lock screen) is up; no input was dispatched. Ask the human to answer the prompt; dispatch resumes automatically once the normal desktop returns.");
|
|
2328
|
+
error.Data["code"] = "SECURE_DESKTOP_ACTIVE";
|
|
2329
|
+
error.Data["outcome"] = "not-dispatched";
|
|
2330
|
+
throw error;
|
|
2331
|
+
}
|
|
2065
2332
|
public static void Arm() { _armed = true; Handoff.Enable(CycleLit && !_exited, HumanClock.ElapsedMilliseconds); }
|
|
2066
2333
|
public static void Disarm() { _armed = false; Handoff.Enable(false, HumanClock.ElapsedMilliseconds); PublishHandoff(); }
|
|
2067
2334
|
public static void AgentActed() { _agentActAt = Environment.TickCount; }
|
|
@@ -2082,13 +2349,14 @@ namespace DshComputerUse.Worker
|
|
|
2082
2349
|
vk == 0xA0 || vk == 0xA1 || vk == 0xA2 || vk == 0xA3 ||
|
|
2083
2350
|
vk == 0xA4 || vk == 0xA5 || vk == 0x5B || vk == 0x5C;
|
|
2084
2351
|
}
|
|
2085
|
-
public static void RecordHumanInput(int key, bool down, bool up)
|
|
2352
|
+
public static void RecordHumanInput(int key, bool down, bool up) { RecordHumanInput(key, down, up, false); }
|
|
2353
|
+
public static void RecordHumanInput(int key, bool down, bool up, bool deliberate)
|
|
2086
2354
|
{
|
|
2087
2355
|
Interlocked.Increment(ref _physicalInputEpoch);
|
|
2088
2356
|
long now = HumanClock.ElapsedMilliseconds;
|
|
2089
2357
|
bool was = Handoff.Active;
|
|
2090
2358
|
Handoff.Enable(_armed && CycleLit && !_exited, now);
|
|
2091
|
-
Handoff.Record(key, down, up, now);
|
|
2359
|
+
Handoff.Record(key, down, up, now, deliberate);
|
|
2092
2360
|
if (!was && Handoff.Active && Interlocked.CompareExchange(ref _handoffWorkQueued, 1, 0) == 0)
|
|
2093
2361
|
ThreadPool.QueueUserWorkItem(delegate {
|
|
2094
2362
|
try { Program.ReleaseOwnedInputs(); PublishHandoff(); }
|
|
@@ -2115,6 +2383,7 @@ namespace DshComputerUse.Worker
|
|
|
2115
2383
|
{
|
|
2116
2384
|
if (!ControlListenersEnabled) return;
|
|
2117
2385
|
SyncExitedFromFile();
|
|
2386
|
+
PollSecureDesktop();
|
|
2118
2387
|
long now = HumanClock.ElapsedMilliseconds;
|
|
2119
2388
|
Handoff.Enable(_armed && CycleLit && !_exited, now);
|
|
2120
2389
|
Handoff.Tick(now);
|
|
@@ -2444,7 +2713,7 @@ namespace DshComputerUse.Worker
|
|
|
2444
2713
|
static readonly HashSet<string> ReadOnlyOps = new HashSet<string> {
|
|
2445
2714
|
"selftest", "cursor", "panic", "resume", "recover", "echoargs", "ping", "screens",
|
|
2446
2715
|
"windows", "uia", "uiaFromPoint", "uiaFocused", "waitForIdle", "clipRead",
|
|
2447
|
-
"capture", "annotate", "waitStable", "waitChange", "focus", "frameSig", "probe",
|
|
2716
|
+
"capture", "ocr", "annotate", "waitStable", "waitChange", "focus", "frameSig", "probe",
|
|
2448
2717
|
// diagnostics that must be able to ANSWER while a brake is up: they send no input and
|
|
2449
2718
|
// light nothing, which is exactly when someone needs to inspect the release decision
|
|
2450
2719
|
"keyClass", "stuckPlan",
|
|
@@ -2532,6 +2801,7 @@ namespace DshComputerUse.Worker
|
|
|
2532
2801
|
try { Glow.Stopped(false); } catch { }
|
|
2533
2802
|
// Re-arming re-baselines the human monitor: the travel that tripped the brake must not
|
|
2534
2803
|
// still be sitting in the accumulator, or the very next poll would brake again.
|
|
2804
|
+
if (CycleLit && !_exited) Glow.Thinking(0);
|
|
2535
2805
|
_agentActAt = 0;
|
|
2536
2806
|
}
|
|
2537
2807
|
if (was) Notify("resume", "");
|
|
@@ -2721,12 +2991,16 @@ namespace DshComputerUse.Worker
|
|
|
2721
2991
|
bool finished;
|
|
2722
2992
|
ClipboardLease(IClipboardAccess access, List<ClipboardBytes> previous, uint written)
|
|
2723
2993
|
{ this.access = access; this.previous = previous; this.written = written; }
|
|
2994
|
+
/// Returns null when the previous clipboard cannot be preserved (private formats, odd
|
|
2995
|
+
/// sizes). The caller then downgrades to a plain overwrite-paste with a warning rather
|
|
2996
|
+
/// than refusing — see Snapshot's incident note (2026-11-04).
|
|
2724
2997
|
public static ClipboardLease Begin(IClipboardAccess access, string text)
|
|
2725
2998
|
{
|
|
2726
2999
|
using (var session = access.Open())
|
|
2727
3000
|
{
|
|
2728
3001
|
if (session.Sequence == 0) throw new Exception("Clipboard sequence unavailable; clipboard was not changed.");
|
|
2729
|
-
var previous = session.Snapshot(); //
|
|
3002
|
+
var previous = session.Snapshot(); // null = cannot preserve: caller downgrades, we do not clear anything here
|
|
3003
|
+
if (previous == null) return null;
|
|
2730
3004
|
try
|
|
2731
3005
|
{
|
|
2732
3006
|
session.Replace(new List<ClipboardBytes> {
|
|
@@ -2765,8 +3039,9 @@ namespace DshComputerUse.Worker
|
|
|
2765
3039
|
}
|
|
2766
3040
|
}
|
|
2767
3041
|
|
|
2768
|
-
// Copy only explicitly supported HGLOBAL formats. Unknown/private or GDI formats
|
|
2769
|
-
//
|
|
3042
|
+
// Copy only explicitly supported HGLOBAL formats. Unknown/private or GDI formats cannot be
|
|
3043
|
+
// snapshotted — Snapshot returns null for them and the caller downgrades to
|
|
3044
|
+
// paste-without-restore with a warning (incident 2026-11-04) instead of losing the paste.
|
|
2770
3045
|
public sealed class NativeClipboardAccess : IClipboardAccess, IDisposable
|
|
2771
3046
|
{
|
|
2772
3047
|
readonly NativeWindow window = new NativeWindow();
|
|
@@ -2790,14 +3065,6 @@ namespace DshComputerUse.Worker
|
|
|
2790
3065
|
[DllImport("kernel32.dll")] static extern UIntPtr GlobalSize(IntPtr memory);
|
|
2791
3066
|
[DllImport("kernel32.dll")] static extern IntPtr GlobalFree(IntPtr memory);
|
|
2792
3067
|
[DllImport("kernel32.dll")] static extern void SetLastError(uint error);
|
|
2793
|
-
static Exception Unavailable(string reason)
|
|
2794
|
-
{
|
|
2795
|
-
var error = new Exception("CLIPBOARD_PRESERVATION_UNAVAILABLE: " + reason +
|
|
2796
|
-
"; clipboard unchanged. Use mode:unicode if suitable for the target editor.");
|
|
2797
|
-
error.Data["code"] = "CLIPBOARD_PRESERVATION_UNAVAILABLE";
|
|
2798
|
-
error.Data["outcome"] = "not-dispatched";
|
|
2799
|
-
return error;
|
|
2800
|
-
}
|
|
2801
3068
|
public static bool Supported(uint format, string name)
|
|
2802
3069
|
{
|
|
2803
3070
|
if (format == 1 || format == 7 || format == 8 || format == 13 || format == 15 || format == 16 || format == 17) return true;
|
|
@@ -2819,6 +3086,13 @@ namespace DshComputerUse.Worker
|
|
|
2819
3086
|
}
|
|
2820
3087
|
public uint Sequence { get { return GetClipboardSequenceNumber(); } }
|
|
2821
3088
|
public void Dispose() { if (opened) { CloseClipboard(); opened = false; } }
|
|
3089
|
+
/// Returns null when the current clipboard cannot be fully snapshotted (private or GDI
|
|
3090
|
+
/// formats, too many/oversized entries, lock failures). NULL MEANS "cannot preserve",
|
|
3091
|
+
/// not "refuse to work" (human incident 2026-11-04): a WeChat-session clipboard carried
|
|
3092
|
+
/// private format 49161, paste was refused, the JS auto-fallback re-typed 37 CJK chars
|
|
3093
|
+
/// as unicode keystrokes and a garbled message actually went out. Losing an opaque
|
|
3094
|
+
/// clipboard item is a recoverable local cost; a mis-typed external message is not —
|
|
3095
|
+
/// so the caller downgrades to paste-without-restore instead of refusing.
|
|
2822
3096
|
public List<ClipboardBytes> Snapshot()
|
|
2823
3097
|
{
|
|
2824
3098
|
var formats = new List<uint>();
|
|
@@ -2829,27 +3103,26 @@ namespace DshComputerUse.Worker
|
|
|
2829
3103
|
format = EnumClipboardFormats(format);
|
|
2830
3104
|
if (format == 0)
|
|
2831
3105
|
{
|
|
2832
|
-
if (Marshal.GetLastWin32Error() != 0)
|
|
3106
|
+
if (Marshal.GetLastWin32Error() != 0) return null;
|
|
2833
3107
|
break;
|
|
2834
3108
|
}
|
|
2835
3109
|
var name = new StringBuilder(256);
|
|
2836
3110
|
if (format >= 0xC000) GetClipboardFormatName(format, name, name.Capacity);
|
|
2837
|
-
if (!Supported(format, name.ToString()))
|
|
3111
|
+
if (!Supported(format, name.ToString())) return null;
|
|
2838
3112
|
formats.Add(format);
|
|
2839
|
-
if (formats.Count > 64)
|
|
3113
|
+
if (formats.Count > 64) return null;
|
|
2840
3114
|
}
|
|
2841
3115
|
var result = new List<ClipboardBytes>();
|
|
2842
3116
|
long total = 0;
|
|
2843
3117
|
foreach (uint id in formats)
|
|
2844
3118
|
{
|
|
2845
3119
|
IntPtr handle = GetClipboardData(id);
|
|
2846
|
-
if (handle == IntPtr.Zero)
|
|
3120
|
+
if (handle == IntPtr.Zero) return null;
|
|
2847
3121
|
ulong size = GlobalSize(handle).ToUInt64();
|
|
2848
|
-
if (size == 0 || size > 32 * 1024 * 1024 || total + (long)size > 32 * 1024 * 1024)
|
|
2849
|
-
throw Unavailable("clipboard snapshot exceeds supported size");
|
|
3122
|
+
if (size == 0 || size > 32 * 1024 * 1024 || total + (long)size > 32 * 1024 * 1024) return null;
|
|
2850
3123
|
total += (long)size;
|
|
2851
3124
|
IntPtr data = GlobalLock(handle);
|
|
2852
|
-
if (data == IntPtr.Zero)
|
|
3125
|
+
if (data == IntPtr.Zero) return null;
|
|
2853
3126
|
try
|
|
2854
3127
|
{
|
|
2855
3128
|
var bytes = new byte[(int)size]; Marshal.Copy(data, bytes, 0, bytes.Length);
|
|
@@ -2901,7 +3174,7 @@ namespace DshComputerUse.Worker
|
|
|
2901
3174
|
static string _dpiNote = "unset";
|
|
2902
3175
|
static readonly HashSet<string> ObsOps = new HashSet<string>
|
|
2903
3176
|
{
|
|
2904
|
-
"ping", "echoargs", "screens", "cursor", "capture", "annotate",
|
|
3177
|
+
"ping", "echoargs", "screens", "cursor", "capture", "ocr", "annotate",
|
|
2905
3178
|
"windows", "uia", "uiaFromPoint", "uiaFocused", "waitForIdle", "clipRead",
|
|
2906
3179
|
"selftest", "indicator", "waitStable", "waitChange", "focus", "frameSig", "probe"
|
|
2907
3180
|
};
|
|
@@ -2962,6 +3235,8 @@ namespace DshComputerUse.Worker
|
|
|
2962
3235
|
}
|
|
2963
3236
|
public static void CheckHumanInput()
|
|
2964
3237
|
{
|
|
3238
|
+
if (Panic.ExitRequested)
|
|
3239
|
+
throw InputInterrupted("CYCLE-ENDED", "Ctrl+Alt+Q requested exit; no further input will be sent for this operation.");
|
|
2965
3240
|
if (Panic.PauseRequested)
|
|
2966
3241
|
throw InputInterrupted("ABORTED", "Ctrl+Esc requested a manual pause; wait for manual recovery.");
|
|
2967
3242
|
if (_operationLive && _operationCancelGeneration != Interlocked.Read(ref _cancelGeneration))
|
|
@@ -3017,16 +3292,16 @@ namespace DshComputerUse.Worker
|
|
|
3017
3292
|
while (!Glow.PreparationVisible)
|
|
3018
3293
|
{
|
|
3019
3294
|
Panic.Check(op); CheckHumanInput();
|
|
3020
|
-
if (startup.ElapsedMilliseconds >= 5000)
|
|
3021
|
-
throw InputInterrupted("PREPARATION_UNAVAILABLE", "The preparation notice could not be displayed; no input was dispatched.");
|
|
3295
|
+
if (startup.ElapsedMilliseconds >= 5000) throw PreparationUnavailable("first-frame");
|
|
3022
3296
|
Thread.Sleep(10);
|
|
3023
3297
|
}
|
|
3024
3298
|
var timer = Stopwatch.StartNew();
|
|
3025
|
-
while (timer.ElapsedMilliseconds < Glow.PreparationMs)
|
|
3299
|
+
while (timer.ElapsedMilliseconds < Glow.PreparationMs || !Glow.PreparationComplete)
|
|
3026
3300
|
{
|
|
3027
3301
|
Panic.Check(op); CheckHumanInput();
|
|
3302
|
+
if (timer.ElapsedMilliseconds >= 5000) throw PreparationUnavailable("countdown-frame");
|
|
3028
3303
|
int remaining = Glow.PreparationMs - (int)timer.ElapsedMilliseconds;
|
|
3029
|
-
|
|
3304
|
+
Thread.Sleep(remaining > 0 ? Math.Min(25, remaining) : 10);
|
|
3030
3305
|
}
|
|
3031
3306
|
Panic.Check(op); CheckHumanInput();
|
|
3032
3307
|
if (inputEpoch != Panic.InputEpoch || foreground != PreparationForeground())
|
|
@@ -3036,6 +3311,14 @@ namespace DshComputerUse.Worker
|
|
|
3036
3311
|
}
|
|
3037
3312
|
finally { Glow.Prepare(false); }
|
|
3038
3313
|
}
|
|
3314
|
+
static Exception PreparationUnavailable(string phase)
|
|
3315
|
+
{
|
|
3316
|
+
var health = Glow.Health();
|
|
3317
|
+
health["failurePhase"] = phase;
|
|
3318
|
+
var error = InputInterrupted("PREPARATION_UNAVAILABLE", "The preparation notice could not be displayed through completion; no input was dispatched. glowHealth=" + Json.Serialize(health));
|
|
3319
|
+
error.Data["glowHealth"] = health;
|
|
3320
|
+
return error;
|
|
3321
|
+
}
|
|
3039
3322
|
public static bool CancelInput(int requestId)
|
|
3040
3323
|
{
|
|
3041
3324
|
if (requestId <= 0 || requestId <= _lastCompletedRequestId) return false;
|
|
@@ -3271,6 +3554,10 @@ namespace DshComputerUse.Worker
|
|
|
3271
3554
|
else if (kind == "resume")
|
|
3272
3555
|
hint = "Re-armed: a human pressed Ctrl+Alt+R (the worker notifies `resume` from Clear() " +
|
|
3273
3556
|
"alone) — working may continue from the break point.";
|
|
3557
|
+
else if (kind == "secure")
|
|
3558
|
+
hint = why == "active"
|
|
3559
|
+
? "The Windows secure desktop is up (a UAC prompt or the lock screen). Input dispatch is PAUSED — nothing you send will be dispatched until it closes. Tell the human a system dialog needs their answer; dispatch resumes automatically the moment the normal desktop returns, then simply retry the step."
|
|
3560
|
+
: "The normal desktop is back. Input dispatch has resumed automatically — retry the interrupted step and continue the task.";
|
|
3274
3561
|
else
|
|
3275
3562
|
hint = "worker state change \"" + kind + "\": " + why;
|
|
3276
3563
|
WriteLine(Json.Serialize(Dict(
|
|
@@ -3301,6 +3588,11 @@ namespace DshComputerUse.Worker
|
|
|
3301
3588
|
Dictionary<string, object> resp;
|
|
3302
3589
|
try { resp = Handle(oneshot); }
|
|
3303
3590
|
catch (Exception ex) { resp = Err(ex); }
|
|
3591
|
+
if (!Panic.FinishOneShotControls())
|
|
3592
|
+
{
|
|
3593
|
+
Program.AuditRecord("CONTROL-DRAIN-TIMEOUT", "one-shot control finalization exceeded 5s");
|
|
3594
|
+
resp = Err("CONTROL-DRAIN-TIMEOUT: physical brake finalization could not be confirmed; verify STOP/EXITED before continuing.");
|
|
3595
|
+
}
|
|
3304
3596
|
AuditActuation(oneshot, resp);
|
|
3305
3597
|
WriteLine(Json.Serialize(resp));
|
|
3306
3598
|
return 0;
|
|
@@ -3387,6 +3679,11 @@ namespace DshComputerUse.Worker
|
|
|
3387
3679
|
var args = request.TryGetValue("args", out rawArgs) ? rawArgs as Dictionary<string, object> : null;
|
|
3388
3680
|
bool input = ActOps.Contains(op) || op == "shiftClick" || op == "selftest" ||
|
|
3389
3681
|
(op == "focus" && args != null && GetB(args, "refocus", false));
|
|
3682
|
+
bool dryInput = input && args != null && GetB(args, "dryRun", false);
|
|
3683
|
+
// Only the coordinate click probe has a native read-only implementation.
|
|
3684
|
+
// No other input may use the flag to skip the controls and then dispatch.
|
|
3685
|
+
if (dryInput && (op != "click" || !args.ContainsKey("x") || !args.ContainsKey("y")))
|
|
3686
|
+
return Err("UNSUPPORTED_DRY_RUN: native input simulation requires a click with x and y; no input was dispatched.");
|
|
3390
3687
|
EnsureDesktopOwnership(op);
|
|
3391
3688
|
int requestId = GetI(request, "id", 0);
|
|
3392
3689
|
bool ignored;
|
|
@@ -3394,13 +3691,14 @@ namespace DshComputerUse.Worker
|
|
|
3394
3691
|
_operationHumanRevision = Panic.HumanRevision;
|
|
3395
3692
|
_operationCancelGeneration = Interlocked.Read(ref _cancelGeneration);
|
|
3396
3693
|
_operationDispatched = false; _operationLive = input;
|
|
3694
|
+
bool scopedCycle = false;
|
|
3397
3695
|
try
|
|
3398
3696
|
{
|
|
3399
3697
|
if (CancelledRequests.TryRemove(requestId, out ignored))
|
|
3400
3698
|
throw InputInterrupted("INPUT_CANCELLED", "cancelled before dispatch.");
|
|
3401
3699
|
if (input) CheckHumanInput();
|
|
3402
3700
|
string pauseId = Panic.TemporaryPauseId;
|
|
3403
|
-
Dictionary<string, object> result = HandleCore(line);
|
|
3701
|
+
Dictionary<string, object> result = HandleCore(line, input, dryInput, ref scopedCycle);
|
|
3404
3702
|
if (input) CheckHumanInput();
|
|
3405
3703
|
if (!string.IsNullOrEmpty(pauseId) && GetB(result, "ok", false) &&
|
|
3406
3704
|
(op == "windows" || op == "uia" || op == "uiaFromPoint" || op == "uiaFocused" || op == "capture"))
|
|
@@ -3415,13 +3713,14 @@ namespace DshComputerUse.Worker
|
|
|
3415
3713
|
finally { _operationActing = false; Panic.EndActuation(); Glow.Prepare(false); }
|
|
3416
3714
|
}
|
|
3417
3715
|
_operationLive = false; _activeRequestId = 0;
|
|
3716
|
+
if (scopedCycle) Panic.TryEndCycle();
|
|
3418
3717
|
if (_releaseDesktopPending) ReleaseDesktopOwnership();
|
|
3419
3718
|
_lastCompletedRequestId = Math.Max(_lastCompletedRequestId, requestId);
|
|
3420
3719
|
CancelledRequests.TryRemove(requestId, out ignored);
|
|
3421
3720
|
}
|
|
3422
3721
|
}
|
|
3423
3722
|
|
|
3424
|
-
static Dictionary<string, object> HandleCore(string line)
|
|
3723
|
+
static Dictionary<string, object> HandleCore(string line, bool input, bool dryInput, ref bool scopedCycle)
|
|
3425
3724
|
{
|
|
3426
3725
|
Dictionary<string, object> req;
|
|
3427
3726
|
try { req = Json.Deserialize<Dictionary<string, object>>(line); }
|
|
@@ -3454,11 +3753,22 @@ namespace DshComputerUse.Worker
|
|
|
3454
3753
|
// human re-arms it (CheckOp's allowlist keeps the health probe and re-arm alive).
|
|
3455
3754
|
// Emergency stop gates EVERY op that could drive the machine; a DRY RUN drives nothing
|
|
3456
3755
|
// and is therefore treated as the observation it really is.
|
|
3457
|
-
Panic.CheckOp(
|
|
3756
|
+
Panic.CheckOp(dryInput ? "probe" : op);
|
|
3458
3757
|
|
|
3459
|
-
|
|
3460
|
-
|
|
3461
|
-
|
|
3758
|
+
if (input && !dryInput)
|
|
3759
|
+
{
|
|
3760
|
+
// Direct native input (including --op) has no JS cycle signature.
|
|
3761
|
+
// Arm physical controls before the purple cue and release this scope
|
|
3762
|
+
// when the request ends. A human pause or exit retains its own latch.
|
|
3763
|
+
if (!Panic.CycleLit)
|
|
3764
|
+
{
|
|
3765
|
+
scopedCycle = Panic.TryLightCycle(0);
|
|
3766
|
+
if (!scopedCycle) Panic.Check(op);
|
|
3767
|
+
}
|
|
3768
|
+
Panic.RequireControlHooks();
|
|
3769
|
+
Panic.RequireNormalDesktop();
|
|
3770
|
+
PrepareActuation(op, req);
|
|
3771
|
+
}
|
|
3462
3772
|
|
|
3463
3773
|
switch (op)
|
|
3464
3774
|
{
|
|
@@ -3470,7 +3780,7 @@ namespace DshComputerUse.Worker
|
|
|
3470
3780
|
case "beginAsk": return Ok(BeginAskOp(a));
|
|
3471
3781
|
case "endAsk": return Ok(EndAskOp(a));
|
|
3472
3782
|
case "exit": return Ok(ExitOp(a));
|
|
3473
|
-
case "ping": return Ok(Dict("pong", true, "cooperation", Panic.CooperationState()));
|
|
3783
|
+
case "ping": return Ok(Dict("pong", true, "cooperation", Panic.CooperationState(), "glowHealth", Glow.Health()));
|
|
3474
3784
|
// TEST SEAM (D, 2026-09-13). A fixture may not re-implement the rule it measures, so
|
|
3475
3785
|
// this op drives the PRODUCTION transition (Panic.ClassifyAndTrackKey — the same
|
|
3476
3786
|
// function the low-level hook calls) with explicit evidence: ours / injected / vk.
|
|
@@ -3536,6 +3846,7 @@ namespace DshComputerUse.Worker
|
|
|
3536
3846
|
case "screens": return Ok(VirtualScreenDict());
|
|
3537
3847
|
case "cursor": POINT cp; Native.GetCursorPos(out cp); return Ok(PtDict(cp.X, cp.Y));
|
|
3538
3848
|
case "capture": return Ok(Capture(a));
|
|
3849
|
+
case "ocr": return Ok(Ocr(a));
|
|
3539
3850
|
case "annotate": return Ok(Annotate(a));
|
|
3540
3851
|
case "move": FailsafeCheck(a); return Ok(Move(a));
|
|
3541
3852
|
case "click": FailsafeCheck(a); return Ok(Click(a));
|
|
@@ -3557,13 +3868,15 @@ namespace DshComputerUse.Worker
|
|
|
3557
3868
|
case "probe":
|
|
3558
3869
|
{
|
|
3559
3870
|
string what = GetS(a, "what", "point");
|
|
3560
|
-
Dictionary<string, object> pr = what == "focus" ? ProbeFocus()
|
|
3871
|
+
Dictionary<string, object> pr = what == "glow" ? Dict() : what == "focus" ? ProbeFocus()
|
|
3561
3872
|
: (what == "window" ? WinBrief(FindHwnd(a)) : ProbePoint(GetI(a, "x", 0), GetI(a, "y", 0)));
|
|
3873
|
+
pr["glowHealth"] = Glow.Health();
|
|
3562
3874
|
// The state machine as a FACT, in the one op that exists to answer "what is
|
|
3563
3875
|
// actually true right now". Read-only, so asking never changes the answer.
|
|
3564
3876
|
pr["cycle"] = Dict(
|
|
3565
3877
|
"cycleLit", Panic.CycleLit,
|
|
3566
3878
|
"armed", Panic.Armed,
|
|
3879
|
+
"secureDesktop", Panic.SecureDesktopActive,
|
|
3567
3880
|
"engaged", Panic.Engaged,
|
|
3568
3881
|
"stopped", Panic.Stopped,
|
|
3569
3882
|
"exited", Panic.Exited,
|
|
@@ -3645,6 +3958,90 @@ namespace DshComputerUse.Worker
|
|
|
3645
3958
|
|
|
3646
3959
|
static long NowMs() { return (long)(DateTime.UtcNow - Epoch).TotalMilliseconds; }
|
|
3647
3960
|
|
|
3961
|
+
// ---------- OCR: text WITH bounding boxes, straight from the worker ----------
|
|
3962
|
+
// Zero-UIA apps (WeChat incident 2026-11-04) burned one screenshot per decision because
|
|
3963
|
+
// finding "傳送" or a contact name meant reading pixels with model vision. Windows' inbox
|
|
3964
|
+
// OCR (Windows.Media.Ocr — ships with the OS, zero external dependencies) returns text
|
|
3965
|
+
// with per-word rectangles, so the model can locate controls BY NAME and spend vision
|
|
3966
|
+
// only where looks actually matter. Coordinates are ABSOLUTE screen pixels (region origin
|
|
3967
|
+
// added), ready for computer_click without further math.
|
|
3968
|
+
static OcrEngine _ocrEngine;
|
|
3969
|
+
static string _ocrLanguage;
|
|
3970
|
+
static Dictionary<string, object> Ocr(Dictionary<string, object> a)
|
|
3971
|
+
{
|
|
3972
|
+
ValidateKickoffCapture(a);
|
|
3973
|
+
if (_ocrEngine == null)
|
|
3974
|
+
{
|
|
3975
|
+
_ocrEngine = OcrEngine.TryCreateFromUserProfileLanguages();
|
|
3976
|
+
if (_ocrEngine != null) _ocrLanguage = _ocrEngine.RecognizerLanguage.LanguageTag;
|
|
3977
|
+
}
|
|
3978
|
+
if (_ocrEngine == null)
|
|
3979
|
+
throw new Exception("OCR_UNAVAILABLE: no OCR recognizer language is installed on this machine. " +
|
|
3980
|
+
"Install one (Settings > Time & language > Language > add a language with 'Text recognition') " +
|
|
3981
|
+
"or read regions with computer_shot instead.");
|
|
3982
|
+
int vx = Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN);
|
|
3983
|
+
int vy = Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN);
|
|
3984
|
+
int vw = Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN);
|
|
3985
|
+
int vh = Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN);
|
|
3986
|
+
int x = GetI(a, "x", vx), y = GetI(a, "y", vy);
|
|
3987
|
+
int w = GetI(a, "width", vw), h = GetI(a, "height", vh);
|
|
3988
|
+
if (GetB(a, "clamp", true))
|
|
3989
|
+
{
|
|
3990
|
+
int x2 = Math.Max(x, vx), y2 = Math.Max(y, vy);
|
|
3991
|
+
int x3 = Math.Min(x + w, vx + vw), y3 = Math.Min(y + h, vy + vh);
|
|
3992
|
+
if (x3 <= x2 || y3 <= y2) throw new Exception("region outside virtual desktop");
|
|
3993
|
+
x = x2; y = y2; w = x3 - x2; h = y3 - y2;
|
|
3994
|
+
}
|
|
3995
|
+
// OCR cost scales with pixels; one decision never needs more than 16M of them.
|
|
3996
|
+
if ((long)w * h > 16L * 1024 * 1024)
|
|
3997
|
+
throw new Exception("OCR region exceeds 16M pixels (" + w + "x" + h + "); crop tighter.");
|
|
3998
|
+
byte[] bgra = new byte[w * h * 4];
|
|
3999
|
+
IntPtr screen = IntPtr.Zero, mem = IntPtr.Zero, dib = IntPtr.Zero, oldBmp = IntPtr.Zero;
|
|
4000
|
+
try
|
|
4001
|
+
{
|
|
4002
|
+
screen = Native.GetDC(IntPtr.Zero);
|
|
4003
|
+
mem = Native.CreateCompatibleDC(screen);
|
|
4004
|
+
Native.BITMAPINFO bmi = new Native.BITMAPINFO();
|
|
4005
|
+
bmi.bmiHeader.biSize = (uint)Marshal.SizeOf(typeof(Native.BITMAPINFOHEADER));
|
|
4006
|
+
bmi.bmiHeader.biWidth = w; bmi.bmiHeader.biHeight = -h; // top-down
|
|
4007
|
+
bmi.bmiHeader.biPlanes = 1; bmi.bmiHeader.biBitCount = 32;
|
|
4008
|
+
bmi.bmiHeader.biCompression = 0;
|
|
4009
|
+
IntPtr bits;
|
|
4010
|
+
dib = Native.CreateDIBSection(screen, ref bmi, 0, out bits, IntPtr.Zero, 0);
|
|
4011
|
+
if (dib == IntPtr.Zero) throw new Exception("OCR: CreateDIBSection failed");
|
|
4012
|
+
oldBmp = Native.SelectObject(mem, dib);
|
|
4013
|
+
Native.BitBlt(mem, 0, 0, w, h, screen, x, y, 0x00CC0020 | 0x40000000);
|
|
4014
|
+
Marshal.Copy(bits, bgra, 0, bgra.Length);
|
|
4015
|
+
}
|
|
4016
|
+
finally
|
|
4017
|
+
{
|
|
4018
|
+
if (oldBmp != IntPtr.Zero && oldBmp != new IntPtr(-1)) Native.SelectObject(mem, oldBmp);
|
|
4019
|
+
if (dib != IntPtr.Zero) Native.DeleteObject(dib);
|
|
4020
|
+
if (mem != IntPtr.Zero) Native.DeleteDC(mem);
|
|
4021
|
+
if (screen != IntPtr.Zero) Native.ReleaseDC(IntPtr.Zero, screen);
|
|
4022
|
+
}
|
|
4023
|
+
SoftwareBitmap bitmap = SoftwareBitmap.CreateCopyFromBuffer(
|
|
4024
|
+
WindowsRuntimeBufferExtensions.AsBuffer(bgra), BitmapPixelFormat.Bgra8, w, h);
|
|
4025
|
+
OcrResult result = _ocrEngine.RecognizeAsync(bitmap).AsTask().GetAwaiter().GetResult();
|
|
4026
|
+
List<object> lines = new List<object>();
|
|
4027
|
+
StringBuilder joined = new StringBuilder();
|
|
4028
|
+
foreach (OcrLine line in result.Lines)
|
|
4029
|
+
{
|
|
4030
|
+
List<object> words = new List<object>();
|
|
4031
|
+
foreach (OcrWord word in line.Words)
|
|
4032
|
+
{
|
|
4033
|
+
words.Add(Dict("text", word.Text,
|
|
4034
|
+
"x", x + (int)word.BoundingRect.X, "y", y + (int)word.BoundingRect.Y,
|
|
4035
|
+
"w", (int)word.BoundingRect.Width, "h", (int)word.BoundingRect.Height));
|
|
4036
|
+
}
|
|
4037
|
+
lines.Add(Dict("text", line.Text, "words", words));
|
|
4038
|
+
if (joined.Length > 0) joined.Append('\n');
|
|
4039
|
+
joined.Append(line.Text);
|
|
4040
|
+
}
|
|
4041
|
+
return Dict("lines", lines, "text", joined.ToString(), "language", _ocrLanguage,
|
|
4042
|
+
"region", new int[] { x, y, w, h });
|
|
4043
|
+
}
|
|
4044
|
+
|
|
3648
4045
|
static Dictionary<string, object> Capture(Dictionary<string, object> a)
|
|
3649
4046
|
{
|
|
3650
4047
|
ValidateKickoffCapture(a); // before even the pixel-stability sampler
|
|
@@ -3948,7 +4345,8 @@ namespace DshComputerUse.Worker
|
|
|
3948
4345
|
string btn = GetS(a, "button", "left");
|
|
3949
4346
|
int clicks = Math.Max(1, Math.Min(3, GetI(a, "clicks", 1)));
|
|
3950
4347
|
int gap = GetI(a, "intervalMs", 60);
|
|
3951
|
-
|
|
4348
|
+
bool dryRun = GetB(a, "dryRun", false);
|
|
4349
|
+
if (!dryRun) AgentActing(a, "click");
|
|
3952
4350
|
CheckHumanInput();
|
|
3953
4351
|
// If the caller NAMED the window it wants to click in, put that window in front
|
|
3954
4352
|
// first: a click on a covered pixel is delivered to whatever is on top of it (the
|
|
@@ -3959,7 +4357,6 @@ namespace DshComputerUse.Worker
|
|
|
3959
4357
|
// would change the thing it measures. That bug hid a real occlusion for a whole test
|
|
3960
4358
|
// round (2026-09-13): every dry run reported "ok" because it had just activated the
|
|
3961
4359
|
// window it was asked about.
|
|
3962
|
-
bool dryRun = GetB(a, "dryRun", false);
|
|
3963
4360
|
// A caller-supplied window is re-asserted (raised, ALT tap, titlebar click) on the
|
|
3964
4361
|
// next line, so it has to be checked BEFORE that: the pixel check further down comes
|
|
3965
4362
|
// far too late to protect a window the worker was told to bring forward.
|
|
@@ -4122,8 +4519,10 @@ namespace DshComputerUse.Worker
|
|
|
4122
4519
|
{
|
|
4123
4520
|
clickOut["changedPct"] = changedPct;
|
|
4124
4521
|
if (changedPct < 0.15 && !a.ContainsKey("verify"))
|
|
4125
|
-
clickOut["warning"] = "nothing changed near the click point within ~240ms
|
|
4126
|
-
"
|
|
4522
|
+
clickOut["warning"] = "nothing changed near the click point within ~240ms. This is EXPECTED when the " +
|
|
4523
|
+
"click lands on an already-focused editor or a control already in the desired state (a caret blink " +
|
|
4524
|
+
"alone sits below the threshold — test report 2026-11-04); it can also mean a dead/disabled control " +
|
|
4525
|
+
"or a wrong pixel. Read focus/receiver above and only retry if they disagree with the intent.";
|
|
4127
4526
|
}
|
|
4128
4527
|
return clickOut;
|
|
4129
4528
|
}
|
|
@@ -4534,11 +4933,16 @@ namespace DshComputerUse.Worker
|
|
|
4534
4933
|
|
|
4535
4934
|
// Explicit window scope excludes unrelated desktop animation without loosening the
|
|
4536
4935
|
// change threshold. A moved/minimized/replaced window invalidates the observation.
|
|
4537
|
-
static Func<byte[]> WaitSampler(Dictionary<string, object> a, int step, out object scope
|
|
4936
|
+
static Func<byte[]> WaitSampler(Dictionary<string, object> a, int step, out object scope,
|
|
4937
|
+
out int rx, out int ry, out int rw, out int rh)
|
|
4538
4938
|
{
|
|
4539
4939
|
if (!a.ContainsKey("hwnd") || GetI(a, "hwnd", 0) == 0)
|
|
4540
4940
|
{
|
|
4541
|
-
|
|
4941
|
+
rx = Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN);
|
|
4942
|
+
ry = Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN);
|
|
4943
|
+
rw = Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN);
|
|
4944
|
+
rh = Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN);
|
|
4945
|
+
scope = Dict("kind", "virtual-desktop", "region", Dict("x", rx, "y", ry, "width", rw, "height", rh));
|
|
4542
4946
|
return delegate { return FrameSignature(step); };
|
|
4543
4947
|
}
|
|
4544
4948
|
IntPtr h = new IntPtr(GetI(a, "hwnd", 0));
|
|
@@ -4550,7 +4954,8 @@ namespace DshComputerUse.Worker
|
|
|
4550
4954
|
Rectangle visible = Rectangle.Intersect(virtualRect, Rectangle.FromLTRB(original.Left, original.Top, original.Right, original.Bottom));
|
|
4551
4955
|
if (visible.Width <= 16 || visible.Height <= 16)
|
|
4552
4956
|
throw new Exception("WAIT_TARGET_UNAVAILABLE: window has no usable visible screen region");
|
|
4553
|
-
|
|
4957
|
+
rx = visible.X; ry = visible.Y; rw = visible.Width; rh = visible.Height;
|
|
4958
|
+
scope = Dict("kind", "window", "hwnd", h.ToInt64(), "region", Dict("x", rx, "y", ry, "width", rw, "height", rh));
|
|
4554
4959
|
return delegate {
|
|
4555
4960
|
RECT current;
|
|
4556
4961
|
if (!Native.IsWindow(h) || Native.IsIconic(h) || PidOf(h) != pid || !Native.GetWindowRect(h, out current) ||
|
|
@@ -4560,6 +4965,29 @@ namespace DshComputerUse.Worker
|
|
|
4560
4965
|
};
|
|
4561
4966
|
}
|
|
4562
4967
|
|
|
4968
|
+
/// Bounding box (absolute screen px) of every signature cell that differs beyond the
|
|
4969
|
+
/// threshold — "where did it change", not just "did it" (test report 2026-11-04: a
|
|
4970
|
+
/// 0.344% diff cannot distinguish a sent message from a blinking caret without this).
|
|
4971
|
+
static int[] SigDiffBounds(byte[] a, byte[] b, int threshold, int cols, int step, int rx, int ry)
|
|
4972
|
+
{
|
|
4973
|
+
if (a == null || b == null || a.Length != b.Length || cols <= 0) return null;
|
|
4974
|
+
int minX = -1, minY = -1, maxX = -1, maxY = -1;
|
|
4975
|
+
for (int i = 0; i < a.Length; i++)
|
|
4976
|
+
{
|
|
4977
|
+
int d = a[i] - b[i];
|
|
4978
|
+
if (d < 0) d = -d;
|
|
4979
|
+
if (d <= threshold) continue;
|
|
4980
|
+
int c = i % cols, r = i / cols;
|
|
4981
|
+
if (minX < 0 || c < minX) minX = c;
|
|
4982
|
+
if (c > maxX) maxX = c;
|
|
4983
|
+
if (minY < 0 || r < minY) minY = r;
|
|
4984
|
+
if (r > maxY) maxY = r;
|
|
4985
|
+
}
|
|
4986
|
+
if (minX < 0) return null;
|
|
4987
|
+
return new int[] { rx + minX * step, ry + minY * step,
|
|
4988
|
+
(maxX - minX + 1) * step, (maxY - minY + 1) * step };
|
|
4989
|
+
}
|
|
4990
|
+
|
|
4563
4991
|
static Dictionary<string, object> WaitStable(Dictionary<string, object> a)
|
|
4564
4992
|
{
|
|
4565
4993
|
int capMs = Math.Max(400, GetI(a, "timeoutMs", 8000));
|
|
@@ -4568,8 +4996,34 @@ namespace DshComputerUse.Worker
|
|
|
4568
4996
|
int step = Math.Max(4, Math.Min(64, GetI(a, "step", 8)));
|
|
4569
4997
|
double diffPct = GetD(a, "diffPct", 0.30);
|
|
4570
4998
|
int waited, estimate; double lastDiff;
|
|
4571
|
-
|
|
4999
|
+
int rx, ry, rw, rh;
|
|
5000
|
+
object scope; Func<byte[]> sample = WaitSampler(a, step, out scope, out rx, out ry, out rw, out rh);
|
|
4572
5001
|
bool ok = SettleSampleWait(capMs, stableMs, pollMs, diffPct, sample, out waited, out lastDiff, out estimate);
|
|
5002
|
+
if (!ok)
|
|
5003
|
+
{
|
|
5004
|
+
// CARET / BLINK DETECTION (test report 2026-11-04): a blinking caret is a ~1-2 Hz
|
|
5005
|
+
// periodic sub-percent change that never settles — Notepad burned the full 8 s cap
|
|
5006
|
+
// on lastDiffPct 0.388%. Probe one extra window (~2.4 s covers several blink
|
|
5007
|
+
// cycles): if EVERY consecutive diff is tiny but nonzero, the only motion is the
|
|
5008
|
+
// caret — call it stable and say exactly why.
|
|
5009
|
+
int probes = Math.Max(6, 2400 / pollMs);
|
|
5010
|
+
byte[] prev = sample();
|
|
5011
|
+
double max = 0; bool zero = false, large = false;
|
|
5012
|
+
for (int i = 0; i < probes; i++)
|
|
5013
|
+
{
|
|
5014
|
+
Thread.Sleep(pollMs);
|
|
5015
|
+
byte[] cur = sample();
|
|
5016
|
+
double d = SigDiffPct(prev, cur, 8);
|
|
5017
|
+
prev = cur;
|
|
5018
|
+
if (d == 0) zero = true;
|
|
5019
|
+
if (d > 1.0) large = true;
|
|
5020
|
+
if (d > max) max = d;
|
|
5021
|
+
}
|
|
5022
|
+
if (!large && !zero && max < 1.0)
|
|
5023
|
+
return Dict("stable", true, "waitedMs", waited, "lastDiffPct", Math.Round(max, 3),
|
|
5024
|
+
"estimateMs", estimate, "scope", scope, "caretSteady", true,
|
|
5025
|
+
"note", "the only sampled motion is a small periodic change consistent with a blinking caret (~1-2 Hz); treated as stable. lastDiffPct here is the LARGEST probe difference. This still does not verify task completion.");
|
|
5026
|
+
}
|
|
4573
5027
|
return Dict("stable", ok, "waitedMs", waited, "lastDiffPct", Math.Round(lastDiff, 3),
|
|
4574
5028
|
"estimateMs", estimate, "scope", scope,
|
|
4575
5029
|
"note", ok ? "sampled pixels settled; this does not verify task completion"
|
|
@@ -4582,10 +5036,12 @@ namespace DshComputerUse.Worker
|
|
|
4582
5036
|
int pollMs = Math.Max(100, GetI(a, "pollMs", 220));
|
|
4583
5037
|
int step = Math.Max(4, Math.Min(64, GetI(a, "step", 8)));
|
|
4584
5038
|
double diffPct = GetD(a, "diffPct", 0.30);
|
|
4585
|
-
|
|
5039
|
+
int rx, ry, rw, rh;
|
|
5040
|
+
object scope; Func<byte[]> sample = WaitSampler(a, step, out scope, out rx, out ry, out rw, out rh);
|
|
4586
5041
|
byte[] first = sample();
|
|
4587
5042
|
long t0 = NowMs();
|
|
4588
5043
|
int samples = 0;
|
|
5044
|
+
int cols = (rw - 1) / step + 1;
|
|
4589
5045
|
while (NowMs() - t0 < timeoutMs)
|
|
4590
5046
|
{
|
|
4591
5047
|
Thread.Sleep(pollMs);
|
|
@@ -4593,7 +5049,15 @@ namespace DshComputerUse.Worker
|
|
|
4593
5049
|
samples++;
|
|
4594
5050
|
double d = SigDiffPct(first, cur, 8);
|
|
4595
5051
|
if (d > diffPct)
|
|
4596
|
-
|
|
5052
|
+
{
|
|
5053
|
+
// WHERE it changed (test report 2026-11-04): 0.344% cannot tell a sent
|
|
5054
|
+
// message from a caret blink; the bounding box can — input area vs caret.
|
|
5055
|
+
int[] bounds = SigDiffBounds(first, cur, 8, cols, step, rx, ry);
|
|
5056
|
+
Dictionary<string, object> hit = Dict("changed", true, "afterMs", NowMs() - t0,
|
|
5057
|
+
"diffPct", Math.Round(d, 3), "samples", samples, "scope", scope);
|
|
5058
|
+
if (bounds != null) hit["changeBounds"] = Dict("x", bounds[0], "y", bounds[1], "width", bounds[2], "height", bounds[3]);
|
|
5059
|
+
return hit;
|
|
5060
|
+
}
|
|
4597
5061
|
}
|
|
4598
5062
|
return Dict("changed", false, "afterMs", NowMs() - t0, "samples", samples, "scope", scope,
|
|
4599
5063
|
"note", "no sampled pixel change exceeded the threshold; inspect task state before retrying a write");
|
|
@@ -4892,6 +5356,24 @@ namespace DshComputerUse.Worker
|
|
|
4892
5356
|
{
|
|
4893
5357
|
CheckHumanInput();
|
|
4894
5358
|
ClipboardLease lease = ClipboardLease.Begin(access, text);
|
|
5359
|
+
if (lease == null)
|
|
5360
|
+
{
|
|
5361
|
+
// UNPRESERVABLE CLIPBOARD, 2026-11-04: the previous clipboard holds
|
|
5362
|
+
// private/oversized content we cannot snapshot. Overwrite and paste
|
|
5363
|
+
// anyway — a lost opaque clipboard item is recoverable, while the old
|
|
5364
|
+
// refusal fed CJK text to the unicode fallback and an IME-garbled
|
|
5365
|
+
// message actually went out (WeChat, format 49161).
|
|
5366
|
+
MarkMutation();
|
|
5367
|
+
Clipboard.SetDataObject(text, true);
|
|
5368
|
+
Thread.Sleep(60);
|
|
5369
|
+
Key(Dict("combo", "ctrl+v", "holdMs", 40, "indicatorMs", 800));
|
|
5370
|
+
Dictionary<string, object> downgraded = Dict("mode", "paste", "chars", text.Length, "focus", focus,
|
|
5371
|
+
"clipboardRestore", "skipped-unpreservable-clipboard");
|
|
5372
|
+
downgraded["clipboardWarning"] = "The previous clipboard could not be preserved (private or oversized formats) and was overwritten by this paste. Restore it manually if it mattered.";
|
|
5373
|
+
if (typeWarn != null) downgraded["warning"] = typeWarn;
|
|
5374
|
+
if (_lastRefocus != null) downgraded["refocused"] = _lastRefocus;
|
|
5375
|
+
return downgraded;
|
|
5376
|
+
}
|
|
4895
5377
|
Exception inputError = null;
|
|
4896
5378
|
try
|
|
4897
5379
|
{
|
|
@@ -4995,12 +5477,18 @@ namespace DshComputerUse.Worker
|
|
|
4995
5477
|
|
|
4996
5478
|
static Dictionary<string, object> Windows(Dictionary<string, object> a)
|
|
4997
5479
|
{
|
|
5480
|
+
uint self = (uint)System.Diagnostics.Process.GetCurrentProcess().Id;
|
|
4998
5481
|
List<object> list = new List<object>();
|
|
4999
5482
|
Native.EnumWindows(delegate(IntPtr h, IntPtr l)
|
|
5000
5483
|
{
|
|
5001
5484
|
if (!Native.IsWindowVisible(h)) return true;
|
|
5002
5485
|
StringBuilder sb = new StringBuilder(512);
|
|
5003
5486
|
if (Native.GetWindowText(h, sb, 512) == 0) return true;
|
|
5487
|
+
// Hide this worker's own click-through indicator overlay (test report 2026-11-04):
|
|
5488
|
+
// it is a full-screen topmost layer, never a target, and listing it only invites
|
|
5489
|
+
// the model to "activate" the thing that must never take focus.
|
|
5490
|
+
uint pid; Native.GetWindowThreadProcessId(h, out pid);
|
|
5491
|
+
if (pid == self) return true;
|
|
5004
5492
|
list.Add(WinDict(h));
|
|
5005
5493
|
return true;
|
|
5006
5494
|
}, IntPtr.Zero);
|
|
@@ -5061,15 +5549,16 @@ namespace DshComputerUse.Worker
|
|
|
5061
5549
|
// The whole ladder now lives in RefocusWindow (shared with the keyboard focus
|
|
5062
5550
|
// guard, which must re-assert the same window before typing).
|
|
5063
5551
|
string how = RefocusWindow(h, GetB(a, "clickFallback", true));
|
|
5064
|
-
|
|
5552
|
+
uint confirmedPid = 0;
|
|
5553
|
+
if (how == null || !Native.IsWindow(h) || !IsForeground(h) || (confirmedPid = PidOf(h)) == 0) how = "failed";
|
|
5065
5554
|
else
|
|
5066
5555
|
{
|
|
5067
5556
|
SetSticky(h);
|
|
5068
|
-
_kickoffFocus.Confirm(h.ToInt64(),
|
|
5557
|
+
_kickoffFocus.Confirm(h.ToInt64(), confirmedPid);
|
|
5069
5558
|
}
|
|
5070
5559
|
CheckHumanInput();
|
|
5071
5560
|
return Dict("activated", how != "failed", "activatedVia", how,
|
|
5072
|
-
"hwnd", h.ToInt64(),
|
|
5561
|
+
"hwnd", h.ToInt64(), "pid", confirmedPid == 0 ? null : (object)confirmedPid,
|
|
5073
5562
|
"hint", how == "failed"
|
|
5074
5563
|
? "foreground lock refused every method; click the window (or its taskbar button) and retry"
|
|
5075
5564
|
: null);
|
|
@@ -5108,13 +5597,21 @@ namespace DshComputerUse.Worker
|
|
|
5108
5597
|
IntPtr fg = TopLevel(Native.GetForegroundWindow());
|
|
5109
5598
|
string problem = _kickoffFocus.CaptureProblem(Panic.CycleLit, Panic.Stopped, Panic.Exited,
|
|
5110
5599
|
target != IntPtr.Zero && Native.IsWindow(target), fg.ToInt64(), PidOf(fg), IsHostWindow(fg));
|
|
5111
|
-
if (problem != null)
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
"
|
|
5115
|
-
|
|
5116
|
-
|
|
5117
|
-
|
|
5600
|
+
if (problem != null)
|
|
5601
|
+
{
|
|
5602
|
+
// NAME THE ACTUAL FOREGROUND (test report 2026-11-04): "identify the intended
|
|
5603
|
+
// window" without saying who holds the foreground forced an extra computer_state
|
|
5604
|
+
// round trip just to learn the one fact this refusal already measured.
|
|
5605
|
+
StringBuilder fgTitle = new StringBuilder(256);
|
|
5606
|
+
if (fg != IntPtr.Zero) Native.GetWindowText(fg, fgTitle, fgTitle.Capacity);
|
|
5607
|
+
throw new Exception(
|
|
5608
|
+
"KICKOFF-FOCUS-REQUIRED: " + problem + ". No screenshot was returned. " +
|
|
5609
|
+
"Foreground now: hwnd=" + fg.ToInt64() + ", pid=" + PidOf(fg) +
|
|
5610
|
+
", title=\"" + fgTitle.ToString() + "\". " +
|
|
5611
|
+
"If the reported foreground HWND and PID match the target already verified for this task, reactivate it. " +
|
|
5612
|
+
"Otherwise identify the intended window with computer_state {windows:true}. " +
|
|
5613
|
+
"Use computer_window {hwnd:...,op:'activate'} and require activated:true before retrying the screenshot.");
|
|
5614
|
+
}
|
|
5118
5615
|
}
|
|
5119
5616
|
|
|
5120
5617
|
static IntPtr TopLevel(IntPtr h)
|
|
@@ -6529,6 +7026,7 @@ namespace DshComputerUse.Worker
|
|
|
6529
7026
|
static Dictionary<string, object> Err(Exception error)
|
|
6530
7027
|
{
|
|
6531
7028
|
var result = Err(error.Message);
|
|
7029
|
+
if (error.Data["glowHealth"] != null) result["glowHealth"] = error.Data["glowHealth"];
|
|
6532
7030
|
if (error.Data["code"] != null)
|
|
6533
7031
|
{
|
|
6534
7032
|
result["code"] = (string)error.Data["code"];
|