deepseek-computer-use 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +317 -2
- package/cordis.patch.yml +28 -0
- package/lib/bridge.js +224 -0
- package/lib/build-helper.mjs +106 -0
- package/lib/build-native.mjs +52 -0
- package/lib/build.mjs +22 -0
- package/lib/config.js +95 -0
- package/lib/index.js +159 -0
- package/lib/safety.js +247 -0
- package/lib/tools.js +534 -0
- package/native/Driver.cs +1857 -0
- package/native/Json.cs +274 -0
- package/native/Overlay.cs +725 -0
- package/native/Program.cs +160 -0
- package/native/audio/Audio.csproj +17 -0
- package/native/audio/Program.cs +95 -0
- package/native/wgcfw/Capture.csproj +23 -0
- package/native/wgcfw/Program.cs +249 -0
- package/package.json +66 -4
- package/skills/computer-use/SKILL.md +90 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
|
|
2
|
+
using System;
|
|
3
|
+
using System.Collections.Generic;
|
|
4
|
+
using System.Globalization;
|
|
5
|
+
using System.IO;
|
|
6
|
+
using System.Text;
|
|
7
|
+
using System.Threading;
|
|
8
|
+
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
// stdio JSON-lines bridge: one request object per line in, one response object
|
|
11
|
+
// per line out. The DSH plugin owns the process lifetime; this helper owns the
|
|
12
|
+
// UI Automation / SendInput / capture state so element indexes stay valid.
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
public static class Program
|
|
15
|
+
{
|
|
16
|
+
static IPlatformDriver _driver;
|
|
17
|
+
/// Set by the close RPC: the loop stops after flushing the response.
|
|
18
|
+
public static bool CloseRequested;
|
|
19
|
+
/// 130 mirrors Codex's "the user pressed Esc" exit code.
|
|
20
|
+
public static int RequestedExitCode;
|
|
21
|
+
|
|
22
|
+
public static int Main(string[] args)
|
|
23
|
+
{
|
|
24
|
+
try
|
|
25
|
+
{
|
|
26
|
+
Console.OutputEncoding = new UTF8Encoding(false);
|
|
27
|
+
Console.InputEncoding = new UTF8Encoding(false);
|
|
28
|
+
}
|
|
29
|
+
catch { }
|
|
30
|
+
|
|
31
|
+
_driver = CreateDriver();
|
|
32
|
+
|
|
33
|
+
if (args.Length > 0 && args[0] == "--self-test")
|
|
34
|
+
{
|
|
35
|
+
return SelfTest();
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
string line;
|
|
39
|
+
while (true)
|
|
40
|
+
{
|
|
41
|
+
try { line = Console.ReadLine(); }
|
|
42
|
+
catch (Exception) { break; }
|
|
43
|
+
if (line == null) break;
|
|
44
|
+
if (line.Trim().Length == 0) continue;
|
|
45
|
+
string response = Handle(line);
|
|
46
|
+
Console.Out.Write(response);
|
|
47
|
+
Console.Out.Write('\n');
|
|
48
|
+
Console.Out.Flush();
|
|
49
|
+
if (CloseRequested || RequestedExitCode != 0) break;
|
|
50
|
+
}
|
|
51
|
+
try { if (_driver != null) _driver.Dispose(); } catch { }
|
|
52
|
+
return RequestedExitCode;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
static IPlatformDriver CreateDriver()
|
|
56
|
+
{
|
|
57
|
+
string forced = Environment.GetEnvironmentVariable("DSH_CU_PLATFORM");
|
|
58
|
+
if (!string.IsNullOrEmpty(forced) && forced != "windows")
|
|
59
|
+
throw new NotSupportedException("platform '" + forced + "' has no driver implementation yet; only windows is available");
|
|
60
|
+
return new WindowsDriver();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
static int SelfTest()
|
|
64
|
+
{
|
|
65
|
+
Dictionary<string, object> ping = _driver.Ping(new Dictionary<string, object>());
|
|
66
|
+
Dictionary<string, object> wins = _driver.ListWindows(new Dictionary<string, object>());
|
|
67
|
+
Dictionary<string, object> report = new Dictionary<string, object>();
|
|
68
|
+
report["ok"] = true;
|
|
69
|
+
report["platform"] = _driver.PlatformName;
|
|
70
|
+
report["windowCount"] = wins["count"];
|
|
71
|
+
report["driver"] = ping;
|
|
72
|
+
Console.Out.Write(Json.ToJson(report));
|
|
73
|
+
Console.Out.Write('\n');
|
|
74
|
+
Console.Out.Flush();
|
|
75
|
+
return 0;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
static string Handle(string line)
|
|
79
|
+
{
|
|
80
|
+
long id = 0;
|
|
81
|
+
try
|
|
82
|
+
{
|
|
83
|
+
Dictionary<string, object> req = Json.AsObject(Json.Parse(line));
|
|
84
|
+
if (req == null) throw new ToolError("bad_request", "request must be a JSON object");
|
|
85
|
+
id = Json.GetLong(req, "id", 0);
|
|
86
|
+
string method = Json.GetString(req, "method");
|
|
87
|
+
Dictionary<string, object> p = Json.AsObject(Json.Get(req, "params"));
|
|
88
|
+
if (p == null) p = new Dictionary<string, object>();
|
|
89
|
+
// Codex ships the budget with every call (x-oai-cua-request-budget-ms) and the
|
|
90
|
+
// helper honours it; here it bounds capture waits and marks overruns.
|
|
91
|
+
Budget.Begin(Json.GetLong(p, "__budgetMs", 0));
|
|
92
|
+
try { if (_driver != null) _driver.Maintenance(); } catch { }
|
|
93
|
+
object result = Dispatch(method, p);
|
|
94
|
+
return Ok(id, result);
|
|
95
|
+
}
|
|
96
|
+
catch (ToolError te)
|
|
97
|
+
{
|
|
98
|
+
return Fail(id, te.Code, te.Message);
|
|
99
|
+
}
|
|
100
|
+
catch (Exception ex)
|
|
101
|
+
{
|
|
102
|
+
return Fail(id, "internal_error", ex.GetType().Name + ": " + ex.Message);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
static object Dispatch(string method, Dictionary<string, object> p)
|
|
107
|
+
{
|
|
108
|
+
if (string.IsNullOrEmpty(method)) throw new ToolError("bad_request", "request.method is required");
|
|
109
|
+
switch (method)
|
|
110
|
+
{
|
|
111
|
+
case "ping": return _driver.Ping(p);
|
|
112
|
+
case "hello": return _driver.Hello(p);
|
|
113
|
+
case "restore_cursor": return _driver.RestoreCursor(p);
|
|
114
|
+
case "close": return _driver.Close(p);
|
|
115
|
+
case "list_windows": return _driver.ListWindows(p);
|
|
116
|
+
case "list_apps": return _driver.ListApps(p);
|
|
117
|
+
case "get_window": return _driver.GetWindow(p);
|
|
118
|
+
case "launch_app": return _driver.LaunchApp(p);
|
|
119
|
+
case "get_window_state": return _driver.GetWindowState(p);
|
|
120
|
+
case "click": return _driver.Click(p);
|
|
121
|
+
case "press_key": return _driver.PressKey(p);
|
|
122
|
+
case "type_text": return _driver.TypeText(p);
|
|
123
|
+
case "scroll": return _driver.Scroll(p);
|
|
124
|
+
case "drag": return _driver.Drag(p);
|
|
125
|
+
case "set_value": return _driver.SetValue(p);
|
|
126
|
+
case "perform_secondary_action": return _driver.PerformSecondaryAction(p);
|
|
127
|
+
case "activate_window": return _driver.ActivateWindow(p);
|
|
128
|
+
case "overlay_show": return _driver.OverlayShow(p);
|
|
129
|
+
case "overlay_hide": return _driver.OverlayHide(p);
|
|
130
|
+
case "configure": return _driver.Configure(p);
|
|
131
|
+
case "approve_app": return _driver.ApproveApp(p);
|
|
132
|
+
case "end_turn": return _driver.EndTurn(p);
|
|
133
|
+
case "start_audio_recording": return _driver.StartAudioRecording(p);
|
|
134
|
+
case "stop_audio_recording": return _driver.StopAudioRecording(p);
|
|
135
|
+
case "shutdown": return new Dictionary<string, object> { { "bye", true } };
|
|
136
|
+
default: throw new ToolError("unknown_method", "unknown method '" + method + "'");
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
static string Ok(long id, object result)
|
|
141
|
+
{
|
|
142
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
143
|
+
d["id"] = id;
|
|
144
|
+
d["ok"] = true;
|
|
145
|
+
d["result"] = result;
|
|
146
|
+
return Json.ToJson(d);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
static string Fail(long id, string code, string message)
|
|
150
|
+
{
|
|
151
|
+
Dictionary<string, object> err = new Dictionary<string, object>();
|
|
152
|
+
err["code"] = code;
|
|
153
|
+
err["message"] = message;
|
|
154
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
155
|
+
d["id"] = id;
|
|
156
|
+
d["ok"] = false;
|
|
157
|
+
d["error"] = err;
|
|
158
|
+
return Json.ToJson(d);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
<Project Sdk="Microsoft.NET.Sdk">
|
|
2
|
+
<PropertyGroup>
|
|
3
|
+
<OutputType>Exe</OutputType>
|
|
4
|
+
<TargetFramework>net48</TargetFramework>
|
|
5
|
+
<LangVersion>latest</LangVersion>
|
|
6
|
+
<PlatformTarget>x64</PlatformTarget>
|
|
7
|
+
<AssemblyName>dsh-cu-audio</AssemblyName>
|
|
8
|
+
<RootNamespace>DshCuAudio</RootNamespace>
|
|
9
|
+
</PropertyGroup>
|
|
10
|
+
<ItemGroup>
|
|
11
|
+
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.3" PrivateAssets="all" />
|
|
12
|
+
<PackageReference Include="NAudio" Version="2.2.1" />
|
|
13
|
+
</ItemGroup>
|
|
14
|
+
<ItemGroup>
|
|
15
|
+
<Reference Include="System.Windows.Forms" />
|
|
16
|
+
</ItemGroup>
|
|
17
|
+
</Project>
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
using System;
|
|
2
|
+
using System.IO;
|
|
3
|
+
using System.Threading;
|
|
4
|
+
using NAudio.Wave;
|
|
5
|
+
using NAudio.Wave.SampleProviders;
|
|
6
|
+
|
|
7
|
+
namespace DshCuAudio
|
|
8
|
+
{
|
|
9
|
+
/// WASAPI loopback recorder: captures whatever the machine is playing, which is
|
|
10
|
+
/// what Codex's start/stop_audio_recording expose (gated behind SKY_ENABLE_AUDIO).
|
|
11
|
+
/// Stops on max duration, on the helper's stop file, or on Ctrl+C.
|
|
12
|
+
internal static class Program
|
|
13
|
+
{
|
|
14
|
+
private static int Main(string[] args)
|
|
15
|
+
{
|
|
16
|
+
string output = Path.Combine(Path.GetTempPath(), "dsh-cu-audio.wav");
|
|
17
|
+
int maxMs = 60000;
|
|
18
|
+
string stopFile = null;
|
|
19
|
+
for (int i = 0; i < args.Length - 1; i++)
|
|
20
|
+
{
|
|
21
|
+
if (args[i] == "--out") output = args[i + 1];
|
|
22
|
+
else if (args[i] == "--max-ms") int.TryParse(args[i + 1], out maxMs);
|
|
23
|
+
else if (args[i] == "--stop-file") stopFile = args[i + 1];
|
|
24
|
+
}
|
|
25
|
+
if (maxMs < 500) maxMs = 500;
|
|
26
|
+
|
|
27
|
+
try
|
|
28
|
+
{
|
|
29
|
+
// Record at the device mix format first, then normalise to the 24 kHz
|
|
30
|
+
// stereo 16-bit WAV that Codex returns.
|
|
31
|
+
string devicePath = output + ".device.wav";
|
|
32
|
+
try { if (File.Exists(devicePath)) File.Delete(devicePath); } catch { }
|
|
33
|
+
using (var capture = new WasapiLoopbackCapture())
|
|
34
|
+
using (var writer = new WaveFileWriter(devicePath, capture.WaveFormat))
|
|
35
|
+
{
|
|
36
|
+
var stopped = new ManualResetEventSlim(false);
|
|
37
|
+
Exception failure = null;
|
|
38
|
+
capture.DataAvailable += (sender, e) =>
|
|
39
|
+
{
|
|
40
|
+
try
|
|
41
|
+
{
|
|
42
|
+
if (e.BytesRecorded > 0) writer.Write(e.Buffer, 0, e.BytesRecorded);
|
|
43
|
+
}
|
|
44
|
+
catch (Exception ex) { failure = ex; stopped.Set(); }
|
|
45
|
+
};
|
|
46
|
+
capture.RecordingStopped += (sender, e) => { stopped.Set(); };
|
|
47
|
+
|
|
48
|
+
capture.StartRecording();
|
|
49
|
+
Console.WriteLine("recording started: " + capture.WaveFormat.SampleRate + "Hz " +
|
|
50
|
+
capture.WaveFormat.Channels + "ch -> " + output);
|
|
51
|
+
|
|
52
|
+
var deadline = DateTime.UtcNow.AddMilliseconds(maxMs);
|
|
53
|
+
while (DateTime.UtcNow < deadline && !stopped.IsSet)
|
|
54
|
+
{
|
|
55
|
+
if (stopFile != null && File.Exists(stopFile)) break;
|
|
56
|
+
Thread.Sleep(150);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
capture.StopRecording();
|
|
60
|
+
stopped.Wait(3000);
|
|
61
|
+
writer.Flush();
|
|
62
|
+
|
|
63
|
+
if (failure != null) { Console.Error.WriteLine(failure.GetType().Name + ": " + failure.Message); return 6; }
|
|
64
|
+
|
|
65
|
+
writer.Dispose();
|
|
66
|
+
Normalize(devicePath, output);
|
|
67
|
+
try { File.Delete(devicePath); } catch { }
|
|
68
|
+
long bytes = new FileInfo(output).Length;
|
|
69
|
+
Console.WriteLine("recorded 24kHz stereo, " + bytes + " bytes -> " + output);
|
|
70
|
+
return 0;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
catch (Exception ex)
|
|
74
|
+
{
|
|
75
|
+
Console.Error.WriteLine(ex.GetType().Name + ": " + ex.Message);
|
|
76
|
+
return 5;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/// Resample/convert the device-format capture into 24 kHz stereo 16-bit PCM.
|
|
81
|
+
static void Normalize(string source, string destination)
|
|
82
|
+
{
|
|
83
|
+
using (var reader = new WaveFileReader(source))
|
|
84
|
+
{
|
|
85
|
+
ISampleProvider samples = reader.ToSampleProvider();
|
|
86
|
+
if (samples.WaveFormat.Channels == 1)
|
|
87
|
+
samples = new MonoToStereoSampleProvider(samples);
|
|
88
|
+
ISampleProvider resampled = samples.WaveFormat.SampleRate == 24000
|
|
89
|
+
? samples
|
|
90
|
+
: new WdlResamplingSampleProvider(samples, 24000);
|
|
91
|
+
WaveFileWriter.CreateWaveFile16(destination, resampled);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
<Project Sdk="Microsoft.NET.Sdk">
|
|
2
|
+
<PropertyGroup>
|
|
3
|
+
<OutputType>Exe</OutputType>
|
|
4
|
+
<TargetFramework>net48</TargetFramework>
|
|
5
|
+
<LangVersion>latest</LangVersion>
|
|
6
|
+
<Nullable>disable</Nullable>
|
|
7
|
+
<PlatformTarget>x64</PlatformTarget>
|
|
8
|
+
<AssemblyName>dsh-cu-capture</AssemblyName>
|
|
9
|
+
<RootNamespace>DshCuCapture</RootNamespace>
|
|
10
|
+
</PropertyGroup>
|
|
11
|
+
<ItemGroup>
|
|
12
|
+
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.3" PrivateAssets="all" />
|
|
13
|
+
<PackageReference Include="SharpDX.Direct3D11" Version="4.2.0" />
|
|
14
|
+
<PackageReference Include="SharpDX.DXGI" Version="4.2.0" />
|
|
15
|
+
</ItemGroup>
|
|
16
|
+
<ItemGroup>
|
|
17
|
+
<Reference Include="Windows.winmd">
|
|
18
|
+
<HintPath>C:\Program Files (x86)\Windows Kits\10\UnionMetadata\10.0.22621.0\Windows.winmd</HintPath>
|
|
19
|
+
<IsWinMDFile>true</IsWinMDFile>
|
|
20
|
+
</Reference>
|
|
21
|
+
<Reference Include="System.Runtime.WindowsRuntime" />
|
|
22
|
+
</ItemGroup>
|
|
23
|
+
</Project>
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
using System;
|
|
2
|
+
using System.Drawing;
|
|
3
|
+
using System.Drawing.Imaging;
|
|
4
|
+
using System.Runtime.InteropServices;
|
|
5
|
+
using System.Threading;
|
|
6
|
+
using SharpDX.Direct3D;
|
|
7
|
+
using SharpDX.Direct3D11;
|
|
8
|
+
using SharpDX.DXGI;
|
|
9
|
+
using Windows.Graphics.Capture;
|
|
10
|
+
using Windows.Graphics.DirectX;
|
|
11
|
+
using Windows.Graphics.DirectX.Direct3D11;
|
|
12
|
+
using D3D11Device = SharpDX.Direct3D11.Device;
|
|
13
|
+
using D3D11Texture2D = SharpDX.Direct3D11.Texture2D;
|
|
14
|
+
using DXGIDevice = SharpDX.DXGI.Device;
|
|
15
|
+
|
|
16
|
+
namespace DshCuCapture
|
|
17
|
+
{
|
|
18
|
+
/// Windows.Graphics.Capture for .NET Framework 4.8 - the same API Codex uses,
|
|
19
|
+
/// with no .NET 8 runtime dependency (Framework 4.8 ships with Windows 10/11).
|
|
20
|
+
internal static class Program
|
|
21
|
+
{
|
|
22
|
+
[ComImport]
|
|
23
|
+
[Guid("A9B3D012-3DF2-4EE3-B8D1-8695F457D3C1")]
|
|
24
|
+
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
|
|
25
|
+
private interface IDirect3DDxgiInterfaceAccess
|
|
26
|
+
{
|
|
27
|
+
[PreserveSig] int GetInterface(ref Guid iid, out IntPtr result);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
[DllImport("d3d11.dll", ExactSpelling = true)]
|
|
31
|
+
private static extern int CreateDirect3D11DeviceFromDXGIDevice(IntPtr dxgiDevice, out IntPtr graphicsDevice);
|
|
32
|
+
|
|
33
|
+
[DllImport("combase.dll", CharSet = CharSet.Unicode)]
|
|
34
|
+
private static extern int WindowsCreateString(string source, int length, out IntPtr hstring);
|
|
35
|
+
|
|
36
|
+
[DllImport("combase.dll")]
|
|
37
|
+
private static extern int RoGetActivationFactory(IntPtr activatableClassId, ref Guid iid, out IntPtr factory);
|
|
38
|
+
|
|
39
|
+
[DllImport("combase.dll")]
|
|
40
|
+
private static extern int WindowsDeleteString(IntPtr hstring);
|
|
41
|
+
|
|
42
|
+
[DllImport("kernel32.dll", EntryPoint = "RtlMoveMemory")]
|
|
43
|
+
private static extern void CopyMemory(IntPtr destination, IntPtr source, int length);
|
|
44
|
+
|
|
45
|
+
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
|
|
46
|
+
private delegate int CreateForWindowDelegate(IntPtr thisPtr, IntPtr hwnd, ref Guid iid, out IntPtr item);
|
|
47
|
+
private delegate int CreateForMonitorDelegate(IntPtr thisPtr, IntPtr hmon, ref Guid iid, out IntPtr item);
|
|
48
|
+
|
|
49
|
+
[DllImport("user32.dll")] private static extern IntPtr MonitorFromPoint(POINT pt, uint flags);
|
|
50
|
+
[StructLayout(LayoutKind.Sequential)] private struct POINT { public int X, Y; }
|
|
51
|
+
|
|
52
|
+
private static bool _suppressBorder = true;
|
|
53
|
+
|
|
54
|
+
private sealed class Slot
|
|
55
|
+
{
|
|
56
|
+
public Bitmap Bitmap;
|
|
57
|
+
public Exception Error;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
private static int Main(string[] args)
|
|
61
|
+
{
|
|
62
|
+
long hwnd = 0;
|
|
63
|
+
bool monitor = false;
|
|
64
|
+
bool redactAll = false;
|
|
65
|
+
bool suppressBorder = true;
|
|
66
|
+
var redactRects = new System.Collections.Generic.List<System.Drawing.Rectangle>();
|
|
67
|
+
string output = "capture.png";
|
|
68
|
+
int timeoutMs = 2000;
|
|
69
|
+
for (int i = 0; i < args.Length - 1; i++)
|
|
70
|
+
{
|
|
71
|
+
if (args[i] == "--redact")
|
|
72
|
+
{
|
|
73
|
+
string[] parts = args[i + 1].Split(',');
|
|
74
|
+
if (parts.Length == 4)
|
|
75
|
+
redactRects.Add(new System.Drawing.Rectangle(int.Parse(parts[0]), int.Parse(parts[1]), int.Parse(parts[2]), int.Parse(parts[3])));
|
|
76
|
+
}
|
|
77
|
+
else if (args[i] == "--hwnd") long.TryParse(args[i + 1], out hwnd);
|
|
78
|
+
else if (args[i] == "--out") output = args[i + 1];
|
|
79
|
+
else if (args[i] == "--timeout") int.TryParse(args[i + 1], out timeoutMs);
|
|
80
|
+
}
|
|
81
|
+
for (int i = 0; i < args.Length; i++)
|
|
82
|
+
{
|
|
83
|
+
if (args[i] == "--monitor") monitor = true;
|
|
84
|
+
else if (args[i] == "--redact-all") redactAll = true;
|
|
85
|
+
else if (args[i] == "--keep-border") suppressBorder = false;
|
|
86
|
+
}
|
|
87
|
+
if (hwnd == 0 && !monitor) { Console.Error.WriteLine("--hwnd or --monitor is required"); return 2; }
|
|
88
|
+
|
|
89
|
+
try
|
|
90
|
+
{
|
|
91
|
+
GraphicsCaptureItem item = monitor ? CreateMonitorItem() : CreateItem(new IntPtr(hwnd));
|
|
92
|
+
using (D3D11Device device = new D3D11Device(DriverType.Hardware, DeviceCreationFlags.BgraSupport))
|
|
93
|
+
using (DXGIDevice dxgi = device.QueryInterface<DXGIDevice>())
|
|
94
|
+
{
|
|
95
|
+
int hr = CreateDirect3D11DeviceFromDXGIDevice(dxgi.NativePointer, out IntPtr winrtDevicePtr);
|
|
96
|
+
if (hr < 0) { Console.Error.WriteLine("CreateDirect3D11DeviceFromDXGIDevice hr=0x" + hr.ToString("X8")); return 3; }
|
|
97
|
+
IDirect3DDevice winrtDevice;
|
|
98
|
+
try { winrtDevice = (IDirect3DDevice)Marshal.GetObjectForIUnknown(winrtDevicePtr); }
|
|
99
|
+
finally { Marshal.Release(winrtDevicePtr); }
|
|
100
|
+
|
|
101
|
+
_suppressBorder = suppressBorder;
|
|
102
|
+
var slot = new Slot();
|
|
103
|
+
using (var pool = Direct3D11CaptureFramePool.CreateFreeThreaded(
|
|
104
|
+
winrtDevice, DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, item.Size))
|
|
105
|
+
using (var session = pool.CreateCaptureSession(item))
|
|
106
|
+
using (var done = new ManualResetEventSlim(false))
|
|
107
|
+
{
|
|
108
|
+
try { session.IsCursorCaptureEnabled = false; } catch { }
|
|
109
|
+
// Windows 11 draws a yellow frame around any captured window. Kill it
|
|
110
|
+
// (Codex does the same through its capture-border access path) so the
|
|
111
|
+
// border never shows up in the picture or on the user's desktop.
|
|
112
|
+
if (_suppressBorder) { try { session.IsBorderRequired = false; } catch { } }
|
|
113
|
+
|
|
114
|
+
Direct3D11CaptureFramePool owner = pool;
|
|
115
|
+
D3D11Device gpu = device;
|
|
116
|
+
pool.FrameArrived += (sender, e) =>
|
|
117
|
+
{
|
|
118
|
+
if (slot.Bitmap != null) return;
|
|
119
|
+
try
|
|
120
|
+
{
|
|
121
|
+
using (var frame = owner.TryGetNextFrame())
|
|
122
|
+
{
|
|
123
|
+
if (frame == null) return;
|
|
124
|
+
var access = (IDirect3DDxgiInterfaceAccess)frame.Surface;
|
|
125
|
+
Guid iid = new Guid("6f15aaf2-d208-4e89-9ab4-489535d34f9c"); // ID3D11Texture2D
|
|
126
|
+
IntPtr texPtr;
|
|
127
|
+
if (access.GetInterface(ref iid, out texPtr) < 0 || texPtr == IntPtr.Zero) return;
|
|
128
|
+
using (var texture = new D3D11Texture2D(texPtr))
|
|
129
|
+
{
|
|
130
|
+
var desc = texture.Description;
|
|
131
|
+
var stagingDesc = new Texture2DDescription
|
|
132
|
+
{
|
|
133
|
+
Width = desc.Width,
|
|
134
|
+
Height = desc.Height,
|
|
135
|
+
MipLevels = 1,
|
|
136
|
+
ArraySize = 1,
|
|
137
|
+
Format = desc.Format,
|
|
138
|
+
SampleDescription = new SampleDescription(1, 0),
|
|
139
|
+
Usage = ResourceUsage.Staging,
|
|
140
|
+
BindFlags = BindFlags.None,
|
|
141
|
+
CpuAccessFlags = CpuAccessFlags.Read,
|
|
142
|
+
OptionFlags = ResourceOptionFlags.None,
|
|
143
|
+
};
|
|
144
|
+
using (var staging = new D3D11Texture2D(gpu, stagingDesc))
|
|
145
|
+
{
|
|
146
|
+
var context = gpu.ImmediateContext;
|
|
147
|
+
context.CopyResource(texture, staging);
|
|
148
|
+
SharpDX.DataBox box = context.MapSubresource(staging, 0, MapMode.Read, SharpDX.Direct3D11.MapFlags.None);
|
|
149
|
+
try
|
|
150
|
+
{
|
|
151
|
+
Bitmap bmp = new Bitmap(desc.Width, desc.Height, PixelFormat.Format32bppArgb);
|
|
152
|
+
BitmapData data = bmp.LockBits(new Rectangle(0, 0, desc.Width, desc.Height), ImageLockMode.WriteOnly, PixelFormat.Format32bppArgb);
|
|
153
|
+
for (int y = 0; y < desc.Height; y++)
|
|
154
|
+
{
|
|
155
|
+
IntPtr src = IntPtr.Add(box.DataPointer, y * box.RowPitch);
|
|
156
|
+
IntPtr dst = IntPtr.Add(data.Scan0, y * data.Stride);
|
|
157
|
+
CopyMemory(dst, src, desc.Width * 4);
|
|
158
|
+
}
|
|
159
|
+
bmp.UnlockBits(data);
|
|
160
|
+
slot.Bitmap = bmp;
|
|
161
|
+
}
|
|
162
|
+
finally { context.UnmapSubresource(staging, 0); }
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
catch (Exception ex) { slot.Error = ex; }
|
|
168
|
+
finally { done.Set(); }
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
session.StartCapture();
|
|
172
|
+
done.Wait(timeoutMs);
|
|
173
|
+
try { session.Dispose(); } catch { }
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
if (slot.Error != null) { Console.Error.WriteLine(slot.Error.GetType().Name + ": " + slot.Error.Message); return 6; }
|
|
177
|
+
if (slot.Bitmap == null) { Console.Error.WriteLine("no frame arrived (timeout or empty surface)"); return 4; }
|
|
178
|
+
|
|
179
|
+
if (redactAll || redactRects.Count > 0)
|
|
180
|
+
{
|
|
181
|
+
using (var g = Graphics.FromImage(slot.Bitmap))
|
|
182
|
+
{
|
|
183
|
+
g.Clear(redactAll ? Color.Black : Color.Transparent);
|
|
184
|
+
foreach (var rect in redactRects) g.FillRectangle(Brushes.Black, rect);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
int w = slot.Bitmap.Width, h = slot.Bitmap.Height;
|
|
188
|
+
slot.Bitmap.Save(output, ImageFormat.Png);
|
|
189
|
+
slot.Bitmap.Dispose();
|
|
190
|
+
Console.WriteLine("wgc " + w + "x" + h + " -> " + output);
|
|
191
|
+
return 0;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
catch (Exception ex)
|
|
195
|
+
{
|
|
196
|
+
Console.Error.WriteLine(ex.GetType().Name + ": " + ex.Message);
|
|
197
|
+
return 5;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
private static GraphicsCaptureItem CreateMonitorItem()
|
|
202
|
+
{
|
|
203
|
+
POINT origin = new POINT { X = 0, Y = 0 };
|
|
204
|
+
IntPtr monitorHandle = MonitorFromPoint(origin, 2); // MONITOR_DEFAULTTONEAREST
|
|
205
|
+
const string className = "Windows.Graphics.Capture.GraphicsCaptureItem";
|
|
206
|
+
WindowsCreateString(className, className.Length, out IntPtr hstr);
|
|
207
|
+
try
|
|
208
|
+
{
|
|
209
|
+
Guid interopIid = new Guid("3628E81B-3CAC-4C60-B7F4-23CE0E0C3356");
|
|
210
|
+
int hr = RoGetActivationFactory(hstr, ref interopIid, out IntPtr factoryPtr);
|
|
211
|
+
if (hr < 0) throw new COMException("RoGetActivationFactory hr=0x" + hr.ToString("X8"), hr);
|
|
212
|
+
IntPtr vtable = Marshal.ReadIntPtr(factoryPtr);
|
|
213
|
+
IntPtr fn = Marshal.ReadIntPtr(vtable, 4 * IntPtr.Size); // CreateForMonitor
|
|
214
|
+
var createForMonitor = (CreateForMonitorDelegate)Marshal.GetDelegateForFunctionPointer(fn, typeof(CreateForMonitorDelegate));
|
|
215
|
+
Guid itemIid = new Guid("79C3F95B-31F7-4EC2-A464-632EF5D30760");
|
|
216
|
+
int hrItem = createForMonitor(factoryPtr, monitorHandle, ref itemIid, out IntPtr itemPtr);
|
|
217
|
+
Marshal.Release(factoryPtr);
|
|
218
|
+
if (hrItem < 0) throw new COMException("CreateForMonitor hr=0x" + hrItem.ToString("X8"), hrItem);
|
|
219
|
+
try { return (GraphicsCaptureItem)Marshal.GetObjectForIUnknown(itemPtr); }
|
|
220
|
+
finally { Marshal.Release(itemPtr); }
|
|
221
|
+
}
|
|
222
|
+
finally { WindowsDeleteString(hstr); }
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
private static GraphicsCaptureItem CreateItem(IntPtr hwnd)
|
|
226
|
+
{
|
|
227
|
+
const string className = "Windows.Graphics.Capture.GraphicsCaptureItem";
|
|
228
|
+
WindowsCreateString(className, className.Length, out IntPtr hstr);
|
|
229
|
+
try
|
|
230
|
+
{
|
|
231
|
+
Guid interopIid = new Guid("3628E81B-3CAC-4C60-B7F4-23CE0E0C3356");
|
|
232
|
+
int hr = RoGetActivationFactory(hstr, ref interopIid, out IntPtr factoryPtr);
|
|
233
|
+
if (hr < 0) throw new COMException("RoGetActivationFactory hr=0x" + hr.ToString("X8"), hr);
|
|
234
|
+
|
|
235
|
+
IntPtr vtable = Marshal.ReadIntPtr(factoryPtr);
|
|
236
|
+
IntPtr fn = Marshal.ReadIntPtr(vtable, 3 * IntPtr.Size);
|
|
237
|
+
var createForWindow = (CreateForWindowDelegate)Marshal.GetDelegateForFunctionPointer(fn, typeof(CreateForWindowDelegate));
|
|
238
|
+
|
|
239
|
+
Guid itemIid = new Guid("79C3F95B-31F7-4EC2-A464-632EF5D30760");
|
|
240
|
+
int hrItem = createForWindow(factoryPtr, hwnd, ref itemIid, out IntPtr itemPtr);
|
|
241
|
+
Marshal.Release(factoryPtr);
|
|
242
|
+
if (hrItem < 0) throw new COMException("CreateForWindow hr=0x" + hrItem.ToString("X8"), hrItem);
|
|
243
|
+
try { return (GraphicsCaptureItem)Marshal.GetObjectForIUnknown(itemPtr); }
|
|
244
|
+
finally { Marshal.Release(itemPtr); }
|
|
245
|
+
}
|
|
246
|
+
finally { WindowsDeleteString(hstr); }
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,68 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "deepseek-computer-use",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Windows Computer Use for DeepSeek Harness: UIA observation, SendInput actions, Windows.Graphics.Capture — modeled on Codex's window2 API. Community plugin, not an official DeepSeek package.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "lib/index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./lib/index.js",
|
|
9
|
+
"./cordis.patch.yml": "./cordis.patch.yml",
|
|
10
|
+
"./package.json": "./package.json"
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"lib",
|
|
14
|
+
"native/Json.cs",
|
|
15
|
+
"native/Driver.cs",
|
|
16
|
+
"native/Program.cs",
|
|
17
|
+
"native/Overlay.cs",
|
|
18
|
+
"native/audio/Audio.csproj",
|
|
19
|
+
"native/audio/Program.cs",
|
|
20
|
+
"native/wgcfw/Capture.csproj",
|
|
21
|
+
"native/wgcfw/Program.cs",
|
|
22
|
+
"skills",
|
|
23
|
+
"cordis.patch.yml",
|
|
24
|
+
"README.md",
|
|
25
|
+
"LICENSE"
|
|
26
|
+
],
|
|
27
|
+
"scripts": {
|
|
28
|
+
"build": "node lib/build.mjs",
|
|
29
|
+
"build:helper": "node lib/build.mjs",
|
|
30
|
+
"test": "node test/integration.mjs",
|
|
31
|
+
"test:all": "node test/integration.mjs && node test/parity.mjs && node test/parity-js.mjs"
|
|
32
|
+
},
|
|
33
|
+
"dsh": {
|
|
34
|
+
"bundle": {
|
|
35
|
+
"patch": "./cordis.patch.yml"
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"engines": {
|
|
39
|
+
"node": ">=18.17"
|
|
40
|
+
},
|
|
41
|
+
"keywords": [
|
|
42
|
+
"dsh",
|
|
43
|
+
"deepseek-harness",
|
|
44
|
+
"dsh-plugin",
|
|
45
|
+
"computer-use",
|
|
46
|
+
"windows",
|
|
47
|
+
"uia",
|
|
48
|
+
"desktop-automation",
|
|
49
|
+
"wgc",
|
|
50
|
+
"accessibility"
|
|
51
|
+
],
|
|
52
|
+
"license": "MIT",
|
|
53
|
+
"author": "STARDUSTLC666",
|
|
54
|
+
"repository": {
|
|
55
|
+
"type": "git",
|
|
56
|
+
"url": "git+https://github.com/STARDUSTLC666/deepseek-computer-use.git"
|
|
57
|
+
},
|
|
58
|
+
"homepage": "https://github.com/STARDUSTLC666/deepseek-computer-use#readme",
|
|
59
|
+
"bugs": {
|
|
60
|
+
"url": "https://github.com/STARDUSTLC666/deepseek-computer-use/issues"
|
|
61
|
+
},
|
|
62
|
+
"os": [
|
|
63
|
+
"win32"
|
|
64
|
+
],
|
|
65
|
+
"publishConfig": {
|
|
66
|
+
"access": "public"
|
|
67
|
+
}
|
|
68
|
+
}
|