@midscene/computer 1.12.7-beta-20260914023554.0 → 1.12.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/darwin/rdp-helper +0 -0
- package/bin/linux/rdp-helper +0 -0
- package/dist/es/cli.mjs +2 -2
- package/dist/es/index.mjs +2 -2
- package/dist/lib/cli.js +2 -2
- package/dist/lib/index.js +2 -2
- package/native/rdp/CMakeLists.txt +22 -0
- package/native/rdp/include/rdp_helper_framebuffer.hpp +24 -0
- package/native/rdp/include/rdp_helper_session.hpp +7 -3
- package/native/rdp/src/png.cpp +5 -2
- package/native/rdp/src/session.cpp +60 -55
- package/native/rdp/tests/screenshot_test.cpp +365 -0
- package/native/rdp/tests/xrdp_desktop.py +135 -0
- package/package.json +3 -3
package/bin/darwin/rdp-helper
CHANGED
|
Binary file
|
package/bin/linux/rdp-helper
CHANGED
|
Binary file
|
package/dist/es/cli.mjs
CHANGED
|
@@ -1212,7 +1212,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1212
1212
|
}
|
|
1213
1213
|
async healthCheck(displays) {
|
|
1214
1214
|
console.log('[HealthCheck] Starting health check...');
|
|
1215
|
-
console.log("[HealthCheck] @midscene/computer v1.12.7
|
|
1215
|
+
console.log("[HealthCheck] @midscene/computer v1.12.7");
|
|
1216
1216
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1217
1217
|
const screenshotTimeout = 15000;
|
|
1218
1218
|
let timeoutId;
|
|
@@ -2688,7 +2688,7 @@ const tools = new ComputerMidsceneTools({
|
|
|
2688
2688
|
});
|
|
2689
2689
|
runToolsCLI(tools, 'midscene-computer', {
|
|
2690
2690
|
stripPrefix: 'computer_',
|
|
2691
|
-
version: "1.12.7
|
|
2691
|
+
version: "1.12.7",
|
|
2692
2692
|
extraCommands: createReportCliCommands()
|
|
2693
2693
|
}).catch((e)=>{
|
|
2694
2694
|
process.exit(reportCLIError(e));
|
package/dist/es/index.mjs
CHANGED
|
@@ -1212,7 +1212,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1212
1212
|
}
|
|
1213
1213
|
async healthCheck(displays) {
|
|
1214
1214
|
console.log('[HealthCheck] Starting health check...');
|
|
1215
|
-
console.log("[HealthCheck] @midscene/computer v1.12.7
|
|
1215
|
+
console.log("[HealthCheck] @midscene/computer v1.12.7");
|
|
1216
1216
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1217
1217
|
const screenshotTimeout = 15000;
|
|
1218
1218
|
let timeoutId;
|
|
@@ -2727,7 +2727,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2727
2727
|
}
|
|
2728
2728
|
}
|
|
2729
2729
|
function version() {
|
|
2730
|
-
const currentVersion = "1.12.7
|
|
2730
|
+
const currentVersion = "1.12.7";
|
|
2731
2731
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2732
2732
|
return currentVersion;
|
|
2733
2733
|
}
|
package/dist/lib/cli.js
CHANGED
|
@@ -1238,7 +1238,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1238
1238
|
}
|
|
1239
1239
|
async healthCheck(displays) {
|
|
1240
1240
|
console.log('[HealthCheck] Starting health check...');
|
|
1241
|
-
console.log("[HealthCheck] @midscene/computer v1.12.7
|
|
1241
|
+
console.log("[HealthCheck] @midscene/computer v1.12.7");
|
|
1242
1242
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1243
1243
|
const screenshotTimeout = 15000;
|
|
1244
1244
|
let timeoutId;
|
|
@@ -2715,7 +2715,7 @@ const tools = new ComputerMidsceneTools({
|
|
|
2715
2715
|
});
|
|
2716
2716
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
2717
2717
|
stripPrefix: 'computer_',
|
|
2718
|
-
version: "1.12.7
|
|
2718
|
+
version: "1.12.7",
|
|
2719
2719
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
2720
2720
|
}).catch((e)=>{
|
|
2721
2721
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|
package/dist/lib/index.js
CHANGED
|
@@ -1264,7 +1264,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1264
1264
|
}
|
|
1265
1265
|
async healthCheck(displays) {
|
|
1266
1266
|
console.log('[HealthCheck] Starting health check...');
|
|
1267
|
-
console.log("[HealthCheck] @midscene/computer v1.12.7
|
|
1267
|
+
console.log("[HealthCheck] @midscene/computer v1.12.7");
|
|
1268
1268
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1269
1269
|
const screenshotTimeout = 15000;
|
|
1270
1270
|
let timeoutId;
|
|
@@ -2785,7 +2785,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2785
2785
|
}
|
|
2786
2786
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
2787
2787
|
function version() {
|
|
2788
|
-
const currentVersion = "1.12.7
|
|
2788
|
+
const currentVersion = "1.12.7";
|
|
2789
2789
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2790
2790
|
return currentVersion;
|
|
2791
2791
|
}
|
|
@@ -92,6 +92,28 @@ if(BUILD_TESTING)
|
|
|
92
92
|
NAME rdp-session-policy-test
|
|
93
93
|
COMMAND rdp-session-policy-test
|
|
94
94
|
)
|
|
95
|
+
|
|
96
|
+
add_executable(
|
|
97
|
+
rdp-screenshot-test
|
|
98
|
+
tests/screenshot_test.cpp
|
|
99
|
+
src/png.cpp
|
|
100
|
+
src/session.cpp
|
|
101
|
+
)
|
|
102
|
+
target_include_directories(
|
|
103
|
+
rdp-screenshot-test PRIVATE
|
|
104
|
+
${CMAKE_CURRENT_SOURCE_DIR}/include
|
|
105
|
+
${FREERDP_INCLUDE_DIRS}
|
|
106
|
+
)
|
|
107
|
+
target_link_libraries(
|
|
108
|
+
rdp-screenshot-test PRIVATE
|
|
109
|
+
${FREERDP_LIBRARIES}
|
|
110
|
+
Threads::Threads
|
|
111
|
+
)
|
|
112
|
+
if(APPLE)
|
|
113
|
+
target_compile_definitions(rdp-screenshot-test PRIVATE _DARWIN_C_SOURCE)
|
|
114
|
+
endif()
|
|
115
|
+
add_test(NAME rdp-screenshot-test COMMAND rdp-screenshot-test)
|
|
116
|
+
set_tests_properties(rdp-screenshot-test PROPERTIES TIMEOUT 30)
|
|
95
117
|
endif()
|
|
96
118
|
|
|
97
119
|
if(APPLE)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <algorithm>
|
|
4
|
+
#include <chrono>
|
|
5
|
+
|
|
6
|
+
#include "rdp_helper_session.hpp"
|
|
7
|
+
|
|
8
|
+
namespace midscene::rdp {
|
|
9
|
+
|
|
10
|
+
// FreeRDP embeds its context first; shared by the callback and native tests.
|
|
11
|
+
struct MidsceneRdpContext {
|
|
12
|
+
rdpContext context;
|
|
13
|
+
FreeRdpSessionTransport* owner = nullptr;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
// Compute the next initial-screenshot wakeup without consulting wall time.
|
|
17
|
+
inline std::chrono::steady_clock::time_point ScreenshotWakeAt(
|
|
18
|
+
std::chrono::steady_clock::time_point started,
|
|
19
|
+
std::chrono::steady_clock::time_point last_update) {
|
|
20
|
+
return std::min(std::max(started, last_update) + std::chrono::milliseconds(300),
|
|
21
|
+
started + std::chrono::seconds(3));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
} // namespace midscene::rdp
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#pragma once
|
|
2
2
|
|
|
3
3
|
#include <atomic>
|
|
4
|
+
#include <chrono>
|
|
4
5
|
#include <condition_variable>
|
|
5
6
|
#include <cstdint>
|
|
6
7
|
#include <mutex>
|
|
@@ -76,7 +77,7 @@ class FreeRdpSessionTransport final : public SessionTransport {
|
|
|
76
77
|
void HookEndPaint(rdpUpdate* update);
|
|
77
78
|
// Records that at least one paint has reached the local framebuffer and
|
|
78
79
|
// wakes Connect().
|
|
79
|
-
void MarkFramePainted(
|
|
80
|
+
void MarkFramePainted();
|
|
80
81
|
// Records a framebuffer update even when it is too blank or uniform to count
|
|
81
82
|
// as the first informative frame.
|
|
82
83
|
void MarkFramebufferUpdated();
|
|
@@ -90,6 +91,7 @@ class FreeRdpSessionTransport final : public SessionTransport {
|
|
|
90
91
|
|
|
91
92
|
private:
|
|
92
93
|
friend struct MidsceneRdpContext;
|
|
94
|
+
friend struct RdpScreenshotTestPeer;
|
|
93
95
|
|
|
94
96
|
freerdp* instance_ = nullptr;
|
|
95
97
|
std::thread event_thread_;
|
|
@@ -110,8 +112,10 @@ class FreeRdpSessionTransport final : public SessionTransport {
|
|
|
110
112
|
std::atomic<bool> session_active_{false};
|
|
111
113
|
std::condition_variable frame_cv_;
|
|
112
114
|
std::mutex frame_mutex_;
|
|
113
|
-
|
|
114
|
-
|
|
115
|
+
// Protected by frame_mutex_; use a monotonic clock for screenshot settling.
|
|
116
|
+
std::chrono::steady_clock::time_point last_frame_update_{};
|
|
117
|
+
// Protected by mutex_; reset for each new connection.
|
|
118
|
+
bool first_screenshot_pending_ = true;
|
|
115
119
|
pEndPaint original_end_paint_ = nullptr;
|
|
116
120
|
uint16_t mouse_x_ = 0;
|
|
117
121
|
uint16_t mouse_y_ = 0;
|
package/native/rdp/src/png.cpp
CHANGED
|
@@ -81,12 +81,15 @@ std::vector<uint8_t> BuildImageData(const RawFrame& frame) {
|
|
|
81
81
|
const uint8_t blue = frame.bgra[pixel_offset];
|
|
82
82
|
const uint8_t green = frame.bgra[pixel_offset + 1];
|
|
83
83
|
const uint8_t red = frame.bgra[pixel_offset + 2];
|
|
84
|
-
const uint8_t alpha = frame.bgra[pixel_offset + 3];
|
|
85
84
|
|
|
86
85
|
image_data.push_back(red);
|
|
87
86
|
image_data.push_back(green);
|
|
88
87
|
image_data.push_back(blue);
|
|
89
|
-
|
|
88
|
+
// FreeRDP's PIXEL_FORMAT_BGRA32 is an opaque BGRX desktop surface. The
|
|
89
|
+
// fourth byte is padding and may be zero, so treating it as alpha makes
|
|
90
|
+
// otherwise valid screenshots transparent and turns them black during
|
|
91
|
+
// PNG-to-JPEG conversion.
|
|
92
|
+
image_data.push_back(0xFF);
|
|
90
93
|
}
|
|
91
94
|
}
|
|
92
95
|
|
|
@@ -37,17 +37,13 @@
|
|
|
37
37
|
#include <winpr/synch.h>
|
|
38
38
|
|
|
39
39
|
#include "rdp_helper_connection_policy.hpp"
|
|
40
|
+
#include "rdp_helper_framebuffer.hpp"
|
|
40
41
|
#include "rdp_helper_session_policy.hpp"
|
|
41
42
|
|
|
42
43
|
namespace midscene::rdp {
|
|
43
44
|
|
|
44
45
|
namespace {
|
|
45
46
|
|
|
46
|
-
struct MidsceneRdpContext {
|
|
47
|
-
rdpContext context;
|
|
48
|
-
FreeRdpSessionTransport* owner = nullptr;
|
|
49
|
-
};
|
|
50
|
-
|
|
51
47
|
struct LocalAddressTcpConnectContext {
|
|
52
48
|
std::string local_address;
|
|
53
49
|
rdpTransportIo default_io;
|
|
@@ -436,11 +432,11 @@ bool HasPendingFramebufferInvalidation(rdpContext* context) {
|
|
|
436
432
|
return hwnd->ninvalid > 0 && hwnd->invalid && !hwnd->invalid->null;
|
|
437
433
|
}
|
|
438
434
|
|
|
439
|
-
|
|
435
|
+
bool HasInformativeFramebuffer(rdpContext* context) {
|
|
440
436
|
if (!context || !context->gdi || !context->gdi->primary_buffer ||
|
|
441
437
|
context->gdi->width <= 0 || context->gdi->height <= 0 ||
|
|
442
438
|
context->gdi->stride == 0) {
|
|
443
|
-
return
|
|
439
|
+
return false;
|
|
444
440
|
}
|
|
445
441
|
|
|
446
442
|
rdpGdi* gdi = context->gdi;
|
|
@@ -448,7 +444,7 @@ std::optional<RawFrame> CaptureInformativeFramebuffer(rdpContext* context) {
|
|
|
448
444
|
const auto height = static_cast<size_t>(gdi->height);
|
|
449
445
|
const auto stride = static_cast<size_t>(gdi->stride);
|
|
450
446
|
if (stride < width * 4) {
|
|
451
|
-
return
|
|
447
|
+
return false;
|
|
452
448
|
}
|
|
453
449
|
|
|
454
450
|
const BYTE* buffer = gdi->primary_buffer;
|
|
@@ -489,15 +485,23 @@ std::optional<RawFrame> CaptureInformativeFramebuffer(rdpContext* context) {
|
|
|
489
485
|
|
|
490
486
|
if (colors.size() < kMinInformativeColorCount ||
|
|
491
487
|
non_black_pixels < min_non_black_pixels) {
|
|
492
|
-
return
|
|
488
|
+
return false;
|
|
493
489
|
}
|
|
490
|
+
return true;
|
|
491
|
+
}
|
|
494
492
|
|
|
493
|
+
// Called under the transport mutex so paints and resizes cannot race the copy.
|
|
494
|
+
RawFrame CopyFramebuffer(const rdpGdi& gdi) {
|
|
495
|
+
if (!gdi.primary_buffer || gdi.width <= 0 || gdi.height <= 0 ||
|
|
496
|
+
gdi.stride < static_cast<size_t>(gdi.width) * 4) {
|
|
497
|
+
throw std::runtime_error("Remote framebuffer is empty or invalid");
|
|
498
|
+
}
|
|
495
499
|
RawFrame frame;
|
|
496
|
-
frame.size.width = gdi
|
|
497
|
-
frame.size.height = gdi
|
|
498
|
-
frame.stride = stride;
|
|
499
|
-
const size_t buffer_size = stride * height;
|
|
500
|
-
frame.bgra.assign(
|
|
500
|
+
frame.size.width = gdi.width;
|
|
501
|
+
frame.size.height = gdi.height;
|
|
502
|
+
frame.stride = static_cast<size_t>(gdi.stride);
|
|
503
|
+
const size_t buffer_size = frame.stride * static_cast<size_t>(gdi.height);
|
|
504
|
+
frame.bgra.assign(gdi.primary_buffer, gdi.primary_buffer + buffer_size);
|
|
501
505
|
return frame;
|
|
502
506
|
}
|
|
503
507
|
|
|
@@ -516,9 +520,8 @@ BOOL MidsceneEndPaint(rdpContext* context) {
|
|
|
516
520
|
typed_context->owner->MarkFramebufferUpdated();
|
|
517
521
|
if (already_painted) {
|
|
518
522
|
typed_context->owner->MarkFramePainted();
|
|
519
|
-
} else if (
|
|
520
|
-
|
|
521
|
-
typed_context->owner->MarkFramePainted(std::move(first_frame));
|
|
523
|
+
} else if (HasInformativeFramebuffer(context)) {
|
|
524
|
+
typed_context->owner->MarkFramePainted();
|
|
522
525
|
}
|
|
523
526
|
}
|
|
524
527
|
}
|
|
@@ -983,9 +986,9 @@ ConnectionInfo FreeRdpSessionTransport::Connect(const ConnectionConfig& config)
|
|
|
983
986
|
framebuffer_updates_.store(0, std::memory_order_relaxed);
|
|
984
987
|
{
|
|
985
988
|
std::lock_guard<std::mutex> frame_lock(frame_mutex_);
|
|
986
|
-
|
|
987
|
-
first_frame_consumed_ = false;
|
|
989
|
+
last_frame_update_ = {};
|
|
988
990
|
}
|
|
991
|
+
first_screenshot_pending_ = true;
|
|
989
992
|
original_end_paint_ = nullptr;
|
|
990
993
|
ClearSessionErrorLocked();
|
|
991
994
|
}
|
|
@@ -1083,36 +1086,40 @@ void FreeRdpSessionTransport::Disconnect() {
|
|
|
1083
1086
|
}
|
|
1084
1087
|
|
|
1085
1088
|
RawFrame FreeRdpSessionTransport::CaptureFrame() {
|
|
1086
|
-
std::
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1089
|
+
const auto started = std::chrono::steady_clock::now();
|
|
1090
|
+
for (;;) {
|
|
1091
|
+
std::unique_lock<std::mutex> lock(mutex_);
|
|
1092
|
+
if (!connected_ || !instance_ || !instance_->context ||
|
|
1093
|
+
!instance_->context->gdi ||
|
|
1094
|
+
!session_active_.load(std::memory_order_relaxed)) {
|
|
1095
|
+
throw std::runtime_error(
|
|
1096
|
+
last_error_ ? "RDP screenshot failed: " + last_error_->message
|
|
1097
|
+
: "No remote framebuffer is available");
|
|
1098
|
+
}
|
|
1090
1099
|
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1100
|
+
if (frames_painted_.load(std::memory_order_relaxed) == 0) {
|
|
1101
|
+
throw std::runtime_error(
|
|
1102
|
+
"Remote framebuffer has not received its first paint yet");
|
|
1103
|
+
}
|
|
1095
1104
|
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
if (!first_frame_consumed_ && first_frame_.has_value()) {
|
|
1099
|
-
first_frame_consumed_ = true;
|
|
1100
|
-
return *first_frame_;
|
|
1105
|
+
if (!first_screenshot_pending_) {
|
|
1106
|
+
return CopyFramebuffer(*instance_->context->gdi);
|
|
1101
1107
|
}
|
|
1102
|
-
}
|
|
1103
1108
|
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1109
|
+
std::unique_lock<std::mutex> frame_lock(frame_mutex_);
|
|
1110
|
+
const auto now = std::chrono::steady_clock::now();
|
|
1111
|
+
const auto wake_at = ScreenshotWakeAt(started, last_frame_update_);
|
|
1112
|
+
if (now >= wake_at) {
|
|
1113
|
+
auto frame = CopyFramebuffer(*instance_->context->gdi);
|
|
1114
|
+
first_screenshot_pending_ = false;
|
|
1115
|
+
return frame;
|
|
1116
|
+
}
|
|
1108
1117
|
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
frame.bgra.assign(gdi->primary_buffer, gdi->primary_buffer + buffer_size);
|
|
1115
|
-
return frame;
|
|
1118
|
+
// Only the first screenshot settles. Release the event-loop mutex so
|
|
1119
|
+
// paints can continue; animations cannot extend the fixed deadline.
|
|
1120
|
+
lock.unlock();
|
|
1121
|
+
frame_cv_.wait_until(frame_lock, wake_at);
|
|
1122
|
+
}
|
|
1116
1123
|
}
|
|
1117
1124
|
|
|
1118
1125
|
Size FreeRdpSessionTransport::GetSize() {
|
|
@@ -1360,9 +1367,9 @@ void FreeRdpSessionTransport::ResetStateLocked() {
|
|
|
1360
1367
|
framebuffer_updates_.store(0, std::memory_order_relaxed);
|
|
1361
1368
|
{
|
|
1362
1369
|
std::lock_guard<std::mutex> frame_lock(frame_mutex_);
|
|
1363
|
-
|
|
1364
|
-
first_frame_consumed_ = false;
|
|
1370
|
+
last_frame_update_ = {};
|
|
1365
1371
|
}
|
|
1372
|
+
first_screenshot_pending_ = true;
|
|
1366
1373
|
original_end_paint_ = nullptr;
|
|
1367
1374
|
mouse_x_ = 0;
|
|
1368
1375
|
mouse_y_ = 0;
|
|
@@ -1388,23 +1395,21 @@ BOOL FreeRdpSessionTransport::CallOriginalEndPaint(rdpContext* context) {
|
|
|
1388
1395
|
return TRUE;
|
|
1389
1396
|
}
|
|
1390
1397
|
|
|
1391
|
-
void FreeRdpSessionTransport::MarkFramePainted(
|
|
1392
|
-
std::optional<RawFrame> first_frame) {
|
|
1398
|
+
void FreeRdpSessionTransport::MarkFramePainted() {
|
|
1393
1399
|
{
|
|
1394
1400
|
std::lock_guard<std::mutex> lock(frame_mutex_);
|
|
1395
|
-
if (first_frame.has_value() &&
|
|
1396
|
-
frames_painted_.load(std::memory_order_relaxed) == 0 &&
|
|
1397
|
-
!first_frame_.has_value()) {
|
|
1398
|
-
first_frame_ = std::move(*first_frame);
|
|
1399
|
-
first_frame_consumed_ = false;
|
|
1400
|
-
}
|
|
1401
1401
|
frames_painted_.fetch_add(1, std::memory_order_relaxed);
|
|
1402
1402
|
}
|
|
1403
1403
|
frame_cv_.notify_all();
|
|
1404
1404
|
}
|
|
1405
1405
|
|
|
1406
1406
|
void FreeRdpSessionTransport::MarkFramebufferUpdated() {
|
|
1407
|
-
|
|
1407
|
+
{
|
|
1408
|
+
std::lock_guard<std::mutex> lock(frame_mutex_);
|
|
1409
|
+
framebuffer_updates_.fetch_add(1, std::memory_order_relaxed);
|
|
1410
|
+
last_frame_update_ = std::chrono::steady_clock::now();
|
|
1411
|
+
}
|
|
1412
|
+
frame_cv_.notify_all();
|
|
1408
1413
|
}
|
|
1409
1414
|
|
|
1410
1415
|
bool FreeRdpSessionTransport::HasFramePainted() const {
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
#include <algorithm>
|
|
2
|
+
#include <chrono>
|
|
3
|
+
#include <future>
|
|
4
|
+
#include <iostream>
|
|
5
|
+
#include <stdexcept>
|
|
6
|
+
#include <string>
|
|
7
|
+
#include <thread>
|
|
8
|
+
|
|
9
|
+
#include <freerdp/gdi/gdi.h>
|
|
10
|
+
|
|
11
|
+
#include "rdp_helper_framebuffer.hpp"
|
|
12
|
+
#include "rdp_helper_png.hpp"
|
|
13
|
+
|
|
14
|
+
namespace midscene::rdp {
|
|
15
|
+
|
|
16
|
+
// Supply a framebuffer without a Windows server, but exercise the real
|
|
17
|
+
// CaptureFrame(), paint notifications, locks, and disconnect wakeups.
|
|
18
|
+
struct RdpScreenshotTestPeer {
|
|
19
|
+
FreeRdpSessionTransport transport;
|
|
20
|
+
freerdp instance{};
|
|
21
|
+
struct TestContext {
|
|
22
|
+
MidsceneRdpContext base{};
|
|
23
|
+
RdpScreenshotTestPeer* peer = nullptr;
|
|
24
|
+
} test_context;
|
|
25
|
+
MidsceneRdpContext& context = test_context.base;
|
|
26
|
+
rdpUpdate update{};
|
|
27
|
+
gdiBitmap primary{};
|
|
28
|
+
GDI_DC dc{};
|
|
29
|
+
GDI_WND window{};
|
|
30
|
+
GDI_RGN invalid{};
|
|
31
|
+
bool end_paint_ok = true;
|
|
32
|
+
int end_paint_calls = 0;
|
|
33
|
+
pEndPaint original_end_paint = nullptr;
|
|
34
|
+
rdpGdi gdi{};
|
|
35
|
+
std::vector<uint8_t> pixels;
|
|
36
|
+
|
|
37
|
+
RdpScreenshotTestPeer() : pixels(68 * 16, 255) {
|
|
38
|
+
gdi.width = 16;
|
|
39
|
+
gdi.height = 16;
|
|
40
|
+
gdi.stride = 68; // Exercise row padding as well as pixel data.
|
|
41
|
+
gdi.primary_buffer = pixels.data();
|
|
42
|
+
context.context.gdi = &gdi;
|
|
43
|
+
test_context.peer = this;
|
|
44
|
+
context.owner = &transport;
|
|
45
|
+
instance.context = &context.context;
|
|
46
|
+
gdi.primary = &primary;
|
|
47
|
+
primary.hdc = &dc;
|
|
48
|
+
dc.hwnd = &window;
|
|
49
|
+
window.invalid = &invalid;
|
|
50
|
+
update.EndPaint = [](rdpContext* context) -> BOOL {
|
|
51
|
+
auto& peer = *reinterpret_cast<TestContext*>(context)->peer;
|
|
52
|
+
++peer.end_paint_calls;
|
|
53
|
+
peer.window.ninvalid = 0;
|
|
54
|
+
peer.invalid.null = TRUE;
|
|
55
|
+
return peer.end_paint_ok ? TRUE : FALSE;
|
|
56
|
+
};
|
|
57
|
+
original_end_paint = update.EndPaint;
|
|
58
|
+
transport.HookEndPaint(&update);
|
|
59
|
+
transport.instance_ = &instance;
|
|
60
|
+
transport.connected_ = true;
|
|
61
|
+
transport.running_ = true;
|
|
62
|
+
transport.session_active_.store(true);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
~RdpScreenshotTestPeer() {
|
|
66
|
+
// These FreeRDP structs belong to the fixture, not freerdp_new().
|
|
67
|
+
transport.instance_ = nullptr;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
void Paint(uint8_t color, int rows = 16) {
|
|
71
|
+
std::lock_guard<std::mutex> lock(transport.mutex_);
|
|
72
|
+
for (int y = 0; y < 16; ++y) {
|
|
73
|
+
for (int x = 0; x < 16; ++x) {
|
|
74
|
+
const size_t offset = y * gdi.stride + x * 4;
|
|
75
|
+
std::fill_n(pixels.data() + offset, 3, y < rows ? color : 0);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
EndPaintLocked(true);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
void EndPaintLocked(bool invalidated) {
|
|
82
|
+
window.ninvalid = invalidated ? 1 : 0;
|
|
83
|
+
invalid.null = invalidated ? FALSE : TRUE;
|
|
84
|
+
update.EndPaint(&context.context);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
void InformativePaint(bool invalidated = true) {
|
|
88
|
+
std::lock_guard<std::mutex> lock(transport.mutex_);
|
|
89
|
+
for (int y = 0; y < 16; ++y) {
|
|
90
|
+
for (int x = 0; x < 16; ++x) {
|
|
91
|
+
const auto offset = y * gdi.stride + x * 4;
|
|
92
|
+
pixels[offset] = static_cast<uint8_t>(y * 16 + x);
|
|
93
|
+
pixels[offset + 1] = 128;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
EndPaintLocked(invalidated);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
std::unique_lock<std::mutex> HoldFrameNotificationLock() {
|
|
100
|
+
return std::unique_lock<std::mutex>(transport.frame_mutex_);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
uint64_t Updates() const {
|
|
104
|
+
return transport.framebuffer_updates_.load();
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
void ResetConnection() {
|
|
108
|
+
std::lock_guard<std::mutex> lock(transport.mutex_);
|
|
109
|
+
transport.ResetStateLocked();
|
|
110
|
+
transport.instance_ = &instance;
|
|
111
|
+
transport.connected_ = true;
|
|
112
|
+
transport.running_ = true;
|
|
113
|
+
transport.session_active_.store(true);
|
|
114
|
+
transport.original_end_paint_ = original_end_paint;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
void LoseSession() {
|
|
118
|
+
{
|
|
119
|
+
std::lock_guard<std::mutex> lock(transport.mutex_);
|
|
120
|
+
transport.connected_ = false;
|
|
121
|
+
transport.last_error_ = ErrorPayload{"session_lost", "test disconnect"};
|
|
122
|
+
}
|
|
123
|
+
transport.SignalSessionInactive();
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
} // namespace midscene::rdp
|
|
128
|
+
|
|
129
|
+
namespace {
|
|
130
|
+
|
|
131
|
+
using namespace std::chrono_literals;
|
|
132
|
+
using midscene::rdp::RawFrame;
|
|
133
|
+
using midscene::rdp::RdpScreenshotTestPeer;
|
|
134
|
+
|
|
135
|
+
void Expect(bool condition, const char* message) {
|
|
136
|
+
if (!condition) {
|
|
137
|
+
throw std::runtime_error(message);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
bool HasColor(const RawFrame& frame, uint8_t color) {
|
|
142
|
+
for (int y = 0; y < frame.size.height; ++y) {
|
|
143
|
+
for (int x = 0; x < frame.size.width; ++x) {
|
|
144
|
+
const size_t offset = y * frame.stride + x * 4;
|
|
145
|
+
for (size_t channel = 0; channel < 3; ++channel) {
|
|
146
|
+
if (frame.bgra[offset + channel] != color) {
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
uint32_t ReadUint32BigEndian(const std::vector<uint8_t>& bytes,
|
|
156
|
+
size_t offset) {
|
|
157
|
+
return (static_cast<uint32_t>(bytes.at(offset)) << 24U) |
|
|
158
|
+
(static_cast<uint32_t>(bytes.at(offset + 1)) << 16U) |
|
|
159
|
+
(static_cast<uint32_t>(bytes.at(offset + 2)) << 8U) |
|
|
160
|
+
static_cast<uint32_t>(bytes.at(offset + 3));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
void TestPngTreatsBgrxPaddingAsOpaque() {
|
|
164
|
+
RawFrame frame;
|
|
165
|
+
frame.size = {1, 1};
|
|
166
|
+
frame.stride = 4;
|
|
167
|
+
frame.bgra = {0x11, 0x22, 0x33, 0x00};
|
|
168
|
+
|
|
169
|
+
const auto png = midscene::rdp::EncodeFrameAsPng(frame);
|
|
170
|
+
size_t offset = 8;
|
|
171
|
+
while (offset + 12 <= png.size()) {
|
|
172
|
+
const size_t length = ReadUint32BigEndian(png, offset);
|
|
173
|
+
const std::string type(png.begin() + static_cast<std::ptrdiff_t>(offset + 4),
|
|
174
|
+
png.begin() + static_cast<std::ptrdiff_t>(offset + 8));
|
|
175
|
+
if (type == "IDAT") {
|
|
176
|
+
const size_t data_offset = offset + 8;
|
|
177
|
+
// The encoder writes one stored DEFLATE block. Its scanline begins
|
|
178
|
+
// after the two-byte zlib header and five-byte block header.
|
|
179
|
+
Expect(length >= 12, "encoded PNG IDAT payload is unexpectedly short");
|
|
180
|
+
Expect(png.at(data_offset + 7) == 0, "PNG scanline filter changed");
|
|
181
|
+
Expect(png.at(data_offset + 8) == 0x33, "PNG red channel changed");
|
|
182
|
+
Expect(png.at(data_offset + 9) == 0x22, "PNG green channel changed");
|
|
183
|
+
Expect(png.at(data_offset + 10) == 0x11, "PNG blue channel changed");
|
|
184
|
+
Expect(png.at(data_offset + 11) == 0xFF,
|
|
185
|
+
"BGRX padding was exposed as transparent PNG alpha");
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
offset += 12 + length;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
throw std::runtime_error("encoded PNG is missing IDAT");
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
void TestEndPaintPipeline() {
|
|
195
|
+
RdpScreenshotTestPeer peer;
|
|
196
|
+
peer.Paint(0);
|
|
197
|
+
Expect(!peer.transport.HasFramePainted(), "black startup passed readiness");
|
|
198
|
+
Expect(peer.Updates() == 1, "blank update was not recorded");
|
|
199
|
+
peer.InformativePaint(false);
|
|
200
|
+
Expect(!peer.transport.HasFramePainted(), "non-invalidated paint passed readiness");
|
|
201
|
+
Expect(peer.Updates() == 1, "non-invalidated paint counted as an update");
|
|
202
|
+
peer.end_paint_ok = false;
|
|
203
|
+
peer.InformativePaint();
|
|
204
|
+
Expect(!peer.transport.HasFramePainted(), "failed original callback passed readiness");
|
|
205
|
+
Expect(peer.Updates() == 1, "failed callback counted as an update");
|
|
206
|
+
peer.end_paint_ok = true;
|
|
207
|
+
peer.InformativePaint();
|
|
208
|
+
Expect(peer.transport.HasFramePainted(), "informative paint did not pass readiness");
|
|
209
|
+
Expect(peer.Updates() == 2, "invalidation was lost after original callback cleared it");
|
|
210
|
+
peer.Paint(0);
|
|
211
|
+
Expect(peer.Updates() == 3, "later black paint did not update settling");
|
|
212
|
+
Expect(peer.end_paint_calls == 5, "original EndPaint was not chained exactly once");
|
|
213
|
+
Expect(HasColor(peer.transport.CaptureFrame(), 0), "live black pixels were lost");
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
void TestScreenshotWakeTimes() {
|
|
217
|
+
const auto start = std::chrono::steady_clock::time_point{} + 10s;
|
|
218
|
+
using midscene::rdp::ScreenshotWakeAt;
|
|
219
|
+
Expect(ScreenshotWakeAt(start, start - 1s) == start + 300ms,
|
|
220
|
+
"first screenshot must wait even after an old paint");
|
|
221
|
+
Expect(ScreenshotWakeAt(start, start + 250ms) == start + 550ms,
|
|
222
|
+
"new paint must extend the quiet window");
|
|
223
|
+
Expect(ScreenshotWakeAt(start, start + 2900ms) == start + 3s,
|
|
224
|
+
"continuous updates must not extend the fixed deadline");
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
void TestPartialFirstPaint() {
|
|
228
|
+
RdpScreenshotTestPeer peer;
|
|
229
|
+
peer.InformativePaint();
|
|
230
|
+
peer.Paint(64, 2);
|
|
231
|
+
// The first informative frame is partial; later pixels must replace it even
|
|
232
|
+
// if both paints arrive before CaptureFrame starts (the old cache bug).
|
|
233
|
+
peer.Paint(192);
|
|
234
|
+
const auto frame = peer.transport.CaptureFrame();
|
|
235
|
+
Expect(HasColor(frame, 192), "first screenshot returned a partial old paint");
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
void TestLaterScreenshotsDoNotWait() {
|
|
239
|
+
RdpScreenshotTestPeer peer;
|
|
240
|
+
peer.InformativePaint();
|
|
241
|
+
peer.Paint(64);
|
|
242
|
+
peer.transport.CaptureFrame();
|
|
243
|
+
for (const uint8_t color : {0, 128}) {
|
|
244
|
+
peer.Paint(color);
|
|
245
|
+
// The fast path must not enter the settling section at all. Holding its
|
|
246
|
+
// mutex detects that structurally, without a tight wall-time assertion.
|
|
247
|
+
auto notification_lock = peer.HoldFrameNotificationLock();
|
|
248
|
+
auto capture = std::async(std::launch::async,
|
|
249
|
+
[&] { return peer.transport.CaptureFrame(); });
|
|
250
|
+
const auto status = capture.wait_for(5s);
|
|
251
|
+
notification_lock.unlock();
|
|
252
|
+
Expect(status == std::future_status::ready,
|
|
253
|
+
"later screenshot unnecessarily entered settling");
|
|
254
|
+
Expect(HasColor(capture.get(), color), "later screenshot returned stale pixels");
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
void TestContinuousUpdatesHaveDeadline() {
|
|
259
|
+
RdpScreenshotTestPeer peer;
|
|
260
|
+
peer.InformativePaint();
|
|
261
|
+
peer.Paint(128);
|
|
262
|
+
std::atomic<bool> stop{false};
|
|
263
|
+
auto painting = std::async(std::launch::async, [&] {
|
|
264
|
+
while (!stop.load()) {
|
|
265
|
+
peer.Paint(128);
|
|
266
|
+
std::this_thread::sleep_for(30ms);
|
|
267
|
+
}
|
|
268
|
+
});
|
|
269
|
+
auto capture = std::async(std::launch::async,
|
|
270
|
+
[&] { return peer.transport.CaptureFrame(); });
|
|
271
|
+
const auto status = capture.wait_for(5s);
|
|
272
|
+
stop.store(true);
|
|
273
|
+
painting.get();
|
|
274
|
+
Expect(status == std::future_status::ready,
|
|
275
|
+
"continuous updates extended the screenshot deadline");
|
|
276
|
+
Expect(HasColor(capture.get(), 128), "deadline did not return the live frame");
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
void TestBlackFirstScreenshot() {
|
|
280
|
+
RdpScreenshotTestPeer peer;
|
|
281
|
+
peer.InformativePaint();
|
|
282
|
+
peer.Paint(0);
|
|
283
|
+
Expect(HasColor(peer.transport.CaptureFrame(), 0),
|
|
284
|
+
"black current framebuffer should be returned after initial settling");
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
void TestReconnectResetsSettling() {
|
|
288
|
+
RdpScreenshotTestPeer peer;
|
|
289
|
+
peer.InformativePaint();
|
|
290
|
+
peer.Paint(64);
|
|
291
|
+
peer.transport.CaptureFrame();
|
|
292
|
+
peer.ResetConnection();
|
|
293
|
+
peer.InformativePaint();
|
|
294
|
+
peer.Paint(128);
|
|
295
|
+
auto capture = std::async(std::launch::async,
|
|
296
|
+
[&] { return peer.transport.CaptureFrame(); });
|
|
297
|
+
Expect(capture.wait_for(50ms) == std::future_status::timeout,
|
|
298
|
+
"new connection did not restore first-screenshot settling");
|
|
299
|
+
Expect(HasColor(capture.get(), 128), "new connection returned old pixels");
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
void TestDisconnectWakesScreenshot(uint8_t color) {
|
|
303
|
+
RdpScreenshotTestPeer peer;
|
|
304
|
+
peer.InformativePaint();
|
|
305
|
+
peer.Paint(color);
|
|
306
|
+
auto capture = std::async(std::launch::async, [&] {
|
|
307
|
+
try {
|
|
308
|
+
peer.transport.CaptureFrame();
|
|
309
|
+
return std::string();
|
|
310
|
+
} catch (const std::exception& error) {
|
|
311
|
+
return std::string(error.what());
|
|
312
|
+
}
|
|
313
|
+
});
|
|
314
|
+
Expect(capture.wait_for(50ms) == std::future_status::timeout,
|
|
315
|
+
"screenshot did not wait for framebuffer updates");
|
|
316
|
+
peer.LoseSession();
|
|
317
|
+
Expect(capture.wait_for(1s) == std::future_status::ready,
|
|
318
|
+
"session loss did not wake the screenshot waiter");
|
|
319
|
+
Expect(capture.get().find("test disconnect") != std::string::npos,
|
|
320
|
+
"screenshot lost the session failure reason");
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
void TestInvalidFramebuffer() {
|
|
324
|
+
for (int invalid_case = 0; invalid_case < 4; ++invalid_case) {
|
|
325
|
+
RdpScreenshotTestPeer peer;
|
|
326
|
+
peer.InformativePaint();
|
|
327
|
+
peer.Paint(64);
|
|
328
|
+
switch (invalid_case) {
|
|
329
|
+
case 0: peer.gdi.primary_buffer = nullptr; break;
|
|
330
|
+
case 1: peer.gdi.width = 0; break;
|
|
331
|
+
case 2: peer.gdi.height = 0; break;
|
|
332
|
+
case 3: peer.gdi.stride = 4; break;
|
|
333
|
+
}
|
|
334
|
+
std::string error_message;
|
|
335
|
+
try {
|
|
336
|
+
peer.transport.CaptureFrame();
|
|
337
|
+
} catch (const std::exception& error) {
|
|
338
|
+
error_message = error.what();
|
|
339
|
+
}
|
|
340
|
+
Expect(error_message == "Remote framebuffer is empty or invalid",
|
|
341
|
+
"invalid framebuffer layout was not rejected");
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
} // namespace
|
|
346
|
+
|
|
347
|
+
int main() {
|
|
348
|
+
try {
|
|
349
|
+
TestPngTreatsBgrxPaddingAsOpaque();
|
|
350
|
+
TestEndPaintPipeline();
|
|
351
|
+
TestScreenshotWakeTimes();
|
|
352
|
+
TestPartialFirstPaint();
|
|
353
|
+
TestLaterScreenshotsDoNotWait();
|
|
354
|
+
TestContinuousUpdatesHaveDeadline();
|
|
355
|
+
TestBlackFirstScreenshot();
|
|
356
|
+
TestReconnectResetsSettling();
|
|
357
|
+
TestDisconnectWakesScreenshot(0);
|
|
358
|
+
TestDisconnectWakesScreenshot(64);
|
|
359
|
+
TestInvalidFramebuffer();
|
|
360
|
+
return 0;
|
|
361
|
+
} catch (const std::exception& error) {
|
|
362
|
+
std::cerr << error.what() << '\n';
|
|
363
|
+
return 1;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
import tkinter as tk
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
WIDTH = 1280
|
|
7
|
+
HEIGHT = 720
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
root = tk.Tk()
|
|
11
|
+
root.title("Midscene RDP CI")
|
|
12
|
+
root.geometry(f"{WIDTH}x{HEIGHT}+0+0")
|
|
13
|
+
root.configure(background="#123456")
|
|
14
|
+
|
|
15
|
+
canvas = tk.Canvas(root, width=WIDTH, height=HEIGHT, highlightthickness=0)
|
|
16
|
+
canvas.pack(fill="both", expand=True)
|
|
17
|
+
desktop_items = []
|
|
18
|
+
for y in range(HEIGHT):
|
|
19
|
+
ratio = y / (HEIGHT - 1)
|
|
20
|
+
red = round(0x12 + (0xAB - 0x12) * ratio)
|
|
21
|
+
green = round(0x34 + (0xCD - 0x34) * ratio)
|
|
22
|
+
blue = round(0x56 + (0xEF - 0x56) * ratio)
|
|
23
|
+
desktop_items.append(
|
|
24
|
+
canvas.create_line(
|
|
25
|
+
0, y, WIDTH, y, fill=f"#{red:02x}{green:02x}{blue:02x}"
|
|
26
|
+
)
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
desktop_items.append(
|
|
30
|
+
canvas.create_text(
|
|
31
|
+
WIDTH // 2,
|
|
32
|
+
80,
|
|
33
|
+
text="Midscene RDP integration test",
|
|
34
|
+
fill="white",
|
|
35
|
+
font=("DejaVu Sans", 32, "bold"),
|
|
36
|
+
)
|
|
37
|
+
)
|
|
38
|
+
entry = tk.Entry(root, font=("DejaVu Sans Mono", 24), width=36)
|
|
39
|
+
desktop_items.append(canvas.create_window(WIDTH // 2, 165, window=entry))
|
|
40
|
+
entry.focus_force()
|
|
41
|
+
|
|
42
|
+
status_value = tk.StringVar(value="Ready for RDP input")
|
|
43
|
+
status = tk.Label(
|
|
44
|
+
root,
|
|
45
|
+
textvariable=status_value,
|
|
46
|
+
font=("DejaVu Sans", 18, "bold"),
|
|
47
|
+
background="#f4e04d",
|
|
48
|
+
width=34,
|
|
49
|
+
)
|
|
50
|
+
desktop_items.append(canvas.create_window(WIDTH // 2, 235, window=status))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def set_status(value):
|
|
54
|
+
status_value.set(value)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
entry.bind("<Return>", lambda _event: set_status("KeyboardPress: Enter"))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def action_target(text, x, y, background):
|
|
61
|
+
target = tk.Label(
|
|
62
|
+
root,
|
|
63
|
+
text=text,
|
|
64
|
+
font=("DejaVu Sans", 16, "bold"),
|
|
65
|
+
background=background,
|
|
66
|
+
width=16,
|
|
67
|
+
height=2,
|
|
68
|
+
)
|
|
69
|
+
desktop_items.append(canvas.create_window(x, y, window=target))
|
|
70
|
+
return target
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
hover_target = action_target("Hover", 180, 340, "#65d46e")
|
|
74
|
+
hover_target.bind("<Enter>", lambda _event: set_status("Hover received"))
|
|
75
|
+
|
|
76
|
+
double_click_target = action_target("Double click", 460, 340, "#ff9f43")
|
|
77
|
+
double_click_target.bind(
|
|
78
|
+
"<Double-Button-1>", lambda _event: set_status("DoubleClick received")
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
right_click_target = action_target("Right click", 740, 340, "#b388ff")
|
|
82
|
+
right_click_target.bind(
|
|
83
|
+
"<Button-3>", lambda _event: set_status("RightClick received")
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
drag_source = action_target("Drag source", 1020, 320, "#4dd0e1")
|
|
87
|
+
drop_target = action_target("Drop target", 1020, 440, "#ef5350")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def finish_drag(event):
|
|
91
|
+
pointer_x = drag_source.winfo_rootx() + event.x
|
|
92
|
+
pointer_y = drag_source.winfo_rooty() + event.y
|
|
93
|
+
if (
|
|
94
|
+
drop_target.winfo_rootx()
|
|
95
|
+
<= pointer_x
|
|
96
|
+
<= drop_target.winfo_rootx() + drop_target.winfo_width()
|
|
97
|
+
and drop_target.winfo_rooty()
|
|
98
|
+
<= pointer_y
|
|
99
|
+
<= drop_target.winfo_rooty() + drop_target.winfo_height()
|
|
100
|
+
):
|
|
101
|
+
set_status("DragAndDrop received")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
drag_source.bind("<ButtonRelease-1>", finish_drag)
|
|
105
|
+
|
|
106
|
+
scroll_target = action_target("Scroll here", 460, 475, "#42a5f5")
|
|
107
|
+
for scroll_event in ("<Button-4>", "<Button-5>", "<MouseWheel>"):
|
|
108
|
+
scroll_target.bind(scroll_event, lambda _event: set_status("Scroll received"))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def show_black_framebuffer():
|
|
112
|
+
for item in desktop_items:
|
|
113
|
+
canvas.itemconfigure(item, state="hidden")
|
|
114
|
+
canvas.configure(background="black")
|
|
115
|
+
root.focus_force()
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def restore_desktop(_event=None):
|
|
119
|
+
canvas.configure(background="#123456")
|
|
120
|
+
set_status("RDP E2E complete")
|
|
121
|
+
for item in desktop_items:
|
|
122
|
+
canvas.itemconfigure(item, state="normal")
|
|
123
|
+
entry.focus_force()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
black_button = tk.Button(
|
|
127
|
+
root,
|
|
128
|
+
text="Show valid black framebuffer",
|
|
129
|
+
command=show_black_framebuffer,
|
|
130
|
+
font=("DejaVu Sans", 18),
|
|
131
|
+
)
|
|
132
|
+
desktop_items.append(canvas.create_window(WIDTH // 2, 625, window=black_button))
|
|
133
|
+
root.bind_all("<Escape>", restore_desktop)
|
|
134
|
+
|
|
135
|
+
root.mainloop()
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.12.7
|
|
3
|
+
"version": "1.12.7",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -33,8 +33,8 @@
|
|
|
33
33
|
"@computer-use/libnut": "^4.2.0",
|
|
34
34
|
"clipboardy": "^4.0.0",
|
|
35
35
|
"screenshot-desktop": "^1.15.3",
|
|
36
|
-
"@midscene/
|
|
37
|
-
"@midscene/
|
|
36
|
+
"@midscene/shared": "1.12.7",
|
|
37
|
+
"@midscene/core": "1.12.7"
|
|
38
38
|
},
|
|
39
39
|
"optionalDependencies": {
|
|
40
40
|
"node-mac-permissions": "2.5.0"
|