litmus-cli 1.5.16-beta.1 → 1.5.16-screen.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/connect.d.ts.map +1 -1
- package/dist/commands/connect.js +6 -0
- package/dist/commands/connect.js.map +1 -1
- package/dist/commands/screen-capture.d.ts +20 -0
- package/dist/commands/screen-capture.d.ts.map +1 -0
- package/dist/commands/screen-capture.js +168 -0
- package/dist/commands/screen-capture.js.map +1 -0
- package/dist/lib/ai-network-hosts.d.ts.map +1 -1
- package/dist/lib/ai-network-hosts.js +5 -0
- package/dist/lib/ai-network-hosts.js.map +1 -1
- package/dist/lib/config.d.ts +2 -0
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js.map +1 -1
- package/dist/lib/credential-scrub.d.ts +30 -0
- package/dist/lib/credential-scrub.d.ts.map +1 -0
- package/dist/lib/credential-scrub.js +176 -0
- package/dist/lib/credential-scrub.js.map +1 -0
- package/dist/lib/hosted-watcher.js +21 -0
- package/dist/lib/hosted-watcher.js.map +1 -1
- package/dist/lib/laptop-capture.d.ts +259 -0
- package/dist/lib/laptop-capture.d.ts.map +1 -0
- package/dist/lib/laptop-capture.js +1347 -0
- package/dist/lib/laptop-capture.js.map +1 -0
- package/dist/lib/schemas.d.ts +1 -0
- package/dist/lib/schemas.d.ts.map +1 -1
- package/dist/lib/schemas.js +3 -0
- package/dist/lib/schemas.js.map +1 -1
- package/dist/native/windows/LitmusCapture.cs +1738 -0
- package/dist/program.js +19 -0
- package/dist/program.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,1738 @@
|
|
|
1
|
+
// ENG-3369 prototype: Litmus Capture for Windows.
|
|
2
|
+
//
|
|
3
|
+
// Compiled per user, with no admin, by the C# compiler every Windows install
|
|
4
|
+
// ships (.NET Framework 4.x), so this file is C# 5: no string interpolation,
|
|
5
|
+
// no `?.`, no expression-bodied members. The hosted watcher
|
|
6
|
+
// (cli/src/lib/laptop-capture.ts) starts it with `--pipe <name> --session <s>`
|
|
7
|
+
// and reads one JSON line per event from that named pipe. Nothing is written
|
|
8
|
+
// to disk here. Keystrokes come from Raw Input, never a low-level keyboard
|
|
9
|
+
// hook, so a slow moment here can never delay another app's typing.
|
|
10
|
+
using System;
|
|
11
|
+
using System.Collections.Generic;
|
|
12
|
+
using System.Diagnostics;
|
|
13
|
+
using System.Drawing;
|
|
14
|
+
using System.Drawing.Imaging;
|
|
15
|
+
using System.IO;
|
|
16
|
+
using System.IO.Pipes;
|
|
17
|
+
using System.Runtime.ExceptionServices;
|
|
18
|
+
using System.Runtime.InteropServices;
|
|
19
|
+
using System.Security;
|
|
20
|
+
using System.Security.Cryptography;
|
|
21
|
+
using System.Text;
|
|
22
|
+
using System.Text.RegularExpressions;
|
|
23
|
+
using System.Threading;
|
|
24
|
+
using System.Web.Script.Serialization;
|
|
25
|
+
using System.Windows.Automation;
|
|
26
|
+
using System.Windows.Automation.Text;
|
|
27
|
+
using System.Windows.Forms;
|
|
28
|
+
|
|
29
|
+
public static class Decisions
|
|
30
|
+
{
|
|
31
|
+
static readonly string[] Terminals = { "windowsterminal.exe", "conhost.exe", "openconsole.exe", "cmd.exe", "powershell.exe", "pwsh.exe", "wsl.exe", "mintty.exe" };
|
|
32
|
+
|
|
33
|
+
public static bool IsTerminal(string exe)
|
|
34
|
+
{
|
|
35
|
+
return exe != null && Array.IndexOf(Terminals, exe.ToLowerInvariant()) >= 0;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
public static bool LooksLikePasswordPrompt(string text)
|
|
39
|
+
{
|
|
40
|
+
if (string.IsNullOrEmpty(text)) return false;
|
|
41
|
+
string[] lines = text.Replace("\r", "").Split('\n');
|
|
42
|
+
for (int i = lines.Length - 1; i >= 0; i--)
|
|
43
|
+
{
|
|
44
|
+
if (lines[i].Trim().Length == 0) continue;
|
|
45
|
+
string line = lines[i].Length > 120 ? lines[i].Substring(lines[i].Length - 120) : lines[i];
|
|
46
|
+
return Regex.IsMatch(line, @"(password|passphrase|passcode|pin)[^:\n]{0,60}:\s*$", RegexOptions.IgnoreCase);
|
|
47
|
+
}
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/// Typed text is never kept in a password field or at a terminal password prompt.
|
|
52
|
+
/// A terminal whose text could not be read (null) counts as a prompt: fail closed.
|
|
53
|
+
public static bool ShouldSkipKeystrokes(bool isPassword, string exe, string terminalText)
|
|
54
|
+
{
|
|
55
|
+
if (isPassword) return true;
|
|
56
|
+
return IsTerminal(exe) && (terminalText == null || LooksLikePasswordPrompt(terminalText));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/// Where a long burst is cut into parts under the server's 4,000 character field
|
|
60
|
+
/// cap: after its last whitespace, so a key is never split between two parts the
|
|
61
|
+
/// scrub sees separately. -1 keeps typing: under 2,000 characters, or one unbroken
|
|
62
|
+
/// run under 3,900 (no key is that long).
|
|
63
|
+
public static int BurstCut(string text)
|
|
64
|
+
{
|
|
65
|
+
if (text.Length < 2000) return -1;
|
|
66
|
+
int cut = text.LastIndexOfAny(new[] { ' ', '\t', '\n' }) + 1;
|
|
67
|
+
return cut > 0 ? cut : text.Length >= 3900 ? text.Length : -1;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
public static string Host(string url)
|
|
71
|
+
{
|
|
72
|
+
if (string.IsNullOrEmpty(url)) return null;
|
|
73
|
+
Uri uri;
|
|
74
|
+
if (!Uri.TryCreate(url.Contains("://") ? url : "https://" + url, UriKind.Absolute, out uri)) return null;
|
|
75
|
+
string host = uri.Host.ToLowerInvariant().TrimEnd('.'); // "mail.google.com." is mail.google.com
|
|
76
|
+
return host.Length > 0 ? host : null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/// The same rule as the server's and the CLI's `excluded_category`.
|
|
80
|
+
public static string ExcludedCategory(Dictionary<string, Dictionary<string, List<string>>> rules, string appId, string url)
|
|
81
|
+
{
|
|
82
|
+
string app = appId == null ? "" : appId.ToLowerInvariant();
|
|
83
|
+
string host = Host(url);
|
|
84
|
+
List<string> categories = new List<string>(rules.Keys);
|
|
85
|
+
categories.Sort(StringComparer.Ordinal);
|
|
86
|
+
foreach (string category in categories)
|
|
87
|
+
{
|
|
88
|
+
Dictionary<string, List<string>> rule = rules[category];
|
|
89
|
+
if (app.Length > 0 && rule.ContainsKey("apps") && rule["apps"].Contains(app)) return category;
|
|
90
|
+
if (host != null && rule.ContainsKey("hosts"))
|
|
91
|
+
foreach (string listed in rule["hosts"])
|
|
92
|
+
if (host == listed || host.EndsWith("." + listed)) return category;
|
|
93
|
+
}
|
|
94
|
+
if (host != null)
|
|
95
|
+
{
|
|
96
|
+
string[] labels = host.Split('.');
|
|
97
|
+
for (int i = 0; i < labels.Length - 1; i++) if (labels[i].Contains("bank")) return "banking";
|
|
98
|
+
}
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/// A page known only by its window title: excluded when the title names an excluded
|
|
103
|
+
/// service as whole words ("Slack | general"), or any word says bank, as hosts do.
|
|
104
|
+
public static string TitleCategory(Dictionary<string, Dictionary<string, List<string>>> rules, string title)
|
|
105
|
+
{
|
|
106
|
+
string padded = " " + Words(title) + " ";
|
|
107
|
+
List<string> categories = new List<string>(rules.Keys);
|
|
108
|
+
categories.Sort(StringComparer.Ordinal);
|
|
109
|
+
foreach (string category in categories)
|
|
110
|
+
{
|
|
111
|
+
List<string> titles;
|
|
112
|
+
if (!rules[category].TryGetValue("titles", out titles)) continue;
|
|
113
|
+
foreach (string name in titles) if (Words(name).Length > 0 && padded.Contains(" " + Words(name) + " ")) return category;
|
|
114
|
+
}
|
|
115
|
+
foreach (string word in padded.Split(' ')) if (word.Contains("bank")) return "banking";
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
static string Words(string text)
|
|
120
|
+
{
|
|
121
|
+
StringBuilder words = new StringBuilder();
|
|
122
|
+
foreach (char c in (text ?? "").ToLowerInvariant()) words.Append(char.IsLetterOrDigit(c) ? c : ' ');
|
|
123
|
+
return string.Join(" ", words.ToString().Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries));
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/// Why a window behind the front one keeps nothing, if it doesn't. A browser window is
|
|
127
|
+
/// judged by its own page, as the front one is: its address. Without one a title can name an
|
|
128
|
+
/// excluded page but never clear one ("Inbox" may be a bank), so the page stays blanked.
|
|
129
|
+
public static string BackgroundExclusion(Dictionary<string, Dictionary<string, List<string>>> rules, string appId, string url, string title, bool browser)
|
|
130
|
+
{
|
|
131
|
+
string category = ExcludedCategory(rules, appId, url);
|
|
132
|
+
if (category != null || !browser || url != null) return category;
|
|
133
|
+
return (string.IsNullOrEmpty(title) ? null : TitleCategory(rules, title)) ?? "unverified";
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/// Why a window keeps nothing, if it doesn't: an excluded app or site, or a browser
|
|
137
|
+
/// whose address is not known yet and so could be showing anything (a bank's page).
|
|
138
|
+
public static string FrameExclusion(Dictionary<string, Dictionary<string, List<string>>> rules, string appId, string url, bool browser)
|
|
139
|
+
{
|
|
140
|
+
string category = ExcludedCategory(rules, appId, url);
|
|
141
|
+
if (category != null) return category;
|
|
142
|
+
return browser && url == null ? "unverified" : null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/// The address bar names the shown page only while nobody is typing in it: text
|
|
146
|
+
/// being typed names a page not yet shown, so it reads as unknown (keeps nothing).
|
|
147
|
+
public static string AddressFromBar(string value, bool editing)
|
|
148
|
+
{
|
|
149
|
+
return editing || string.IsNullOrEmpty(value) ? null : value;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// The patterns of cli/src/lib/credential-scrub.ts (SECRET_VALUE_PATTERNS) and of
|
|
153
|
+
// laptop-capture.ts (PARTIAL_KEY_RE, NAMED_SECRET_RE, CODE_VALUE_RE), character for
|
|
154
|
+
// character (a verbatim string doubles each "); cli/tests/unit/laptop-capture.test.ts
|
|
155
|
+
// fails when they drift. The watcher checks the text again; these find which window to blank.
|
|
156
|
+
public static readonly string[] SecretValuePatterns = {
|
|
157
|
+
@"l(?:wt|gt|rt|sg)_(?:wt\.)?[A-Za-z0-9_-]{16,}",
|
|
158
|
+
@"sk-ant-[A-Za-z0-9_-]{16,}",
|
|
159
|
+
@"sk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{24,}",
|
|
160
|
+
@"sk-[A-Za-z0-9]{24,}",
|
|
161
|
+
@"(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{24,}",
|
|
162
|
+
@"Bearer\s+[A-Za-z0-9._~+/-]{16,}=*",
|
|
163
|
+
@"ey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{6,}",
|
|
164
|
+
@"gh[pousr]_[A-Za-z0-9]{20,}",
|
|
165
|
+
@"xox[baprs]-[A-Za-z0-9-]{10,}",
|
|
166
|
+
@"(?:AKIA|ASIA)[0-9A-Z]{16}",
|
|
167
|
+
@"AIza[0-9A-Za-z_-]{35}",
|
|
168
|
+
@"(?<=:\/\/)(?<uu_user>[^\s:/@]+):(?<uu_pass>[^\s:/@]{3,})(?=@)",
|
|
169
|
+
};
|
|
170
|
+
public const string PartialKeyPattern = @"\b(?:sk-(?:ant-|proj-|svcacct-|admin-)?|l(?:wt|gt|rt|sg)_|gh[pousr]_|xox[baprs]-|AKIA|ASIA|AIza)[A-Za-z0-9_-]{8,}";
|
|
171
|
+
public const string NamedSecretPattern = @"(\b([A-Za-z_]*(?:API_?KEY|TOKEN|SECRET|PASSWORD|PASSWD))(\s*)([=:])\s*)([""']?)([^\s""'`;,()]{12,})\5";
|
|
172
|
+
public const string CodeValuePattern = @"^(?:[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+|[a-z][a-z0-9]*[A-Z][A-Za-z0-9]*|[a-z]+(?:_[a-z0-9]+)+)$";
|
|
173
|
+
static readonly Regex SecretRegex = new Regex("(?:" + string.Join(")|(?:", SecretValuePatterns) + ")|(?:" + PartialKeyPattern + ")", RegexOptions.IgnoreCase);
|
|
174
|
+
static readonly Regex NamedRegex = new Regex(NamedSecretPattern, RegexOptions.IgnoreCase);
|
|
175
|
+
static readonly Regex CodeRegex = new Regex(CodeValuePattern);
|
|
176
|
+
|
|
177
|
+
public delegate string WindowReader(int index, int budgetMs, out bool complete);
|
|
178
|
+
|
|
179
|
+
/// The text of a frame's windows under one deadline they share, the front window first:
|
|
180
|
+
/// text with no credential is returned, a window showing one is added to `credential`, and
|
|
181
|
+
/// any other window read only in part to `unread`, to be blanked. Null when the front window
|
|
182
|
+
/// (index `front`, or -1 when it is not on this display) could not be read in full in time;
|
|
183
|
+
/// `failed` is then its index.
|
|
184
|
+
/// ENG-4731: one unread window used to withhold the whole frame. On a loaded machine some
|
|
185
|
+
/// window (the desktop, the taskbar, one of Paint's 28 popup windows) always ran out of time,
|
|
186
|
+
/// so every frame was withheld. Only the front window must now be read; any other is blanked
|
|
187
|
+
/// when it can't be, so its pixels never leave unchecked. Each gets at least 200 ms and at
|
|
188
|
+
/// most half of what is left, so the windows after it keep time. `clean`: each window
|
|
189
|
+
/// in the text with what it showed, in read order, for Agreed.
|
|
190
|
+
public static string ReadWindows(int count, int front, WindowReader read, IList<string> known, int budgetMs, List<int> credential, List<int> unread, List<KeyValuePair<int, string>> clean, out int failed)
|
|
191
|
+
{
|
|
192
|
+
DateTime deadline = DateTime.UtcNow.AddMilliseconds(budgetMs);
|
|
193
|
+
StringBuilder all = new StringBuilder();
|
|
194
|
+
failed = -1;
|
|
195
|
+
List<int> order = new List<int>();
|
|
196
|
+
if (front >= 0) order.Add(front);
|
|
197
|
+
for (int i = 0; i < count; i++) if (i != front) order.Add(i);
|
|
198
|
+
foreach (int i in order)
|
|
199
|
+
{
|
|
200
|
+
int left = (int)(deadline - DateTime.UtcNow).TotalMilliseconds;
|
|
201
|
+
if (i != front) left = Math.Min(left, Math.Max(200, left / 2));
|
|
202
|
+
bool complete = false;
|
|
203
|
+
string shown = left > 0 ? read(i, left, out complete) : null;
|
|
204
|
+
if (!complete && i != front) { unread.Add(i); continue; }
|
|
205
|
+
if (!complete) { failed = i; return null; }
|
|
206
|
+
if (HasCredential(shown, known)) { credential.Add(i); continue; }
|
|
207
|
+
all.AppendLine(shown);
|
|
208
|
+
clean.Add(new KeyValuePair<int, string>(i, shown));
|
|
209
|
+
}
|
|
210
|
+
return all.ToString();
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/// The text a frame keeps from its two reads, which come before and after its second grab
|
|
214
|
+
/// of pixels: a window that showed the same text both times, front window first. A window
|
|
215
|
+
/// whose text changed in between is added to `changed`, to be blanked; one missing from
|
|
216
|
+
/// `second` was blanked by that read. Null when the front window changed: the frame is
|
|
217
|
+
/// taken again on the next tick.
|
|
218
|
+
/// ENG-4731: the reads used to have to agree as a whole, so on a slow machine, where which
|
|
219
|
+
/// windows run out of time differs from read to read, nearly every frame was dropped.
|
|
220
|
+
public static string Agreed(List<KeyValuePair<long, string>> first, Dictionary<long, string> second, long front, List<long> changed)
|
|
221
|
+
{
|
|
222
|
+
StringBuilder all = new StringBuilder();
|
|
223
|
+
foreach (KeyValuePair<long, string> window in first)
|
|
224
|
+
{
|
|
225
|
+
string again;
|
|
226
|
+
if (!second.TryGetValue(window.Key, out again)) { if (window.Key == front) return null; continue; }
|
|
227
|
+
if (again != window.Value) { if (window.Key == front) return null; changed.Add(window.Key); continue; }
|
|
228
|
+
all.AppendLine(window.Value);
|
|
229
|
+
}
|
|
230
|
+
return all.ToString();
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/// Whether a window's text shows a credential, as the watcher's scrub finds one: a
|
|
234
|
+
/// known value, a key shape or part of one, or a literal given to a credential-named
|
|
235
|
+
/// variable (not code like `token = response.data.token`).
|
|
236
|
+
public static bool HasCredential(string text, IList<string> known)
|
|
237
|
+
{
|
|
238
|
+
if (string.IsNullOrEmpty(text)) return false;
|
|
239
|
+
foreach (string secret in known) if (!string.IsNullOrEmpty(secret) && text.Contains(secret)) return true;
|
|
240
|
+
if (SecretRegex.IsMatch(text)) return true;
|
|
241
|
+
foreach (Match m in NamedRegex.Matches(text))
|
|
242
|
+
{
|
|
243
|
+
bool environment = Regex.IsMatch(m.Groups[2].Value, "^[A-Z0-9_]+$") && m.Groups[4].Value == "=" && m.Groups[3].Value.Length == 0;
|
|
244
|
+
if (m.Groups[5].Value.Length > 0 || environment || !CodeRegex.IsMatch(m.Groups[6].Value)) return true;
|
|
245
|
+
}
|
|
246
|
+
return false;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/// The server keeps at most this many boxes per frame (backend/services/laptop_capture.py).
|
|
250
|
+
public const int MaxRedactions = 50;
|
|
251
|
+
|
|
252
|
+
/// One blanked area as the server keeps it, in frame pixels: the part of `rect`
|
|
253
|
+
/// (screen pixels) on `display`, scaled to the frame. Null when it is off the display.
|
|
254
|
+
static readonly string[] ShellClasses = { "Progman", "WorkerW", "Shell_TrayWnd", "Shell_SecondaryTrayWnd", "XamlExplorerHostIslandWindow",
|
|
255
|
+
"Windows.UI.Core.CoreWindow", "ApplicationFrameWindow", "MultitaskingViewFrame", "ForegroundStaging", "TaskListThumbnailWnd",
|
|
256
|
+
"NotifyIconOverflowWindow", "TopLevelWindowForOverflowXamlIsland" };
|
|
257
|
+
|
|
258
|
+
/// ENG-4737: whether the front window hides what is under it, so a window behind it need
|
|
259
|
+
/// not be filled there. Only a window that can't be see-through counts: not layered or
|
|
260
|
+
/// transparent, not drawn through DirectComposition (Windows Terminal, which can be
|
|
261
|
+
/// translucent), no acrylic backdrop, and not a shell host window (XamlExplorerHostIslandWindow
|
|
262
|
+
/// spans the screen unlayered). Anything else leaves the window behind filled whole.
|
|
263
|
+
public static bool CountsAsCover(int exStyle, string windowClass, int backdrop)
|
|
264
|
+
{
|
|
265
|
+
const int Layered = 0x80000, Transparent = 0x20, NoRedirectionBitmap = 0x200000;
|
|
266
|
+
if ((exStyle & (Layered | Transparent | NoRedirectionBitmap)) != 0) return false;
|
|
267
|
+
if (backdrop == 3) return false; // DWMSBT_TRANSIENTWINDOW: acrylic, shows what is behind it blurred
|
|
268
|
+
return Array.IndexOf(ShellClasses, windowClass ?? "") < 0;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/// The part of a window rect the window surely paints: GetWindowRect includes the invisible
|
|
272
|
+
/// resize border (7 px a side at 100%, more when scaled) and Windows 11 rounds the corners.
|
|
273
|
+
public static Rectangle CoverOf(Rectangle windowRect)
|
|
274
|
+
{
|
|
275
|
+
Rectangle inner = windowRect;
|
|
276
|
+
inner.Inflate(-16, -16);
|
|
277
|
+
return inner.Width > 0 && inner.Height > 0 ? inner : Rectangle.Empty;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/// The parts of `window` left to fill once `cover` (the front window, in front of it) is
|
|
281
|
+
/// taken out: up to four rectangles, or the whole window when they don't overlap.
|
|
282
|
+
public static List<Rectangle> Uncovered(Rectangle window, Rectangle cover)
|
|
283
|
+
{
|
|
284
|
+
List<Rectangle> parts = new List<Rectangle>();
|
|
285
|
+
Rectangle hit = Rectangle.Intersect(window, cover);
|
|
286
|
+
if (hit.Width <= 0 || hit.Height <= 0) { parts.Add(window); return parts; }
|
|
287
|
+
if (hit.Top > window.Top) parts.Add(Rectangle.FromLTRB(window.Left, window.Top, window.Right, hit.Top));
|
|
288
|
+
if (hit.Bottom < window.Bottom) parts.Add(Rectangle.FromLTRB(window.Left, hit.Bottom, window.Right, window.Bottom));
|
|
289
|
+
if (hit.Left > window.Left) parts.Add(Rectangle.FromLTRB(window.Left, hit.Top, hit.Left, hit.Bottom));
|
|
290
|
+
if (hit.Right < window.Right) parts.Add(Rectangle.FromLTRB(hit.Right, hit.Top, window.Right, hit.Bottom));
|
|
291
|
+
return parts;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
public static Dictionary<string, object> Redaction(Rectangle rect, Rectangle display, double scale, string reason, string category)
|
|
295
|
+
{
|
|
296
|
+
Rectangle cut = Rectangle.Intersect(rect, display);
|
|
297
|
+
if (cut.Width <= 0 || cut.Height <= 0) return null;
|
|
298
|
+
cut.Offset(-display.X, -display.Y);
|
|
299
|
+
Dictionary<string, object> box = new Dictionary<string, object>();
|
|
300
|
+
box["x"] = (int)Math.Floor(cut.X * scale);
|
|
301
|
+
box["y"] = (int)Math.Floor(cut.Y * scale);
|
|
302
|
+
// Far edges rounded out on their own: the box encloses the scaled rectangle whole.
|
|
303
|
+
box["w"] = (int)Math.Ceiling(cut.Right * scale) - (int)box["x"];
|
|
304
|
+
box["h"] = (int)Math.Ceiling(cut.Bottom * scale) - (int)box["y"];
|
|
305
|
+
box["reason"] = reason;
|
|
306
|
+
if (category != null) box["category"] = category;
|
|
307
|
+
return box;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/// At most MaxRedactions boxes: past that, the boxes of one reason and category are
|
|
311
|
+
/// joined into the one rectangle around them. Joining only ever blanks more.
|
|
312
|
+
public static List<Dictionary<string, object>> CapRedactions(List<Dictionary<string, object>> boxes)
|
|
313
|
+
{
|
|
314
|
+
if (boxes.Count <= MaxRedactions) return boxes;
|
|
315
|
+
Dictionary<string, Dictionary<string, object>> joined = new Dictionary<string, Dictionary<string, object>>();
|
|
316
|
+
List<string> order = new List<string>();
|
|
317
|
+
foreach (Dictionary<string, object> box in boxes)
|
|
318
|
+
{
|
|
319
|
+
string key = (string)box["reason"] + "|" + (box.ContainsKey("category") ? (string)box["category"] : "");
|
|
320
|
+
Dictionary<string, object> held;
|
|
321
|
+
if (!joined.TryGetValue(key, out held)) { joined[key] = new Dictionary<string, object>(box); order.Add(key); continue; }
|
|
322
|
+
int right = Math.Max((int)held["x"] + (int)held["w"], (int)box["x"] + (int)box["w"]);
|
|
323
|
+
int bottom = Math.Max((int)held["y"] + (int)held["h"], (int)box["y"] + (int)box["h"]);
|
|
324
|
+
held["x"] = Math.Min((int)held["x"], (int)box["x"]);
|
|
325
|
+
held["y"] = Math.Min((int)held["y"], (int)box["y"]);
|
|
326
|
+
held["w"] = right - (int)held["x"];
|
|
327
|
+
held["h"] = bottom - (int)held["y"];
|
|
328
|
+
}
|
|
329
|
+
return order.ConvertAll<Dictionary<string, object>>(delegate (string key) { return joined[key]; });
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
public enum FrameAction { Skip, Keep, RepeatLast }
|
|
333
|
+
|
|
334
|
+
/// The screenshot cadence, as on macOS and Linux, kept per display so a busy one never
|
|
335
|
+
/// takes another's turn: a display is looked at every 5 s; a look that finds it changed
|
|
336
|
+
/// keeps a new frame, and one that finds it static sends the last frame again once 20 s
|
|
337
|
+
/// have passed.
|
|
338
|
+
public sealed class FrameCadence
|
|
339
|
+
{
|
|
340
|
+
public const double ChangingSeconds = 5, StaticSeconds = 20;
|
|
341
|
+
// The checks run on a 1 s timer; a look a few hundred ms early still counts.
|
|
342
|
+
public const double Slack = 0.25;
|
|
343
|
+
readonly Dictionary<string, DateTime> lastLook = new Dictionary<string, DateTime>();
|
|
344
|
+
readonly Dictionary<string, DateTime> lastSent = new Dictionary<string, DateTime>();
|
|
345
|
+
public bool Due(string display, DateTime now)
|
|
346
|
+
{
|
|
347
|
+
DateTime look;
|
|
348
|
+
return !lastLook.TryGetValue(display, out look) || (now - look).TotalSeconds >= ChangingSeconds - Slack;
|
|
349
|
+
}
|
|
350
|
+
public FrameAction Decide(string display, bool changed, DateTime now)
|
|
351
|
+
{
|
|
352
|
+
lastLook[display] = now;
|
|
353
|
+
DateTime sent;
|
|
354
|
+
if (!changed && lastSent.TryGetValue(display, out sent) && (now - sent).TotalSeconds < StaticSeconds - Slack) return FrameAction.Skip;
|
|
355
|
+
lastSent[display] = now;
|
|
356
|
+
return changed ? FrameAction.Keep : FrameAction.RepeatLast;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/// Says an exclusion once when it starts or changes, not on every frame tick.
|
|
361
|
+
public sealed class ExclusionNotice
|
|
362
|
+
{
|
|
363
|
+
string last;
|
|
364
|
+
public bool ShouldEmit(string category)
|
|
365
|
+
{
|
|
366
|
+
lock (this)
|
|
367
|
+
{
|
|
368
|
+
bool emit = category != null && category != last;
|
|
369
|
+
last = category;
|
|
370
|
+
return emit;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/// Whether a frame may be kept: the window in front, and its title, are still the
|
|
376
|
+
/// ones the exclusion check saw. A tab that moved on changes the title.
|
|
377
|
+
public static bool FrontStillMatches(IntPtr seenHwnd, string seenTitle, IntPtr nowHwnd, string nowTitle)
|
|
378
|
+
{
|
|
379
|
+
return seenHwnd == nowHwnd && (seenTitle ?? "") == (nowTitle ?? "");
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/// A flag when this desktop starts being used from another computer: a Remote
|
|
383
|
+
/// Desktop session, or a console being remote-controlled (shadowed). Null otherwise.
|
|
384
|
+
public static string RemoteSessionFlag(bool wasRemote, bool remoteSession, bool remoteControl)
|
|
385
|
+
{
|
|
386
|
+
if (wasRemote || !(remoteSession || remoteControl)) return null;
|
|
387
|
+
return remoteSession ? "This Windows session is being used over Remote Desktop." : "This Windows session is being remote-controlled.";
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
public static int SelfTest()
|
|
391
|
+
{
|
|
392
|
+
List<string> failures = new List<string>();
|
|
393
|
+
Action<bool, string> check = (ok, name) => { if (!ok) failures.Add(name); };
|
|
394
|
+
check(ShouldSkipKeystrokes(true, "chrome.exe", null), "browser password field");
|
|
395
|
+
check(ShouldSkipKeystrokes(true, "credentialuibroker.exe", null), "native credential dialog");
|
|
396
|
+
check(ShouldSkipKeystrokes(false, "WindowsTerminal.exe", "PS C:\\> ssh host\r\nuser@host's password: "), "ssh password prompt");
|
|
397
|
+
check(ShouldSkipKeystrokes(false, "cmd.exe", "C:\\>runas /user:admin cmd\r\nEnter the password for admin:"), "runas prompt");
|
|
398
|
+
check(!ShouldSkipKeystrokes(false, "WindowsTerminal.exe", "PS C:\\> echo password is in the vault\r\nok\r\nPS C:\\> "), "ordinary terminal line");
|
|
399
|
+
check(!ShouldSkipKeystrokes(false, "code.exe", "Password:"), "not a terminal");
|
|
400
|
+
check(ShouldSkipKeystrokes(false, "cmd.exe", null), "unreadable terminal");
|
|
401
|
+
string words = new StringBuilder().Insert(0, "word ", 398).ToString();
|
|
402
|
+
check(BurstCut(words + "sk-proj-ABCDEFGHIJKLMNOPQRSTUVWX12") == words.Length, "long burst cut before a key");
|
|
403
|
+
check(BurstCut(new string('x', 2500)) == -1 && BurstCut(new string('x', 3900)) == 3900 && BurstCut("short") == -1, "unbroken and short bursts");
|
|
404
|
+
check(!ShouldSkipKeystrokes(false, "notepad.exe", null), "not a terminal, no text read");
|
|
405
|
+
Dictionary<string, Dictionary<string, List<string>>> rules = new Dictionary<string, Dictionary<string, List<string>>>();
|
|
406
|
+
Dictionary<string, List<string>> mail = new Dictionary<string, List<string>>();
|
|
407
|
+
mail["apps"] = new List<string> { "outlook.exe" };
|
|
408
|
+
mail["hosts"] = new List<string> { "mail.google.com" };
|
|
409
|
+
rules["mail"] = mail;
|
|
410
|
+
check(ExcludedCategory(rules, "OUTLOOK.EXE", null) == "mail", "mail app");
|
|
411
|
+
check(ExcludedCategory(rules, "chrome.exe", "https://mail.google.com/mail/u/0") == "mail", "mail site");
|
|
412
|
+
check(ExcludedCategory(rules, "chrome.exe", "online.mybank.example/login") == "banking", "bank site");
|
|
413
|
+
check(ExcludedCategory(rules, "chrome.exe", "https://mail.google.com./mail/u/0") == "mail", "a trailing DNS dot is the same host");
|
|
414
|
+
check(ExcludedCategory(rules, "chrome.exe", "https://chatgpt.com/") == null, "ordinary site");
|
|
415
|
+
check(FrameExclusion(rules, "msedge.exe", null, true) == "unverified", "browser address not read yet");
|
|
416
|
+
// ENG-4737: a maximized browser behind a smaller editor is filled around the editor, not over it.
|
|
417
|
+
Rectangle browser = new Rectangle(-7, 0, 1934, 1047), editor = new Rectangle(400, 200, 900, 600);
|
|
418
|
+
Rectangle cover = CoverOf(editor);
|
|
419
|
+
List<Rectangle> around = Uncovered(browser, cover);
|
|
420
|
+
long area = 0;
|
|
421
|
+
foreach (Rectangle part in around) { area += (long)part.Width * part.Height; check(!part.IntersectsWith(cover), "no fill over the front window"); }
|
|
422
|
+
check(around.Count == 4 && area == (long)browser.Width * browser.Height - (long)cover.Width * cover.Height, "the rest of the browser still filled");
|
|
423
|
+
check(cover == new Rectangle(416, 216, 868, 568), "cover leaves the resize border and corners filled");
|
|
424
|
+
check(Uncovered(browser, Rectangle.Empty).Count == 1 && Uncovered(browser, Rectangle.Empty)[0] == browser, "no cover: filled whole");
|
|
425
|
+
check(Uncovered(new Rectangle(0, 0, 100, 100), new Rectangle(500, 500, 10, 10))[0] == new Rectangle(0, 0, 100, 100), "apart: filled whole");
|
|
426
|
+
check(CountsAsCover(0x100, "Notepad", 0) && CountsAsCover(0, "Chrome_WidgetWin_1", 1), "an opaque front window covers");
|
|
427
|
+
check(!CountsAsCover(0x80000, "ConsoleWindowClass", 0), "layered (console opacity) does not cover");
|
|
428
|
+
check(!CountsAsCover(0x200000, "CASCADIA_HOSTING_WINDOW_CLASS", 0), "DirectComposition (Windows Terminal) does not cover");
|
|
429
|
+
check(!CountsAsCover(0x20, "Notepad", 0) && !CountsAsCover(0, "Notepad", 3), "transparent or acrylic does not cover");
|
|
430
|
+
check(!CountsAsCover(0, "XamlExplorerHostIslandWindow", 0) && !CountsAsCover(0, "Shell_TrayWnd", 0), "shell windows do not cover");
|
|
431
|
+
check(FrameExclusion(rules, "msedge.exe", "https://www.examplebank.com/", true) == "banking", "bank page in a browser");
|
|
432
|
+
check(FrameExclusion(rules, "msedge.exe", "https://chatgpt.com/", true) == null && FrameExclusion(rules, "notepad.exe", null, false) == null, "ordinary page and app");
|
|
433
|
+
// A browser window behind the front one: sensitive, safe, or unknown.
|
|
434
|
+
Dictionary<string, List<string>> chat = new Dictionary<string, List<string>>();
|
|
435
|
+
chat["apps"] = new List<string> { "slack.exe" };
|
|
436
|
+
chat["hosts"] = new List<string> { "app.slack.com" };
|
|
437
|
+
chat["titles"] = new List<string> { "slack", "microsoft teams" };
|
|
438
|
+
rules["messaging"] = chat;
|
|
439
|
+
check(BackgroundExclusion(rules, "chrome.exe", "https://app.slack.com/client/T1", "Docs", true) == "messaging", "behind: chat site by address");
|
|
440
|
+
check(BackgroundExclusion(rules, "chrome.exe", "https://northwind-bank.localhost/login", null, true) == "banking", "behind: bank word in host");
|
|
441
|
+
check(BackgroundExclusion(rules, "chrome.exe", "https://docs.python.org/3/", "Slack", true) == null, "behind: a safe address wins over its title");
|
|
442
|
+
check(BackgroundExclusion(rules, "chrome.exe", null, "general - Acme - Slack - Google Chrome", true) == "messaging", "behind: chat site by title");
|
|
443
|
+
check(BackgroundExclusion(rules, "chrome.exe", null, "Microsoft Teams | Chat", true) == "messaging", "behind: two-word title");
|
|
444
|
+
check(BackgroundExclusion(rules, "chrome.exe", null, "Log on to Online Banking", true) == "banking", "behind: bank word in title");
|
|
445
|
+
check(TitleCategory(rules, "Slackline basics - Wikipedia") == null, "title words are whole words");
|
|
446
|
+
check(BackgroundExclusion(rules, "chrome.exe", null, "inventory.py - take-home", true) == "unverified" && BackgroundExclusion(rules, "chrome.exe", null, "Inbox", true) == "unverified", "behind: a title never clears a page");
|
|
447
|
+
check(BackgroundExclusion(rules, "chrome.exe", null, null, true) == "unverified" && BackgroundExclusion(rules, "chrome.exe", null, "", true) == "unverified", "behind: page unknown stays blanked");
|
|
448
|
+
check(BackgroundExclusion(rules, "notepad.exe", null, null, false) == null && BackgroundExclusion(rules, "slack.exe", null, null, false) == "messaging", "behind: ordinary and chat apps");
|
|
449
|
+
check(AddressFromBar("https://chatgpt.com/", true) == null && AddressFromBar("https://chatgpt.com/", false) == "https://chatgpt.com/", "address bar being typed in is unknown");
|
|
450
|
+
DateTime t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
|
451
|
+
// Display "1" changes on every look, "2" never: over 2 minutes of 1 s ticks the busy
|
|
452
|
+
// one sends a frame every 5 s and the static one every 20 s, neither starved.
|
|
453
|
+
FrameCadence cadence = new FrameCadence();
|
|
454
|
+
List<FrameAction> busy = new List<FrameAction>(), still = new List<FrameAction>();
|
|
455
|
+
for (int second = 0; second < 120; second++)
|
|
456
|
+
{
|
|
457
|
+
DateTime at = t0.AddSeconds(second);
|
|
458
|
+
if (cadence.Due("1", at)) { FrameAction a = cadence.Decide("1", true, at); if (a != FrameAction.Skip) busy.Add(a); }
|
|
459
|
+
if (cadence.Due("2", at)) { FrameAction a = cadence.Decide("2", second == 0, at); if (a != FrameAction.Skip) still.Add(a); }
|
|
460
|
+
}
|
|
461
|
+
check(busy.Count == 24 && busy.TrueForAll(delegate (FrameAction a) { return a == FrameAction.Keep; }), "changing display: a frame every 5 s");
|
|
462
|
+
check(still.Count == 6 && still[0] == FrameAction.Keep && still.FindAll(delegate (FrameAction a) { return a == FrameAction.RepeatLast; }).Count == 5, "static display: the last frame again every 20 s");
|
|
463
|
+
check(cadence.Due("1", t0.AddSeconds(120)) && !cadence.Due("1", t0.AddSeconds(117)), "looked at every 5 s");
|
|
464
|
+
ExclusionNotice notice = new ExclusionNotice();
|
|
465
|
+
string said = "";
|
|
466
|
+
foreach (string category in new string[] { "unverified", "unverified", "unverified", null, "unverified", "mail" }) said += notice.ShouldEmit(category) ? "1" : "0";
|
|
467
|
+
check(said == "100011", "one excluded row per change");
|
|
468
|
+
IntPtr one = new IntPtr(1), two = new IntPtr(2);
|
|
469
|
+
check(FrontStillMatches(one, "News - Edge", one, "News - Edge"), "same window kept");
|
|
470
|
+
check(!FrontStillMatches(one, "News - Edge", one, "Example Bank - Edge") && !FrontStillMatches(one, "News - Edge", two, "News - Edge"), "window moved on during capture");
|
|
471
|
+
check(RemoteSessionFlag(false, true, false) == "This Windows session is being used over Remote Desktop.", "Remote Desktop session");
|
|
472
|
+
check(RemoteSessionFlag(false, false, true) == "This Windows session is being remote-controlled.", "remote-controlled console");
|
|
473
|
+
check(RemoteSessionFlag(true, true, false) == null, "remote session flagged once");
|
|
474
|
+
check(RemoteSessionFlag(false, false, false) == null, "local console");
|
|
475
|
+
string[] none = new string[0];
|
|
476
|
+
foreach (string line in new[] { "OPENAI_API_KEY=sk-proj-FAKEKEYFORSELFTEST0123456789", "export TOKEN=lwt_abcdefghijklmnopqrstu", "password: \"hunter2hunter2hunter2\"",
|
|
477
|
+
"psql postgres://app:s3cretpw@db.example.com/x", "AKIAABCDEFGHIJKLMNOP" }) // gitleaks:allow
|
|
478
|
+
check(HasCredential(line, none), "credential in " + line);
|
|
479
|
+
foreach (string line in new[] { "token = getToken()", "max_tokens: 4096", "token = response.data.token", "const accessToken = authorizationHeaderValue", "risk-assessment-framework", "" })
|
|
480
|
+
check(!HasCredential(line, none), "no credential in " + line);
|
|
481
|
+
check(HasCredential("litmus init abc123token456", new[] { "abc123token456" }), "known value");
|
|
482
|
+
Dictionary<string, object> box = Redaction(new Rectangle(2020, 50, 268, 2000), new Rectangle(1920, 0, 1920, 1080), 0.75, "excluded_app", "mail");
|
|
483
|
+
check(box != null && (int)box["x"] == 75 && (int)box["y"] == 37 && (int)box["w"] == 201 && (int)box["h"] == 773 && (string)box["category"] == "mail", "box clipped to the display, in frame pixels");
|
|
484
|
+
check(Redaction(new Rectangle(0, 0, 100, 100), new Rectangle(1920, 0, 1920, 1080), 1, "credential", null) == null, "box on another display");
|
|
485
|
+
Dictionary<string, object> odd = Redaction(new Rectangle(101, 51, 201, 100), new Rectangle(0, 0, 1920, 1080), 0.75, "credential", null);
|
|
486
|
+
check((int)odd["x"] == 75 && (int)odd["y"] == 38 && (int)odd["w"] == 152 && (int)odd["h"] == 76, "a scaled box encloses the scaled rectangle");
|
|
487
|
+
List<Dictionary<string, object>> many = new List<Dictionary<string, object>>();
|
|
488
|
+
for (int i = 0; i < 60; i++) many.Add(Redaction(new Rectangle(i * 10, i * 5, 8, 4), new Rectangle(0, 0, 1920, 1080), 1, i % 2 == 0 ? "credential" : "excluded_app", i % 2 == 0 ? null : "mail"));
|
|
489
|
+
List<Dictionary<string, object>> capped = CapRedactions(many);
|
|
490
|
+
check(capped.Count == 2 && (int)capped[0]["x"] == 0 && (int)capped[0]["w"] == 588 && (int)capped[1]["h"] == 294, "60 boxes joined by reason");
|
|
491
|
+
check(CapRedactions(many.GetRange(0, 50)).Count == 50, "50 boxes kept as they are");
|
|
492
|
+
// ENG-4731: windows that each need more than 200 ms are read whole while the frame's budget lasts.
|
|
493
|
+
int[] costs = { 250, 300, 5 };
|
|
494
|
+
WindowReader slow = delegate (int i, int budget, out bool complete) { complete = budget >= costs[i]; Thread.Sleep(Math.Min(budget, costs[i])); return i == 2 ? "OPENAI_API_KEY=sk-proj-" + new string('a', 40) : "window " + i; };
|
|
495
|
+
List<int> keys = new List<int>(), parts = new List<int>();
|
|
496
|
+
int stuck;
|
|
497
|
+
string read = ReadWindows(3, 1, slow, new List<string>(), 1100, keys, parts, new List<KeyValuePair<int, string>>(), out stuck);
|
|
498
|
+
check(read != null && read.Contains("window 0") && read.Contains("window 1") && keys.Count == 1 && keys[0] == 2 && parts.Count == 0 && stuck == -1, "slow windows read whole under one budget");
|
|
499
|
+
costs = new int[] { 700, 1200, 5 };
|
|
500
|
+
keys.Clear();
|
|
501
|
+
check(ReadWindows(3, 1, slow, new List<string>(), 1100, keys, parts, new List<KeyValuePair<int, string>>(), out stuck) == null && stuck == 1, "a front window the budget cannot cover withholds, named");
|
|
502
|
+
costs = new int[] { 300, 300, 5 };
|
|
503
|
+
keys.Clear(); parts.Clear();
|
|
504
|
+
read = ReadWindows(3, 1, slow, new List<string>(), 1100, keys, parts, new List<KeyValuePair<int, string>>(), out stuck);
|
|
505
|
+
check(read != null && parts.Count == 0 && read.Contains("window 0") && read.Contains("window 1") && keys.Count == 1, "a window over 200 ms is read, not blanked");
|
|
506
|
+
costs = new int[] { 900, 300, 5 };
|
|
507
|
+
keys.Clear(); parts.Clear();
|
|
508
|
+
read = ReadWindows(3, 1, slow, new List<string>(), 1100, keys, parts, new List<KeyValuePair<int, string>>(), out stuck);
|
|
509
|
+
check(read != null && parts.Count == 1 && parts[0] == 0 && read.Contains("window 1") && !read.Contains("window 0") && keys.Count == 1, "a window behind or above the front gets half the budget, then is blanked");
|
|
510
|
+
// The front window is read first: windows ahead of it in z-order (Paint's popups, the taskbar) can't starve it.
|
|
511
|
+
costs = new int[] { 900, 600, 5 };
|
|
512
|
+
keys.Clear(); parts.Clear();
|
|
513
|
+
read = ReadWindows(3, 1, slow, new List<string>(), 1100, keys, parts, new List<KeyValuePair<int, string>>(), out stuck);
|
|
514
|
+
check(read != null && read.Contains("window 1") && parts.Count == 1 && parts[0] == 0 && keys.Count == 1 && keys[0] == 2, "the front window is read first and the one ahead of it blanked");
|
|
515
|
+
// A credential in an unread window can't reach the text: it is blanked, never returned.
|
|
516
|
+
costs = new int[] { 5, 5, 900 };
|
|
517
|
+
keys.Clear(); parts.Clear();
|
|
518
|
+
read = ReadWindows(3, 0, slow, new List<string>(), 400, keys, parts, new List<KeyValuePair<int, string>>(), out stuck);
|
|
519
|
+
check(read != null && !read.Contains("sk-proj-") && keys.Count == 0 && parts.Count == 1 && parts[0] == 2, "an unread window's text is not returned");
|
|
520
|
+
// No front window on this display: every window may be blanked, none withholds.
|
|
521
|
+
costs = new int[] { 900, 900, 900 };
|
|
522
|
+
keys.Clear(); parts.Clear();
|
|
523
|
+
check(ReadWindows(3, -1, slow, new List<string>(), 400, keys, parts, new List<KeyValuePair<int, string>>(), out stuck) != null && parts.Count == 3 && stuck == -1, "no front window: unread windows are blanked");
|
|
524
|
+
// ENG-4731: the two reads around the second grab agree window by window.
|
|
525
|
+
List<KeyValuePair<long, string>> before = new List<KeyValuePair<long, string>> { new KeyValuePair<long, string>(7, "editor"), new KeyValuePair<long, string>(8, "desktop"), new KeyValuePair<long, string>(9, "taskbar 10:41") };
|
|
526
|
+
Dictionary<long, string> after = new Dictionary<long, string> { { 7, "editor" }, { 8, "desktop" }, { 9, "taskbar 10:42" } };
|
|
527
|
+
List<long> moved = new List<long>();
|
|
528
|
+
string agreed = Agreed(before, after, 7, moved);
|
|
529
|
+
check(agreed != null && agreed.Contains("editor") && agreed.Contains("desktop") && !agreed.Contains("taskbar") && moved.Count == 1 && moved[0] == 9, "a window behind that changed is blanked, the frame kept");
|
|
530
|
+
after.Remove(8);
|
|
531
|
+
moved.Clear();
|
|
532
|
+
agreed = Agreed(before, after, 7, moved);
|
|
533
|
+
check(agreed != null && !agreed.Contains("desktop") && moved.Count == 1, "a window the second read blanked is left out");
|
|
534
|
+
after[7] = "editor, typed";
|
|
535
|
+
check(Agreed(before, after, 7, new List<long>()) == null, "the front window changed: taken again");
|
|
536
|
+
after.Remove(7);
|
|
537
|
+
check(Agreed(before, after, 7, new List<long>()) == null, "the front window missing from the second read: taken again");
|
|
538
|
+
if (failures.Count == 0) { Console.WriteLine("self-test ok"); return 0; }
|
|
539
|
+
Console.WriteLine("self-test failed: " + string.Join(", ", failures.ToArray()));
|
|
540
|
+
return 1;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
static class Native
|
|
545
|
+
{
|
|
546
|
+
public delegate void WinEventProc(IntPtr hook, uint evt, IntPtr hwnd, int idObject, int idChild, uint thread, uint time);
|
|
547
|
+
public delegate bool EnumProc(IntPtr hwnd, IntPtr lParam);
|
|
548
|
+
[DllImport("user32.dll")] public static extern IntPtr SetWinEventHook(uint min, uint max, IntPtr module, WinEventProc proc, uint pid, uint thread, uint flags);
|
|
549
|
+
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
|
550
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetClassName(IntPtr hwnd, StringBuilder name, int count);
|
|
551
|
+
[DllImport("kernel32.dll")] public static extern bool AttachConsole(uint pid);
|
|
552
|
+
[DllImport("kernel32.dll")] public static extern bool FreeConsole();
|
|
553
|
+
[DllImport("kernel32.dll")] public static extern bool SetConsoleCtrlHandler(IntPtr handler, bool add);
|
|
554
|
+
[DllImport("kernel32.dll", CharSet = CharSet.Unicode)] public static extern IntPtr CreateFile(string name, uint access, uint share, IntPtr security, uint creation, uint flags, IntPtr template);
|
|
555
|
+
[DllImport("kernel32.dll")] public static extern bool CloseHandle(IntPtr handle);
|
|
556
|
+
[DllImport("kernel32.dll")] public static extern bool GetConsoleScreenBufferInfo(IntPtr output, out CONSOLE_SCREEN_BUFFER_INFO info);
|
|
557
|
+
[DllImport("kernel32.dll", CharSet = CharSet.Unicode)] public static extern bool ReadConsoleOutputCharacter(IntPtr output, [Out] char[] text, uint length, COORD at, out uint read);
|
|
558
|
+
[DllImport("user32.dll")] public static extern int GetSystemMetrics(int index);
|
|
559
|
+
[DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint pid);
|
|
560
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowText(IntPtr hwnd, StringBuilder text, int count);
|
|
561
|
+
[DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc proc, IntPtr lParam);
|
|
562
|
+
[DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr hwnd);
|
|
563
|
+
[DllImport("user32.dll")] public static extern bool IsIconic(IntPtr hwnd);
|
|
564
|
+
[DllImport("dwmapi.dll")] public static extern int DwmGetWindowAttribute(IntPtr hwnd, int attribute, out int value, int size);
|
|
565
|
+
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hwnd, out RECT rect);
|
|
566
|
+
[DllImport("user32.dll")] public static extern int GetWindowLong(IntPtr hwnd, int index);
|
|
567
|
+
[DllImport("user32.dll")] public static extern int GetWindowRgnBox(IntPtr hwnd, out RECT rect);
|
|
568
|
+
[DllImport("user32.dll")] public static extern bool GetWindowDisplayAffinity(IntPtr hwnd, out uint affinity);
|
|
569
|
+
[DllImport("user32.dll")] public static extern bool RegisterRawInputDevices(RAWINPUTDEVICE[] devices, uint count, uint size);
|
|
570
|
+
[DllImport("user32.dll")] public static extern uint GetRawInputData(IntPtr raw, uint command, IntPtr data, ref uint size, uint headerSize);
|
|
571
|
+
[DllImport("user32.dll")] public static extern bool AddClipboardFormatListener(IntPtr hwnd);
|
|
572
|
+
[DllImport("user32.dll")] public static extern short GetKeyState(int key);
|
|
573
|
+
[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int key);
|
|
574
|
+
[DllImport("user32.dll")] public static extern IntPtr GetKeyboardLayout(uint thread);
|
|
575
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int ToUnicodeEx(uint key, uint scan, byte[] state, StringBuilder buffer, int size, uint flags, IntPtr layout);
|
|
576
|
+
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT point);
|
|
577
|
+
[DllImport("user32.dll")] public static extern uint RegisterClipboardFormat(string name);
|
|
578
|
+
[DllImport("user32.dll")] public static extern bool IsClipboardFormatAvailable(uint format);
|
|
579
|
+
[DllImport("winmm.dll")] public static extern int waveInOpen(out IntPtr handle, int device, ref WAVEFORMATEX format, IntPtr callback, IntPtr instance, int flags);
|
|
580
|
+
[DllImport("winmm.dll")] public static extern int waveInPrepareHeader(IntPtr handle, IntPtr header, int size);
|
|
581
|
+
[DllImport("winmm.dll")] public static extern int waveInUnprepareHeader(IntPtr handle, IntPtr header, int size);
|
|
582
|
+
[DllImport("winmm.dll")] public static extern int waveInAddBuffer(IntPtr handle, IntPtr header, int size);
|
|
583
|
+
[DllImport("winmm.dll")] public static extern int waveInStart(IntPtr handle);
|
|
584
|
+
[DllImport("winmm.dll")] public static extern int waveInReset(IntPtr handle);
|
|
585
|
+
[DllImport("winmm.dll")] public static extern int waveInClose(IntPtr handle);
|
|
586
|
+
|
|
587
|
+
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left, Top, Right, Bottom; }
|
|
588
|
+
[StructLayout(LayoutKind.Sequential)] public struct POINT { public int X, Y; }
|
|
589
|
+
[StructLayout(LayoutKind.Sequential)] public struct COORD { public short X, Y; }
|
|
590
|
+
[StructLayout(LayoutKind.Sequential)] public struct SMALL_RECT { public short Left, Top, Right, Bottom; }
|
|
591
|
+
[StructLayout(LayoutKind.Sequential)] public struct CONSOLE_SCREEN_BUFFER_INFO { public COORD Size, Cursor; public short Attributes; public SMALL_RECT Window; public COORD MaxWindow; }
|
|
592
|
+
[StructLayout(LayoutKind.Sequential)] public struct RAWINPUTDEVICE { public ushort UsagePage; public ushort Usage; public uint Flags; public IntPtr Target; }
|
|
593
|
+
[StructLayout(LayoutKind.Sequential)] public struct WAVEFORMATEX { public ushort wFormatTag, nChannels; public uint nSamplesPerSec, nAvgBytesPerSec; public ushort nBlockAlign, wBitsPerSample, cbSize; }
|
|
594
|
+
[StructLayout(LayoutKind.Sequential)] public struct WAVEHDR { public IntPtr lpData; public uint dwBufferLength, dwBytesRecorded; public IntPtr dwUser; public uint dwFlags, dwLoops; public IntPtr lpNext, reserved; }
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
class Capture : NativeWindow
|
|
598
|
+
{
|
|
599
|
+
const int WM_INPUT = 0x00FF, WM_CLIPBOARDUPDATE = 0x031D, MM_WIM_DATA = 0x3C0;
|
|
600
|
+
readonly NamedPipeClientStream pipe;
|
|
601
|
+
readonly object writeLock = new object();
|
|
602
|
+
readonly JavaScriptSerializer json = new JavaScriptSerializer { MaxJsonLength = int.MaxValue };
|
|
603
|
+
Dictionary<string, Dictionary<string, List<string>>> exclusions = new Dictionary<string, Dictionary<string, List<string>>>();
|
|
604
|
+
/// The values the watcher redacts by name (the assessment token), so a window showing one is blanked too.
|
|
605
|
+
List<string> known = new List<string>();
|
|
606
|
+
bool audioAllowed;
|
|
607
|
+
Native.WinEventProc foregroundProc;
|
|
608
|
+
System.Windows.Forms.Timer timer;
|
|
609
|
+
int tick;
|
|
610
|
+
|
|
611
|
+
// Front window
|
|
612
|
+
IntPtr frontHwnd = IntPtr.Zero;
|
|
613
|
+
string frontExe, frontName, frontTitle, frontUrl;
|
|
614
|
+
// Typing
|
|
615
|
+
StringBuilder burst = new StringBuilder();
|
|
616
|
+
bool burstSecure;
|
|
617
|
+
int verifiedLength; // how much of the burst was typed before the last field check that passed
|
|
618
|
+
string burstExe, burstName;
|
|
619
|
+
DateTime lastKey = DateTime.MinValue;
|
|
620
|
+
volatile bool focusIsPassword;
|
|
621
|
+
// Modifiers as Raw Input delivered them. Reading them later with
|
|
622
|
+
// GetAsyncKeyState sees the key already released (SendKeys, fast typists).
|
|
623
|
+
bool shiftDown, ctrlDown, altDown;
|
|
624
|
+
DateTime lastFieldCheck = DateTime.MinValue;
|
|
625
|
+
bool recheckField;
|
|
626
|
+
// Screens
|
|
627
|
+
Dictionary<string, byte[]> lastDigest = new Dictionary<string, byte[]>();
|
|
628
|
+
int displayCount;
|
|
629
|
+
HashSet<uint> flaggedHidden = new HashSet<uint>();
|
|
630
|
+
uint concealedFormat, excludeHistoryFormat;
|
|
631
|
+
// Audio
|
|
632
|
+
IntPtr wave = IntPtr.Zero;
|
|
633
|
+
List<IntPtr> waveHeaders = new List<IntPtr>();
|
|
634
|
+
MemoryStream audio = new MemoryStream();
|
|
635
|
+
bool micDenied;
|
|
636
|
+
bool started;
|
|
637
|
+
// Counts of what arrived, so a heartbeat says the sensor is working, not only alive.
|
|
638
|
+
int keyCount, clickCount, frameCount;
|
|
639
|
+
|
|
640
|
+
public Capture(NamedPipeClientStream pipe)
|
|
641
|
+
{
|
|
642
|
+
this.pipe = pipe;
|
|
643
|
+
CreateParams cp = new CreateParams();
|
|
644
|
+
cp.Parent = new IntPtr(-3); // HWND_MESSAGE: no window anyone sees
|
|
645
|
+
CreateHandle(cp);
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
public void Send(Dictionary<string, object> obj)
|
|
649
|
+
{
|
|
650
|
+
byte[] bytes = Encoding.UTF8.GetBytes(json.Serialize(obj) + "\n");
|
|
651
|
+
lock (writeLock)
|
|
652
|
+
{
|
|
653
|
+
try { pipe.Write(bytes, 0, bytes.Length); pipe.Flush(); }
|
|
654
|
+
catch (IOException) { }
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
static string Now() { return DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); }
|
|
659
|
+
|
|
660
|
+
void Emit(string kind, bool withApp, Dictionary<string, object> data)
|
|
661
|
+
{
|
|
662
|
+
Dictionary<string, object> e = new Dictionary<string, object>();
|
|
663
|
+
e["kind"] = kind; e["ts"] = Now(); e["data"] = data ?? new Dictionary<string, object>();
|
|
664
|
+
if (withApp && frontExe != null) { e["app"] = frontName ?? frontExe; e["app_id"] = frontExe; }
|
|
665
|
+
Send(e);
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
static Dictionary<string, object> D(params object[] pairs)
|
|
669
|
+
{
|
|
670
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
671
|
+
for (int i = 0; i + 1 < pairs.Length; i += 2) d[(string)pairs[i]] = pairs[i + 1];
|
|
672
|
+
return d;
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
/// Why a window keeps nothing, if it doesn't: an excluded app or site, or a browser
|
|
676
|
+
/// whose address could not be read and so could be showing anything.
|
|
677
|
+
string ExcludedFor(string exe)
|
|
678
|
+
{
|
|
679
|
+
return Decisions.FrameExclusion(exclusions, exe, frontUrl, Array.IndexOf(Browsers, exe) >= 0);
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
string CurrentExcluded() { return ExcludedFor(frontExe); }
|
|
683
|
+
|
|
684
|
+
public void Configure(Dictionary<string, object> message)
|
|
685
|
+
{
|
|
686
|
+
audioAllowed = message.ContainsKey("audio") && message["audio"] is bool && (bool)message["audio"];
|
|
687
|
+
Dictionary<string, Dictionary<string, List<string>>> rules = new Dictionary<string, Dictionary<string, List<string>>>();
|
|
688
|
+
object raw;
|
|
689
|
+
if (message.TryGetValue("exclusions", out raw) && raw is Dictionary<string, object>)
|
|
690
|
+
{
|
|
691
|
+
foreach (KeyValuePair<string, object> category in (Dictionary<string, object>)raw)
|
|
692
|
+
{
|
|
693
|
+
Dictionary<string, List<string>> rule = new Dictionary<string, List<string>>();
|
|
694
|
+
Dictionary<string, object> parts = category.Value as Dictionary<string, object>;
|
|
695
|
+
if (parts == null) continue;
|
|
696
|
+
foreach (string key in new[] { "apps", "hosts", "titles" })
|
|
697
|
+
{
|
|
698
|
+
List<string> values = new List<string>();
|
|
699
|
+
object list;
|
|
700
|
+
if (parts.TryGetValue(key, out list) && list is System.Collections.ArrayList)
|
|
701
|
+
foreach (object v in (System.Collections.ArrayList)list) values.Add(((string)v).ToLowerInvariant());
|
|
702
|
+
rule[key] = values;
|
|
703
|
+
}
|
|
704
|
+
rules[category.Key] = rule;
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
exclusions = rules;
|
|
708
|
+
// Named apart from the loop's `values` above: C# 5 refuses an outer local that a nested scope already declares.
|
|
709
|
+
List<string> secrets = new List<string>();
|
|
710
|
+
object knownRaw;
|
|
711
|
+
if (message.TryGetValue("known", out knownRaw) && knownRaw is System.Collections.ArrayList)
|
|
712
|
+
foreach (object item in (System.Collections.ArrayList)knownRaw) { string secret = item as string; if (!string.IsNullOrEmpty(secret)) secrets.Add(secret); }
|
|
713
|
+
known = secrets;
|
|
714
|
+
if (!started) { started = true; Start(); return; }
|
|
715
|
+
// A later config only changes the audio switch.
|
|
716
|
+
if (audioAllowed && wave == IntPtr.Zero) StartAudio();
|
|
717
|
+
if (!audioAllowed) StopAudio();
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
void Start()
|
|
721
|
+
{
|
|
722
|
+
Native.RAWINPUTDEVICE[] devices = new Native.RAWINPUTDEVICE[2];
|
|
723
|
+
devices[0].UsagePage = 1; devices[0].Usage = 6; devices[0].Flags = 0x100; devices[0].Target = Handle; // keyboard, RIDEV_INPUTSINK
|
|
724
|
+
devices[1].UsagePage = 1; devices[1].Usage = 2; devices[1].Flags = 0x100; devices[1].Target = Handle; // mouse
|
|
725
|
+
Native.RegisterRawInputDevices(devices, 2, (uint)Marshal.SizeOf(typeof(Native.RAWINPUTDEVICE)));
|
|
726
|
+
Native.AddClipboardFormatListener(Handle);
|
|
727
|
+
concealedFormat = Native.RegisterClipboardFormat("ExcludeClipboardContentFromMonitorProcessing");
|
|
728
|
+
excludeHistoryFormat = Native.RegisterClipboardFormat("CanIncludeInClipboardHistory");
|
|
729
|
+
foregroundProc = new Native.WinEventProc(OnForeground);
|
|
730
|
+
Native.SetWinEventHook(3, 3, IntPtr.Zero, foregroundProc, 0, 0, 0); // EVENT_SYSTEM_FOREGROUND, out of context
|
|
731
|
+
try
|
|
732
|
+
{
|
|
733
|
+
Automation.AddAutomationFocusChangedEventHandler(delegate (object sender, AutomationFocusChangedEventArgs e)
|
|
734
|
+
{
|
|
735
|
+
try
|
|
736
|
+
{
|
|
737
|
+
AutomationElement element = sender as AutomationElement;
|
|
738
|
+
bool secure = element != null && element.Current.IsPassword;
|
|
739
|
+
focusIsPassword = secure;
|
|
740
|
+
if (secure) burstSecure = true;
|
|
741
|
+
}
|
|
742
|
+
catch (Exception) { }
|
|
743
|
+
});
|
|
744
|
+
}
|
|
745
|
+
catch (Exception) { }
|
|
746
|
+
if (audioAllowed) StartAudio();
|
|
747
|
+
Status();
|
|
748
|
+
CheckDisplays();
|
|
749
|
+
CheckHiddenWindows();
|
|
750
|
+
CheckRemoteSession();
|
|
751
|
+
RefreshFront("start");
|
|
752
|
+
timer = new System.Windows.Forms.Timer();
|
|
753
|
+
timer.Interval = 500;
|
|
754
|
+
timer.Tick += delegate { OnTick(); };
|
|
755
|
+
timer.Start();
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
void OnTick()
|
|
759
|
+
{
|
|
760
|
+
tick++;
|
|
761
|
+
if (burst.Length > 0 && (DateTime.UtcNow - lastKey).TotalSeconds > 1.0) FlushBurst();
|
|
762
|
+
if (tick % 2 == 0) RefreshFront("poll");
|
|
763
|
+
if (tick % 10 == 0) { Status(); CheckDisplays(); CheckHiddenWindows(); CheckRemoteSession(); }
|
|
764
|
+
// Each display is looked at when its own 5 s comes round (FrameCadence); asking every
|
|
765
|
+
// second keeps a look an app switch took early from pushing the next one to 10 s.
|
|
766
|
+
if (tick % 2 == 1) CaptureScreens("visual check");
|
|
767
|
+
if (tick % 60 == 0 && wave != IntPtr.Zero) FlushAudio();
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
void Status()
|
|
771
|
+
{
|
|
772
|
+
Dictionary<string, object> perms = D("screen", "granted", "accessibility", "granted", "input", "granted");
|
|
773
|
+
if (audioAllowed) perms["microphone"] = micDenied ? "denied" : "granted";
|
|
774
|
+
Send(D("kind", "status", "data", D("permissions", perms, "displays", Screen.AllScreens.Length, "keys", keyCount, "clicks", clickCount, "frames", frameCount)));
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
// ── Front app, window, address ──────────────────────────────────────────
|
|
778
|
+
|
|
779
|
+
void OnForeground(IntPtr hook, uint evt, IntPtr hwnd, int idObject, int idChild, uint thread, uint time) { RefreshFront("app switch"); }
|
|
780
|
+
|
|
781
|
+
static readonly Dictionary<uint, string[]> processNames = new Dictionary<uint, string[]>();
|
|
782
|
+
|
|
783
|
+
static string ProcessExe(IntPtr hwnd, out string name)
|
|
784
|
+
{
|
|
785
|
+
name = null;
|
|
786
|
+
uint pid;
|
|
787
|
+
Native.GetWindowThreadProcessId(hwnd, out pid);
|
|
788
|
+
string[] known;
|
|
789
|
+
if (processNames.TryGetValue(pid, out known)) { name = known[1]; return known[0]; }
|
|
790
|
+
try
|
|
791
|
+
{
|
|
792
|
+
Process p = Process.GetProcessById((int)pid);
|
|
793
|
+
name = p.ProcessName;
|
|
794
|
+
try { string description = p.MainModule.FileVersionInfo.FileDescription; if (!string.IsNullOrEmpty(description)) name = description; } catch (Exception) { }
|
|
795
|
+
string exe = p.ProcessName.ToLowerInvariant() + ".exe";
|
|
796
|
+
lock (processNames) processNames[pid] = new[] { exe, name };
|
|
797
|
+
return exe;
|
|
798
|
+
}
|
|
799
|
+
catch (Exception) { return null; }
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
static string WindowTitle(IntPtr hwnd)
|
|
803
|
+
{
|
|
804
|
+
StringBuilder text = new StringBuilder(512);
|
|
805
|
+
Native.GetWindowText(hwnd, text, text.Capacity);
|
|
806
|
+
return text.ToString();
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
/// UI Automation calls run on a worker with a deadline, so a hung app
|
|
810
|
+
/// cannot stall this one.
|
|
811
|
+
static T Bounded<T>(Func<T> work, int ms, T fallback)
|
|
812
|
+
{
|
|
813
|
+
T result = fallback;
|
|
814
|
+
Thread worker = new Thread(delegate () { try { result = work(); } catch (Exception) { } });
|
|
815
|
+
worker.IsBackground = true;
|
|
816
|
+
worker.Start();
|
|
817
|
+
return worker.Join(ms) ? result : fallback;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
static readonly string[] Browsers = { "chrome.exe", "msedge.exe", "brave.exe", "firefox.exe", "opera.exe", "vivaldi.exe", "arc.exe" };
|
|
821
|
+
|
|
822
|
+
// The address bar found by a search, per window (the front one and those behind it).
|
|
823
|
+
// One bar serves every tab, so reading it again follows tab switches without a FindAll every second.
|
|
824
|
+
static readonly Dictionary<IntPtr, AutomationElement> addressBars = new Dictionary<IntPtr, AutomationElement>();
|
|
825
|
+
static readonly object addressLock = new object();
|
|
826
|
+
|
|
827
|
+
static string BrowserUrl(IntPtr hwnd)
|
|
828
|
+
{
|
|
829
|
+
// One read at a time: a read that outlived its timeout (Bounded) still owns the
|
|
830
|
+
// cache, so a call that finds it busy reads as unknown, which keeps nothing.
|
|
831
|
+
if (!Monitor.TryEnter(addressLock)) return null;
|
|
832
|
+
try { return BrowserUrlLocked(hwnd); }
|
|
833
|
+
finally { Monitor.Exit(addressLock); }
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
static string BrowserUrlLocked(IntPtr hwnd)
|
|
837
|
+
{
|
|
838
|
+
AutomationElement cached;
|
|
839
|
+
if (addressBars.TryGetValue(hwnd, out cached))
|
|
840
|
+
{
|
|
841
|
+
try { return ReadAddressBar(cached); }
|
|
842
|
+
catch (Exception) { addressBars.Remove(hwnd); } // gone or stale: search again
|
|
843
|
+
}
|
|
844
|
+
if (addressBars.Count > 16) addressBars.Clear();
|
|
845
|
+
AutomationElement window = AutomationElement.FromHandle(hwnd);
|
|
846
|
+
Condition edits = new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Edit);
|
|
847
|
+
foreach (AutomationElement edit in window.FindAll(TreeScope.Descendants, edits))
|
|
848
|
+
{
|
|
849
|
+
string label = edit.Current.Name ?? "";
|
|
850
|
+
if (label.IndexOf("address", StringComparison.OrdinalIgnoreCase) < 0 && label.IndexOf("search", StringComparison.OrdinalIgnoreCase) < 0) continue;
|
|
851
|
+
object pattern;
|
|
852
|
+
if (!edit.TryGetCurrentPattern(ValuePattern.Pattern, out pattern)) continue;
|
|
853
|
+
addressBars[hwnd] = edit;
|
|
854
|
+
return ReadAddressBar(edit);
|
|
855
|
+
}
|
|
856
|
+
return null;
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
static string ReadAddressBar(AutomationElement edit)
|
|
860
|
+
{
|
|
861
|
+
object pattern = edit.GetCurrentPattern(ValuePattern.Pattern);
|
|
862
|
+
return Decisions.AddressFromBar(((ValuePattern)pattern).Current.Value, edit.Current.HasKeyboardFocus);
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
void RefreshFront(string reason)
|
|
866
|
+
{
|
|
867
|
+
IntPtr hwnd = Native.GetForegroundWindow();
|
|
868
|
+
if (hwnd == IntPtr.Zero) return;
|
|
869
|
+
string name;
|
|
870
|
+
string exe = ProcessExe(hwnd, out name);
|
|
871
|
+
if (exe == null || exe == "litmuscapture.exe") return;
|
|
872
|
+
string title = WindowTitle(hwnd);
|
|
873
|
+
bool switched = hwnd != frontHwnd || exe != frontExe;
|
|
874
|
+
// A read that fails or times out is an unknown address, never the last one.
|
|
875
|
+
string url = Array.IndexOf(Browsers, exe) >= 0 ? Bounded(delegate () { return BrowserUrl(hwnd); }, 800, (string)null) : null;
|
|
876
|
+
if (!switched && title == frontTitle && url == frontUrl) return;
|
|
877
|
+
// A title that changes as you type (an unsaved marker) is not a new burst.
|
|
878
|
+
if (switched) FlushBurst();
|
|
879
|
+
frontHwnd = hwnd; frontExe = exe; frontName = name; frontTitle = title; frontUrl = url;
|
|
880
|
+
string category = CurrentExcluded();
|
|
881
|
+
NoteExclusion(category);
|
|
882
|
+
if (category == null)
|
|
883
|
+
{
|
|
884
|
+
Dictionary<string, object> data = D("title", title ?? "");
|
|
885
|
+
if (url != null) data["url"] = url;
|
|
886
|
+
Emit("app_focus", true, data);
|
|
887
|
+
}
|
|
888
|
+
CaptureScreens(switched ? "app switch" : "window change");
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
// ── Keys and clicks (Raw Input) ──────────────────────────────────────────
|
|
892
|
+
|
|
893
|
+
protected override void WndProc(ref Message m)
|
|
894
|
+
{
|
|
895
|
+
if (m.Msg == WM_INPUT) OnRawInput(m.LParam);
|
|
896
|
+
else if (m.Msg == WM_CLIPBOARDUPDATE) OnClipboard();
|
|
897
|
+
else if (m.Msg == MM_WIM_DATA) OnAudio(m.LParam);
|
|
898
|
+
base.WndProc(ref m);
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
void OnRawInput(IntPtr handle)
|
|
902
|
+
{
|
|
903
|
+
uint headerSize = (uint)(8 + 2 * IntPtr.Size);
|
|
904
|
+
uint size = 0;
|
|
905
|
+
Native.GetRawInputData(handle, 0x10000003, IntPtr.Zero, ref size, headerSize);
|
|
906
|
+
if (size == 0) return;
|
|
907
|
+
IntPtr buffer = Marshal.AllocHGlobal((int)size);
|
|
908
|
+
try
|
|
909
|
+
{
|
|
910
|
+
if (Native.GetRawInputData(handle, 0x10000003, buffer, ref size, headerSize) != size) return;
|
|
911
|
+
int type = Marshal.ReadInt32(buffer);
|
|
912
|
+
IntPtr body = new IntPtr(buffer.ToInt64() + headerSize);
|
|
913
|
+
if (type == 1) // keyboard: MakeCode, Flags, Reserved, VKey
|
|
914
|
+
{
|
|
915
|
+
ushort flags = (ushort)Marshal.ReadInt16(body, 2);
|
|
916
|
+
ushort vkey = (ushort)Marshal.ReadInt16(body, 6);
|
|
917
|
+
ushort make = (ushort)Marshal.ReadInt16(body, 0);
|
|
918
|
+
bool down = (flags & 1) == 0;
|
|
919
|
+
if (vkey == 0x10 || vkey == 0xA0 || vkey == 0xA1) shiftDown = down;
|
|
920
|
+
else if (vkey == 0x11 || vkey == 0xA2 || vkey == 0xA3) ctrlDown = down;
|
|
921
|
+
else if (vkey == 0x12 || vkey == 0xA4 || vkey == 0xA5) altDown = down;
|
|
922
|
+
else if (down) { keyCount++; OnKey(vkey, make); }
|
|
923
|
+
}
|
|
924
|
+
else if (type == 0) // mouse: usFlags, (pad), usButtonFlags
|
|
925
|
+
{
|
|
926
|
+
ushort buttons = (ushort)Marshal.ReadInt16(body, 4);
|
|
927
|
+
if ((buttons & 0x5) != 0) clickCount++;
|
|
928
|
+
if ((buttons & 0x1) != 0) OnClick("left");
|
|
929
|
+
else if ((buttons & 0x4) != 0) OnClick("right");
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
finally { Marshal.FreeHGlobal(buffer); }
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
void OnKey(ushort vkey, ushort make)
|
|
937
|
+
{
|
|
938
|
+
if (vkey == 0x5B || vkey == 0x5C || vkey == 0x14) return;
|
|
939
|
+
if (ctrlDown || altDown)
|
|
940
|
+
{
|
|
941
|
+
if (ctrlDown && !altDown && vkey == 0x56) OnPaste(); // Ctrl+V
|
|
942
|
+
return;
|
|
943
|
+
}
|
|
944
|
+
if (burst.Length > 0 && burstExe != frontExe) FlushBurst();
|
|
945
|
+
if (burst.Length == 0) { burstExe = frontExe; burstName = frontName; recheckField = true; }
|
|
946
|
+
// Which field the keys go to is asked again on every key after Tab, Enter or
|
|
947
|
+
// a click, and otherwise at most every 150 ms: a burst that moves from a
|
|
948
|
+
// username into a password field must not carry the password with it.
|
|
949
|
+
if (recheckField || (DateTime.UtcNow - lastFieldCheck).TotalMilliseconds > 150)
|
|
950
|
+
{
|
|
951
|
+
IntPtr hwnd = frontHwnd;
|
|
952
|
+
bool? password = Bounded<bool?>(delegate () { AutomationElement f = AutomationElement.FocusedElement; return f != null && f.Current.IsPassword; }, 250, null);
|
|
953
|
+
string tail = Decisions.IsTerminal(frontExe) ? Bounded(delegate () { return TerminalTail(hwnd); }, 250, (string)null) : null;
|
|
954
|
+
lastFieldCheck = DateTime.UtcNow; recheckField = false;
|
|
955
|
+
if (password == null || Decisions.ShouldSkipKeystrokes(password.Value, frontExe, tail))
|
|
956
|
+
{
|
|
957
|
+
// What was typed before the last check that passed went to a field that was
|
|
958
|
+
// not a password; keep that, never what came after.
|
|
959
|
+
FlushVerified();
|
|
960
|
+
if (password == true || (tail != null && Decisions.LooksLikePasswordPrompt(tail))) burstSecure = true;
|
|
961
|
+
else { recheckField = true; return; } // could not tell (a slow app): drop this key, ask again on the next
|
|
962
|
+
}
|
|
963
|
+
else verifiedLength = burst.Length;
|
|
964
|
+
}
|
|
965
|
+
if (focusIsPassword) burstSecure = true;
|
|
966
|
+
lastKey = DateTime.UtcNow;
|
|
967
|
+
if (vkey == 0x08) { if (burst.Length > 0) burst.Length--; return; }
|
|
968
|
+
if (vkey == 0x0D) { burst.Append('\n'); recheckField = true; return; }
|
|
969
|
+
if (vkey == 0x09) { burst.Append('\t'); recheckField = true; return; }
|
|
970
|
+
byte[] state = new byte[256];
|
|
971
|
+
if (shiftDown) state[0x10] = 0x80;
|
|
972
|
+
if ((Native.GetKeyState(0x14) & 1) != 0) state[0x14] = 1;
|
|
973
|
+
uint ownerPid;
|
|
974
|
+
uint thread = Native.GetWindowThreadProcessId(frontHwnd, out ownerPid);
|
|
975
|
+
StringBuilder chars = new StringBuilder(8);
|
|
976
|
+
int count = Native.ToUnicodeEx(vkey, make, state, chars, chars.Capacity, 4, Native.GetKeyboardLayout(thread));
|
|
977
|
+
if (count > 0) foreach (char c in chars.ToString(0, count)) if (!char.IsControl(c)) burst.Append(c);
|
|
978
|
+
int cut = burst.Length < 2000 ? -1 : Decisions.BurstCut(burst.ToString());
|
|
979
|
+
if (cut > 0) FlushPart(cut);
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
static string TerminalTail(IntPtr hwnd)
|
|
983
|
+
{
|
|
984
|
+
string text;
|
|
985
|
+
if (IsConsole(hwnd)) text = ConsoleText(hwnd, false);
|
|
986
|
+
else
|
|
987
|
+
{
|
|
988
|
+
AutomationElement focused = AutomationElement.FocusedElement;
|
|
989
|
+
object pattern;
|
|
990
|
+
if (focused == null || !focused.TryGetCurrentPattern(TextPattern.Pattern, out pattern)) return null;
|
|
991
|
+
text = RangeText(((TextPattern)pattern).DocumentRange, -1);
|
|
992
|
+
}
|
|
993
|
+
if (text == null) return null;
|
|
994
|
+
text = text.TrimEnd();
|
|
995
|
+
return text.Length > 400 ? text.Substring(text.Length - 400) : text;
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
/// A TextPattern range's text, or null. UI Automation's GetText on a classic
|
|
999
|
+
/// console (conhost) raises an access violation on Windows 10 and Server 2022,
|
|
1000
|
+
/// which .NET otherwise lets end the whole process.
|
|
1001
|
+
[HandleProcessCorruptedStateExceptions, SecurityCritical]
|
|
1002
|
+
static string RangeText(TextPatternRange range, int max)
|
|
1003
|
+
{
|
|
1004
|
+
try { return range.GetText(max); } catch (Exception) { return null; }
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
static bool IsConsole(IntPtr hwnd)
|
|
1008
|
+
{
|
|
1009
|
+
StringBuilder name = new StringBuilder(64);
|
|
1010
|
+
Native.GetClassName(hwnd, name, name.Capacity);
|
|
1011
|
+
return name.ToString() == "ConsoleWindowClass";
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
static readonly object consoleLock = new object();
|
|
1015
|
+
|
|
1016
|
+
/// A classic console's text, read through the console API instead of UI Automation
|
|
1017
|
+
/// (see RangeText). `shown`: only what the window shows, for screen text. Otherwise
|
|
1018
|
+
/// the lines up to the cursor at full width, for the password prompt check, which
|
|
1019
|
+
/// must see a prompt even when the window is scrolled away from it. Null when unreadable.
|
|
1020
|
+
static string ConsoleText(IntPtr hwnd, bool shown)
|
|
1021
|
+
{
|
|
1022
|
+
uint pid;
|
|
1023
|
+
Native.GetWindowThreadProcessId(hwnd, out pid);
|
|
1024
|
+
lock (consoleLock)
|
|
1025
|
+
{
|
|
1026
|
+
if (!Native.AttachConsole(pid)) return null;
|
|
1027
|
+
// Attached, a Ctrl+C typed in that console would reach this app too.
|
|
1028
|
+
Native.SetConsoleCtrlHandler(IntPtr.Zero, true);
|
|
1029
|
+
IntPtr output = Native.CreateFile("CONOUT$", 0xC0000000, 3, IntPtr.Zero, 3, 0, IntPtr.Zero);
|
|
1030
|
+
try
|
|
1031
|
+
{
|
|
1032
|
+
Native.CONSOLE_SCREEN_BUFFER_INFO info;
|
|
1033
|
+
if (output == new IntPtr(-1) || !Native.GetConsoleScreenBufferInfo(output, out info)) return null;
|
|
1034
|
+
StringBuilder text = new StringBuilder();
|
|
1035
|
+
short left = shown ? info.Window.Left : (short)0;
|
|
1036
|
+
int width = shown ? info.Window.Right - info.Window.Left + 1 : info.Size.X;
|
|
1037
|
+
int top = shown ? info.Window.Top : Math.Max(0, info.Cursor.Y - 20), bottom = shown ? info.Window.Bottom : info.Cursor.Y;
|
|
1038
|
+
// A char[] and the count read: the console does not end a line with a NUL.
|
|
1039
|
+
char[] line = new char[width];
|
|
1040
|
+
for (int y = top; y <= bottom; y++)
|
|
1041
|
+
{
|
|
1042
|
+
uint read;
|
|
1043
|
+
Native.COORD at = new Native.COORD { X = left, Y = (short)y };
|
|
1044
|
+
if (!Native.ReadConsoleOutputCharacter(output, line, (uint)width, at, out read)) return null;
|
|
1045
|
+
text.AppendLine(new string(line, 0, (int)Math.Min(read, (uint)line.Length)).TrimEnd());
|
|
1046
|
+
}
|
|
1047
|
+
return text.ToString().TrimEnd();
|
|
1048
|
+
}
|
|
1049
|
+
finally { if (output != new IntPtr(-1)) Native.CloseHandle(output); Native.FreeConsole(); }
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
/// Sends the first `cut` characters of the burst and keeps typing the rest.
|
|
1054
|
+
void FlushPart(int cut)
|
|
1055
|
+
{
|
|
1056
|
+
string rest = burst.ToString(cut, burst.Length - cut), exe = burstExe, name = burstName;
|
|
1057
|
+
bool secure = burstSecure;
|
|
1058
|
+
int verified = verifiedLength;
|
|
1059
|
+
burst.Length = cut;
|
|
1060
|
+
FlushBurst();
|
|
1061
|
+
burst.Append(rest); burstExe = exe; burstName = name; burstSecure = secure; verifiedLength = Math.Max(0, verified - cut);
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
void FlushVerified()
|
|
1065
|
+
{
|
|
1066
|
+
string exe = burstExe, name = burstName;
|
|
1067
|
+
if (!burstSecure) burst.Length = Math.Min(verifiedLength, burst.Length);
|
|
1068
|
+
FlushBurst();
|
|
1069
|
+
burstExe = exe; burstName = name;
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
void FlushBurst()
|
|
1073
|
+
{
|
|
1074
|
+
verifiedLength = 0;
|
|
1075
|
+
string text = burst.ToString();
|
|
1076
|
+
bool secure = burstSecure;
|
|
1077
|
+
string exe = burstExe, name = burstName;
|
|
1078
|
+
burst.Length = 0; burstSecure = false; burstExe = null; burstName = null;
|
|
1079
|
+
// A burst that touched a password field is dropped whole.
|
|
1080
|
+
if (text.Length == 0 || secure || ExcludedFor(exe) != null) return;
|
|
1081
|
+
Dictionary<string, object> e = D("kind", "keys", "ts", Now(), "app", name ?? exe, "app_id", exe, "data", D("text", text, "field", ""));
|
|
1082
|
+
Send(e);
|
|
1083
|
+
CaptureScreens("typing pause");
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
void OnClick(string button)
|
|
1087
|
+
{
|
|
1088
|
+
FlushBurst();
|
|
1089
|
+
recheckField = true;
|
|
1090
|
+
if (CurrentExcluded() != null) return;
|
|
1091
|
+
Native.POINT point;
|
|
1092
|
+
Native.GetCursorPos(out point);
|
|
1093
|
+
string[] element = Bounded(delegate ()
|
|
1094
|
+
{
|
|
1095
|
+
AutomationElement at = AutomationElement.FromPoint(new System.Windows.Point(point.X, point.Y));
|
|
1096
|
+
return new[] { at.Current.ControlType.ProgrammaticName.Replace("ControlType.", ""), at.Current.IsPassword ? "" : at.Current.Name };
|
|
1097
|
+
}, 300, new[] { "", "" });
|
|
1098
|
+
Emit("click", true, D("button", button, "role", element[0], "name", element[1].Length > 200 ? element[1].Substring(0, 200) : element[1]));
|
|
1099
|
+
CaptureScreens("click");
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
// ── Clipboard ────────────────────────────────────────────────────────────
|
|
1103
|
+
|
|
1104
|
+
string ClipboardText()
|
|
1105
|
+
{
|
|
1106
|
+
// Password managers mark what they copy; never read it.
|
|
1107
|
+
if (Native.IsClipboardFormatAvailable(concealedFormat) || Native.IsClipboardFormatAvailable(excludeHistoryFormat)) return null;
|
|
1108
|
+
// Reading waits on the app that owns the clipboard, so it runs on its own
|
|
1109
|
+
// STA thread with a deadline: a busy owner must never stall capture.
|
|
1110
|
+
string text = null;
|
|
1111
|
+
Thread reader = new Thread(delegate () { try { if (Clipboard.ContainsText()) text = Clipboard.GetText(); } catch (Exception) { } });
|
|
1112
|
+
reader.SetApartmentState(ApartmentState.STA);
|
|
1113
|
+
reader.IsBackground = true;
|
|
1114
|
+
reader.Start();
|
|
1115
|
+
return reader.Join(500) ? text : null;
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
static string Hash(string text)
|
|
1119
|
+
{
|
|
1120
|
+
using (SHA256 sha = SHA256.Create())
|
|
1121
|
+
{
|
|
1122
|
+
byte[] digest = sha.ComputeHash(Encoding.UTF8.GetBytes(text));
|
|
1123
|
+
return BitConverter.ToString(digest).Replace("-", "").ToLowerInvariant().Substring(0, 16);
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
string lastCopyHash;
|
|
1128
|
+
DateTime lastCopyAt = DateTime.MinValue;
|
|
1129
|
+
|
|
1130
|
+
void OnClipboard()
|
|
1131
|
+
{
|
|
1132
|
+
string text = ClipboardText();
|
|
1133
|
+
if (text == null || CurrentExcluded() != null) return;
|
|
1134
|
+
// One copy can update the clipboard several times (one per format).
|
|
1135
|
+
string digest = Hash(text);
|
|
1136
|
+
if (digest == lastCopyHash && (DateTime.UtcNow - lastCopyAt).TotalSeconds < 2) return;
|
|
1137
|
+
lastCopyHash = digest; lastCopyAt = DateTime.UtcNow;
|
|
1138
|
+
Emit("clipboard_copy", true, D("text", text.Length > 4000 ? text.Substring(0, 4000) : text, "hash", Hash(text), "chars", text.Length));
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
void OnPaste()
|
|
1142
|
+
{
|
|
1143
|
+
FlushBurst();
|
|
1144
|
+
// A paste into a password field (or a terminal password prompt) keeps nothing.
|
|
1145
|
+
IntPtr hwnd = frontHwnd;
|
|
1146
|
+
bool password = Bounded(delegate () { AutomationElement f = AutomationElement.FocusedElement; return f != null && f.Current.IsPassword; }, 250, true);
|
|
1147
|
+
string tail = Decisions.IsTerminal(frontExe) ? Bounded(delegate () { return TerminalTail(hwnd); }, 250, (string)null) : null;
|
|
1148
|
+
if (Decisions.ShouldSkipKeystrokes(password, frontExe, tail)) return;
|
|
1149
|
+
string text = ClipboardText();
|
|
1150
|
+
if (text == null || CurrentExcluded() != null) return;
|
|
1151
|
+
Emit("clipboard_paste", true, D("text", text.Length > 4000 ? text.Substring(0, 4000) : text, "hash", Hash(text), "chars", text.Length));
|
|
1152
|
+
CaptureScreens("paste");
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
// ── Screens, displays, hidden windows ───────────────────────────────────
|
|
1156
|
+
|
|
1157
|
+
static byte[] Digest(Bitmap image)
|
|
1158
|
+
{
|
|
1159
|
+
using (Bitmap small = new Bitmap(image, new Size(64, 36)))
|
|
1160
|
+
{
|
|
1161
|
+
byte[] pixels = new byte[64 * 36];
|
|
1162
|
+
for (int y = 0; y < 36; y++) for (int x = 0; x < 64; x++)
|
|
1163
|
+
{
|
|
1164
|
+
Color c = small.GetPixel(x, y);
|
|
1165
|
+
pixels[y * 64 + x] = (byte)((c.R + c.G + c.B) / 3);
|
|
1166
|
+
}
|
|
1167
|
+
return pixels;
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
/// A screen changed when a few cells moved clearly: an answer appearing changes a
|
|
1172
|
+
/// handful of cells a lot and the average barely at all; a caret moves one a little.
|
|
1173
|
+
static bool Changed(byte[] a, byte[] b)
|
|
1174
|
+
{
|
|
1175
|
+
if (a == null) return true;
|
|
1176
|
+
int moved = 0;
|
|
1177
|
+
for (int i = 0; i < b.Length; i++) if (Math.Abs(a[i] - b[i]) > 8) moved++;
|
|
1178
|
+
return moved >= 3;
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
/// The front window, and every browser window behind it shown on its page's merit, still
|
|
1182
|
+
/// what the capture judged: a frame taken after either moved on shows what no check saw.
|
|
1183
|
+
bool StillShowing()
|
|
1184
|
+
{
|
|
1185
|
+
IntPtr now = Native.GetForegroundWindow();
|
|
1186
|
+
if (!Decisions.FrontStillMatches(frontHwnd, frontTitle, now, WindowTitle(now))) return false;
|
|
1187
|
+
foreach (KeyValuePair<IntPtr, string> shownWindow in shownBehind) if (WindowTitle(shownWindow.Key) != shownWindow.Value) return false;
|
|
1188
|
+
return true;
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
/// Every visible window and where it is, to tell a layout that moved between two reads.
|
|
1192
|
+
static string Layout()
|
|
1193
|
+
{
|
|
1194
|
+
StringBuilder layout = new StringBuilder();
|
|
1195
|
+
Native.EnumWindows(delegate (IntPtr hwnd, IntPtr lParam)
|
|
1196
|
+
{
|
|
1197
|
+
Native.RECT r;
|
|
1198
|
+
if (Native.IsWindowVisible(hwnd) && !Native.IsIconic(hwnd) && Native.GetWindowRect(hwnd, out r))
|
|
1199
|
+
layout.Append(hwnd.ToInt64()).Append(Cloaked(hwnd) ? "c:" : ":").Append(r.Left).Append(',').Append(r.Top).Append(',').Append(r.Right).Append(',').Append(r.Bottom).Append(';');
|
|
1200
|
+
return true;
|
|
1201
|
+
}, IntPtr.Zero);
|
|
1202
|
+
return layout.ToString();
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
/// A window DWM does not draw (a suspended app, another virtual desktop) though it is "visible".
|
|
1206
|
+
static bool Cloaked(IntPtr hwnd) { int cloaked; return Native.DwmGetWindowAttribute(hwnd, 14, out cloaked, 4) == 0 && cloaked != 0; }
|
|
1207
|
+
|
|
1208
|
+
/// Which window kept a frame unchecked, for its withheld reason: the app and window kind, no title.
|
|
1209
|
+
static string Describe(IntPtr hwnd)
|
|
1210
|
+
{
|
|
1211
|
+
string name;
|
|
1212
|
+
StringBuilder kind = new StringBuilder(64);
|
|
1213
|
+
Native.GetClassName(hwnd, kind, kind.Capacity);
|
|
1214
|
+
return (ProcessExe(hwnd, out name) ?? "unknown app") + " " + kind;
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
static List<Rectangle> Fills(List<Rectangle> credentials, List<Rectangle> unread) { List<Rectangle> all = new List<Rectangle>(credentials); all.AddRange(unread); return all; }
|
|
1218
|
+
|
|
1219
|
+
/// Browser windows behind the front one shown on their page's merit, with the title they
|
|
1220
|
+
/// were judged with: read again after the pixels, and blanked if their text can't be read.
|
|
1221
|
+
Dictionary<IntPtr, string> shownBehind = new Dictionary<IntPtr, string>();
|
|
1222
|
+
|
|
1223
|
+
/// ENG-4737: what of the screen the front window surely hides (Decisions.CountsAsCover,
|
|
1224
|
+
/// Decisions.CoverOf), or empty when it may be see-through and windows behind it are filled whole.
|
|
1225
|
+
Rectangle FrontCover()
|
|
1226
|
+
{
|
|
1227
|
+
IntPtr hwnd = frontHwnd;
|
|
1228
|
+
Native.RECT r;
|
|
1229
|
+
int backdrop;
|
|
1230
|
+
Native.RECT shaped;
|
|
1231
|
+
// A window with a region (a skinned, non-rectangular one) shows what is behind it outside that shape.
|
|
1232
|
+
if (hwnd == IntPtr.Zero || Cloaked(hwnd) || !Native.GetWindowRect(hwnd, out r) || Native.GetWindowRgnBox(hwnd, out shaped) != 0) return Rectangle.Empty;
|
|
1233
|
+
if (Native.DwmGetWindowAttribute(hwnd, 38, out backdrop, 4) != 0) backdrop = 0; // DWMWA_SYSTEMBACKDROP_TYPE, Windows 11 22H2+
|
|
1234
|
+
StringBuilder kind = new StringBuilder(64);
|
|
1235
|
+
Native.GetClassName(hwnd, kind, kind.Capacity);
|
|
1236
|
+
if (!Decisions.CountsAsCover(Native.GetWindowLong(hwnd, -20), kind.ToString(), backdrop)) return Rectangle.Empty;
|
|
1237
|
+
return Decisions.CoverOf(Rectangle.FromLTRB(r.Left, r.Top, r.Right, r.Bottom));
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
/// The parts of a window to fill: all of it, or for a window below the front one (EnumWindows
|
|
1241
|
+
/// goes top to bottom) what the front window's cover leaves showing.
|
|
1242
|
+
static List<Rectangle> FillOf(Rectangle rect, bool belowFront, Rectangle cover)
|
|
1243
|
+
{
|
|
1244
|
+
return belowFront ? Decisions.Uncovered(rect, cover) : new List<Rectangle> { rect };
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
/// Windows that keep no pixels, as the parts of them to fill, each with why: its excluded
|
|
1248
|
+
/// category, or "unverified" for a browser window behind the front one whose page can't be
|
|
1249
|
+
/// told (BackgroundExclusion). `whole`: their whole rectangles, which no text read visits.
|
|
1250
|
+
List<KeyValuePair<Rectangle, string>> ExcludedWindows(Rectangle cover, out List<Rectangle> whole)
|
|
1251
|
+
{
|
|
1252
|
+
List<KeyValuePair<Rectangle, string>> rects = new List<KeyValuePair<Rectangle, string>>();
|
|
1253
|
+
List<Rectangle> wholes = new List<Rectangle>();
|
|
1254
|
+
Dictionary<IntPtr, string> behind = new Dictionary<IntPtr, string>();
|
|
1255
|
+
bool belowFront = false;
|
|
1256
|
+
Native.EnumWindows(delegate (IntPtr hwnd, IntPtr lParam)
|
|
1257
|
+
{
|
|
1258
|
+
if (hwnd == frontHwnd) belowFront = true;
|
|
1259
|
+
if (!Native.IsWindowVisible(hwnd) || Native.IsIconic(hwnd)) return true;
|
|
1260
|
+
string ignored;
|
|
1261
|
+
string exe = ProcessExe(hwnd, out ignored);
|
|
1262
|
+
string url = hwnd == frontHwnd ? frontUrl : null;
|
|
1263
|
+
string category = Decisions.ExcludedCategory(exclusions, exe, url);
|
|
1264
|
+
// A browser window behind the front one is judged by its own page, read the way the front one's is.
|
|
1265
|
+
if (category == null && hwnd != frontHwnd && Array.IndexOf(Browsers, exe) >= 0)
|
|
1266
|
+
{
|
|
1267
|
+
IntPtr window = hwnd;
|
|
1268
|
+
string page = Bounded(delegate () { return BrowserUrl(window); }, 300, (string)null);
|
|
1269
|
+
string title = WindowTitle(hwnd);
|
|
1270
|
+
category = Decisions.BackgroundExclusion(exclusions, exe, page, title, true);
|
|
1271
|
+
if (category == null) behind[hwnd] = title;
|
|
1272
|
+
}
|
|
1273
|
+
if (category == null) return true;
|
|
1274
|
+
Native.RECT r;
|
|
1275
|
+
if (!Native.GetWindowRect(hwnd, out r)) return true;
|
|
1276
|
+
Rectangle rect = Rectangle.FromLTRB(r.Left, r.Top, r.Right, r.Bottom);
|
|
1277
|
+
wholes.Add(rect);
|
|
1278
|
+
foreach (Rectangle part in FillOf(rect, belowFront && hwnd != frontHwnd, cover)) rects.Add(new KeyValuePair<Rectangle, string>(part, category));
|
|
1279
|
+
return true;
|
|
1280
|
+
}, IntPtr.Zero);
|
|
1281
|
+
shownBehind = behind;
|
|
1282
|
+
whole = wholes;
|
|
1283
|
+
return rects;
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
static string WindowText(IntPtr hwnd) { bool complete; return WindowText(hwnd, 600, out complete); }
|
|
1287
|
+
|
|
1288
|
+
/// The text a window shows. `complete` is false when a limit stopped the walk
|
|
1289
|
+
/// first or any element could not be read, so a check built on it must not pass.
|
|
1290
|
+
static string WindowText(IntPtr hwnd, int budgetMs, out bool complete)
|
|
1291
|
+
{
|
|
1292
|
+
if (IsConsole(hwnd)) { string shown = ConsoleText(hwnd, true); complete = shown != null; return shown ?? ""; }
|
|
1293
|
+
StringBuilder text = new StringBuilder();
|
|
1294
|
+
bool unread = false;
|
|
1295
|
+
int nodes = 0;
|
|
1296
|
+
DateTime started = DateTime.UtcNow;
|
|
1297
|
+
TreeWalker walker = TreeWalker.ControlViewWalker;
|
|
1298
|
+
Stack<AutomationElement> stack = new Stack<AutomationElement>();
|
|
1299
|
+
stack.Push(AutomationElement.FromHandle(hwnd));
|
|
1300
|
+
while (stack.Count > 0 && nodes < 3000 && text.Length < 20000 && (DateTime.UtcNow - started).TotalMilliseconds < budgetMs)
|
|
1301
|
+
{
|
|
1302
|
+
AutomationElement element = stack.Pop();
|
|
1303
|
+
nodes++;
|
|
1304
|
+
try
|
|
1305
|
+
{
|
|
1306
|
+
if (element.Current.IsPassword) continue;
|
|
1307
|
+
ControlType type = element.Current.ControlType;
|
|
1308
|
+
if (type == ControlType.Text || type == ControlType.Edit || type == ControlType.Document || type == ControlType.ListItem)
|
|
1309
|
+
{
|
|
1310
|
+
string value = element.Current.Name;
|
|
1311
|
+
object pattern;
|
|
1312
|
+
if ((type == ControlType.Edit || type == ControlType.Document) && element.TryGetCurrentPattern(ValuePattern.Pattern, out pattern))
|
|
1313
|
+
value = ((ValuePattern)pattern).Current.Value;
|
|
1314
|
+
if (!string.IsNullOrEmpty(value)) text.AppendLine(value);
|
|
1315
|
+
}
|
|
1316
|
+
// Terminals and rich editors keep their visible text behind TextPattern
|
|
1317
|
+
// (Windows Terminal, conhost). Only the visible ranges: scrollback that
|
|
1318
|
+
// is off screen is neither on the picture nor anything a reviewer saw.
|
|
1319
|
+
object textPattern;
|
|
1320
|
+
if (type != ControlType.Text && element.TryGetCurrentPattern(TextPattern.Pattern, out textPattern))
|
|
1321
|
+
foreach (TextPatternRange range in ((TextPattern)textPattern).GetVisibleRanges())
|
|
1322
|
+
{
|
|
1323
|
+
string body = RangeText(range, 20001);
|
|
1324
|
+
if (body == null || body.Length > 20000) unread = true; // failed, or cut off: the rest went unread
|
|
1325
|
+
if (!string.IsNullOrEmpty(body)) text.AppendLine(body);
|
|
1326
|
+
}
|
|
1327
|
+
List<AutomationElement> children = new List<AutomationElement>();
|
|
1328
|
+
for (AutomationElement child = walker.GetFirstChild(element); child != null; child = walker.GetNextSibling(child)) children.Add(child);
|
|
1329
|
+
for (int i = children.Count - 1; i >= 0; i--) stack.Push(children[i]);
|
|
1330
|
+
}
|
|
1331
|
+
catch (Exception) { unread = true; } // its text and children went unread
|
|
1332
|
+
}
|
|
1333
|
+
complete = stack.Count == 0 && !unread;
|
|
1334
|
+
return text.ToString();
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
/// The text of every window showing on a display, which the watcher checks for
|
|
1338
|
+
/// credentials before it keeps the picture. A window whose text shows a credential
|
|
1339
|
+
/// is left out and returned in `blanked`, to be blanked whole: UI Automation gives
|
|
1340
|
+
/// text, not where on screen it sits; one that could not be read in time is returned
|
|
1341
|
+
/// in `unread`, blanked the same way. Null when the front window could not be read in
|
|
1342
|
+
/// time; the frame is then withheld, and `failed` says which window and why. `windowText`:
|
|
1343
|
+
/// what each window in the text showed, and `fillOf` what to blank of it, for Decisions.Agreed.
|
|
1344
|
+
/// ENG-4731: one frame's read of every window. The front window, read first, took up to
|
|
1345
|
+
/// 1.17 s on the Windows 11 ARM VM under load (Paint, Notepad); 1.1 s withheld those frames.
|
|
1346
|
+
const int ReadBudgetMs = 1600;
|
|
1347
|
+
|
|
1348
|
+
string VisibleText(Rectangle display, List<Rectangle> masked, Rectangle cover, out List<Rectangle> blanked, out List<Rectangle> unread, out string failed,
|
|
1349
|
+
out List<KeyValuePair<long, string>> windowText, out Dictionary<long, List<Rectangle>> fillOf)
|
|
1350
|
+
{
|
|
1351
|
+
List<KeyValuePair<IntPtr, Rectangle>> windows = new List<KeyValuePair<IntPtr, Rectangle>>();
|
|
1352
|
+
bool belowFront = false;
|
|
1353
|
+
Native.EnumWindows(delegate (IntPtr hwnd, IntPtr lParam)
|
|
1354
|
+
{
|
|
1355
|
+
Native.RECT r;
|
|
1356
|
+
bool below = belowFront;
|
|
1357
|
+
if (hwnd == frontHwnd) belowFront = true;
|
|
1358
|
+
if (!Native.IsWindowVisible(hwnd) || Native.IsIconic(hwnd) || Cloaked(hwnd) || !Native.GetWindowRect(hwnd, out r)) return true;
|
|
1359
|
+
Rectangle rect = Rectangle.FromLTRB(r.Left, r.Top, r.Right, r.Bottom);
|
|
1360
|
+
if (rect.Width < 40 || rect.Height < 40 || !rect.IntersectsWith(display) || masked.Contains(rect)) return true;
|
|
1361
|
+
// ENG-4731: a window the front one hides whole has no pixels on the frame, so it is not read.
|
|
1362
|
+
if (below && cover.Contains(rect)) return true;
|
|
1363
|
+
windows.Add(new KeyValuePair<IntPtr, Rectangle>(hwnd, rect));
|
|
1364
|
+
return true;
|
|
1365
|
+
}, IntPtr.Zero);
|
|
1366
|
+
int frontAt = windows.FindIndex(delegate (KeyValuePair<IntPtr, Rectangle> w) { return w.Key == frontHwnd; });
|
|
1367
|
+
List<string> secrets = known;
|
|
1368
|
+
List<Rectangle> found = new List<Rectangle>(), skipped = new List<Rectangle>();
|
|
1369
|
+
List<KeyValuePair<long, string>> readClean = new List<KeyValuePair<long, string>>();
|
|
1370
|
+
Dictionary<long, List<Rectangle>> fills = new Dictionary<long, List<Rectangle>>();
|
|
1371
|
+
for (int i = 0; i < windows.Count; i++) fills[windows[i].Key.ToInt64()] = FillOf(windows[i].Value, frontAt >= 0 && i > frontAt, cover);
|
|
1372
|
+
int reading = -1, stuck = -1;
|
|
1373
|
+
string text = Bounded(delegate ()
|
|
1374
|
+
{
|
|
1375
|
+
List<int> hits = new List<int>(), unreadHits = new List<int>();
|
|
1376
|
+
List<KeyValuePair<int, string>> clean = new List<KeyValuePair<int, string>>();
|
|
1377
|
+
int failedAt;
|
|
1378
|
+
// A window read only in part can't vouch for its picture: it is blanked, less what the
|
|
1379
|
+
// front window covers of it; the front window itself withholds the frame.
|
|
1380
|
+
string all = Decisions.ReadWindows(windows.Count, frontAt, delegate (int i, int budget, out bool complete)
|
|
1381
|
+
{
|
|
1382
|
+
reading = i;
|
|
1383
|
+
IntPtr hwnd = windows[i].Key;
|
|
1384
|
+
// One UI Automation call can block past the budget (a busy app): the window
|
|
1385
|
+
// gets its own deadline, so it costs its own pixels, not the frame.
|
|
1386
|
+
bool[] whole = { false };
|
|
1387
|
+
string body = Bounded(delegate () { bool read; string shown = WindowText(hwnd, budget, out read); whole[0] = read; return shown; }, budget + 100, (string)null);
|
|
1388
|
+
complete = body != null && whole[0];
|
|
1389
|
+
return WindowTitle(hwnd) + "\n" + body;
|
|
1390
|
+
}, secrets, ReadBudgetMs, hits, unreadHits, clean, out failedAt);
|
|
1391
|
+
// Handed over only by a read that finished: a timed-out one may still be running.
|
|
1392
|
+
lock (found)
|
|
1393
|
+
{
|
|
1394
|
+
stuck = failedAt;
|
|
1395
|
+
// Blanked whole, less what the front window covers of one behind it (ENG-4737).
|
|
1396
|
+
foreach (int i in hits) found.AddRange(fills[windows[i].Key.ToInt64()]);
|
|
1397
|
+
foreach (int i in unreadHits) skipped.AddRange(fills[windows[i].Key.ToInt64()]);
|
|
1398
|
+
foreach (KeyValuePair<int, string> w in clean) readClean.Add(new KeyValuePair<long, string>(windows[w.Key].Key.ToInt64(), w.Value));
|
|
1399
|
+
}
|
|
1400
|
+
return all;
|
|
1401
|
+
}, ReadBudgetMs + 400, (string)null);
|
|
1402
|
+
lock (found)
|
|
1403
|
+
{
|
|
1404
|
+
blanked = text == null ? new List<Rectangle>() : new List<Rectangle>(found);
|
|
1405
|
+
unread = text == null ? new List<Rectangle>() : new List<Rectangle>(skipped);
|
|
1406
|
+
windowText = text == null ? new List<KeyValuePair<long, string>>() : new List<KeyValuePair<long, string>>(readClean);
|
|
1407
|
+
int at = stuck >= 0 ? stuck : reading;
|
|
1408
|
+
failed = text != null ? null : at < 0 ? "no window read" : Describe(windows[at].Key) + (stuck >= 0 ? " not read in full" : " timed out");
|
|
1409
|
+
}
|
|
1410
|
+
fillOf = fills;
|
|
1411
|
+
return text;
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
readonly Decisions.ExclusionNotice exclusionNotice = new Decisions.ExclusionNotice();
|
|
1415
|
+
/// One excluded row when what is in front starts or stops keeping nothing; the
|
|
1416
|
+
/// frame timer asks every few seconds and would otherwise log a row each time.
|
|
1417
|
+
void NoteExclusion(string category)
|
|
1418
|
+
{
|
|
1419
|
+
if (exclusionNotice.ShouldEmit(category)) Emit("excluded", false, D("category", category));
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
readonly Decisions.FrameCadence cadence = new Decisions.FrameCadence();
|
|
1423
|
+
/// The last frame sent per display, with the app it showed in front (or null), sent
|
|
1424
|
+
/// again while the display stays still.
|
|
1425
|
+
readonly Dictionary<string, KeyValuePair<string[], Dictionary<string, object>>> lastSent = new Dictionary<string, KeyValuePair<string[], Dictionary<string, object>>>();
|
|
1426
|
+
|
|
1427
|
+
void CaptureScreens(string reason)
|
|
1428
|
+
{
|
|
1429
|
+
string category = CurrentExcluded();
|
|
1430
|
+
NoteExclusion(category);
|
|
1431
|
+
if (category != null) return;
|
|
1432
|
+
Screen[] screens = Screen.AllScreens;
|
|
1433
|
+
// No display due (FrameCadence): no grab and no text read now.
|
|
1434
|
+
if (!Array.Exists(screens, delegate (Screen s) { return cadence.Due(s.DeviceName, DateTime.UtcNow); })) return;
|
|
1435
|
+
Screen front = Screen.FromHandle(frontHwnd);
|
|
1436
|
+
Rectangle cover = FrontCover();
|
|
1437
|
+
List<Rectangle> masked;
|
|
1438
|
+
List<KeyValuePair<Rectangle, string>> hide = ExcludedWindows(cover, out masked);
|
|
1439
|
+
List<Rectangle> fills = hide.ConvertAll<Rectangle>(delegate (KeyValuePair<Rectangle, string> h) { return h.Key; });
|
|
1440
|
+
for (int index = 0; index < screens.Length; index++)
|
|
1441
|
+
{
|
|
1442
|
+
Screen screen = screens[index];
|
|
1443
|
+
if (!cadence.Due(screen.DeviceName, DateTime.UtcNow)) continue;
|
|
1444
|
+
Rectangle bounds = screen.Bounds;
|
|
1445
|
+
using (Bitmap shot = new Bitmap(bounds.Width, bounds.Height, PixelFormat.Format24bppRgb))
|
|
1446
|
+
{
|
|
1447
|
+
using (Graphics g = Graphics.FromImage(shot))
|
|
1448
|
+
{
|
|
1449
|
+
try { g.CopyFromScreen(bounds.Location, Point.Empty, bounds.Size); } catch (Exception) { continue; }
|
|
1450
|
+
// Excluded apps keep no pixels, even in the background.
|
|
1451
|
+
foreach (Rectangle r in fills) { Rectangle cut = Rectangle.Intersect(r, bounds); if (!cut.IsEmpty) { cut.Offset(-bounds.X, -bounds.Y); g.FillRectangle(Brushes.Black, cut); } }
|
|
1452
|
+
}
|
|
1453
|
+
// Read again after the pixels, as on macOS: a frame taken while the front
|
|
1454
|
+
// window moved on (a tab turning into a bank page) shows what the check never saw.
|
|
1455
|
+
if (!StillShowing()) return;
|
|
1456
|
+
byte[] digest = Digest(shot);
|
|
1457
|
+
byte[] previous;
|
|
1458
|
+
lastDigest.TryGetValue(screen.DeviceName, out previous);
|
|
1459
|
+
Decisions.FrameAction action = cadence.Decide(screen.DeviceName, Changed(previous, digest), DateTime.UtcNow);
|
|
1460
|
+
if (action == Decisions.FrameAction.Skip) continue;
|
|
1461
|
+
KeyValuePair<string[], Dictionary<string, object>> held;
|
|
1462
|
+
if (action == Decisions.FrameAction.RepeatLast && lastSent.TryGetValue(screen.DeviceName, out held))
|
|
1463
|
+
{
|
|
1464
|
+
// Still: the last frame again, with nothing read or encoded, under the app it showed.
|
|
1465
|
+
Dictionary<string, object> again = new Dictionary<string, object>(held.Value);
|
|
1466
|
+
again["trigger"] = "unchanged";
|
|
1467
|
+
Dictionary<string, object> e = D("kind", "screen", "ts", Now(), "data", again);
|
|
1468
|
+
if (held.Key != null) { e["app"] = held.Key[0]; e["app_id"] = held.Key[1]; }
|
|
1469
|
+
Send(e);
|
|
1470
|
+
continue;
|
|
1471
|
+
}
|
|
1472
|
+
lastDigest[screen.DeviceName] = digest;
|
|
1473
|
+
int width = Math.Min(1440, bounds.Width);
|
|
1474
|
+
int height = bounds.Height * width / bounds.Width;
|
|
1475
|
+
double scale = (double)width / bounds.Width;
|
|
1476
|
+
// Read before encoding: a window showing a credential is blanked whole, so the
|
|
1477
|
+
// frame is kept with that window blank instead of withheld.
|
|
1478
|
+
List<Rectangle> blanked, unread;
|
|
1479
|
+
List<KeyValuePair<long, string>> windowText;
|
|
1480
|
+
Dictionary<long, List<Rectangle>> fillOf;
|
|
1481
|
+
string layout = Layout(), failed;
|
|
1482
|
+
string visible = VisibleText(bounds, masked, cover, out blanked, out unread, out failed, out windowText, out fillOf);
|
|
1483
|
+
// The text is read after the pixels, so a key shown at the grab and cleared by the read
|
|
1484
|
+
// would pass. Take the pixels again between this read and a second one, and keep them
|
|
1485
|
+
// only when the window layout and the front window agree; a window whose text changed,
|
|
1486
|
+
// or that either read blanked, is blanked (Decisions.Agreed).
|
|
1487
|
+
if (visible != null)
|
|
1488
|
+
{
|
|
1489
|
+
using (Graphics g = Graphics.FromImage(shot))
|
|
1490
|
+
{
|
|
1491
|
+
try { g.CopyFromScreen(bounds.Location, Point.Empty, bounds.Size); } catch (Exception) { continue; }
|
|
1492
|
+
foreach (Rectangle r in fills) { Rectangle cut = Rectangle.Intersect(r, bounds); if (!cut.IsEmpty) { cut.Offset(-bounds.X, -bounds.Y); g.FillRectangle(Brushes.Black, cut); } }
|
|
1493
|
+
}
|
|
1494
|
+
List<Rectangle> blankedAgain, unreadAgain;
|
|
1495
|
+
List<KeyValuePair<long, string>> textAgain;
|
|
1496
|
+
Dictionary<long, List<Rectangle>> fillAgain;
|
|
1497
|
+
string againFailed;
|
|
1498
|
+
string again = VisibleText(bounds, masked, cover, out blankedAgain, out unreadAgain, out againFailed, out textAgain, out fillAgain);
|
|
1499
|
+
Dictionary<long, string> second = new Dictionary<long, string>();
|
|
1500
|
+
foreach (KeyValuePair<long, string> w in textAgain) second[w.Key] = w.Value;
|
|
1501
|
+
List<long> changed = new List<long>();
|
|
1502
|
+
if (again == null) { visible = null; failed = againFailed; }
|
|
1503
|
+
else if ((visible = Decisions.Agreed(windowText, second, frontHwnd.ToInt64(), changed)) == null || Layout() != layout || !StillShowing())
|
|
1504
|
+
{
|
|
1505
|
+
lastDigest.Remove(screen.DeviceName); // looked at again on the next tick
|
|
1506
|
+
continue;
|
|
1507
|
+
}
|
|
1508
|
+
else
|
|
1509
|
+
{
|
|
1510
|
+
foreach (Rectangle r in blankedAgain) if (!blanked.Contains(r)) blanked.Add(r);
|
|
1511
|
+
foreach (Rectangle r in unreadAgain) if (!unread.Contains(r) && !blanked.Contains(r)) unread.Add(r);
|
|
1512
|
+
foreach (long w in changed) foreach (Rectangle r in fillOf[w]) if (!unread.Contains(r) && !blanked.Contains(r)) unread.Add(r);
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
List<Dictionary<string, object>> redactions = new List<Dictionary<string, object>>();
|
|
1516
|
+
foreach (KeyValuePair<Rectangle, string> h in hide)
|
|
1517
|
+
{
|
|
1518
|
+
Dictionary<string, object> box = Decisions.Redaction(h.Key, bounds, scale, "excluded_app", h.Value);
|
|
1519
|
+
if (box != null) redactions.Add(box);
|
|
1520
|
+
}
|
|
1521
|
+
if (blanked.Count + unread.Count > 0)
|
|
1522
|
+
using (Graphics g = Graphics.FromImage(shot))
|
|
1523
|
+
foreach (Rectangle r in Fills(blanked, unread))
|
|
1524
|
+
{
|
|
1525
|
+
Rectangle cut = Rectangle.Intersect(r, bounds);
|
|
1526
|
+
if (cut.IsEmpty) continue;
|
|
1527
|
+
cut.Offset(-bounds.X, -bounds.Y);
|
|
1528
|
+
g.FillRectangle(Brushes.Black, cut);
|
|
1529
|
+
bool credential = blanked.Contains(r);
|
|
1530
|
+
Dictionary<string, object> box = Decisions.Redaction(r, bounds, scale, credential ? "credential" : "excluded_app", credential ? null : "unverified");
|
|
1531
|
+
if (box != null) redactions.Add(box);
|
|
1532
|
+
}
|
|
1533
|
+
byte[] jpeg;
|
|
1534
|
+
using (Bitmap scaled = new Bitmap(shot, new Size(width, height)))
|
|
1535
|
+
using (MemoryStream stream = new MemoryStream())
|
|
1536
|
+
{
|
|
1537
|
+
ImageCodecInfo codec = Array.Find(ImageCodecInfo.GetImageEncoders(), delegate (ImageCodecInfo c) { return c.MimeType == "image/jpeg"; });
|
|
1538
|
+
EncoderParameters quality = new EncoderParameters(1);
|
|
1539
|
+
quality.Param[0] = new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, 55L);
|
|
1540
|
+
scaled.Save(stream, codec, quality);
|
|
1541
|
+
jpeg = stream.ToArray();
|
|
1542
|
+
}
|
|
1543
|
+
bool isFront = screen.DeviceName == front.DeviceName;
|
|
1544
|
+
Dictionary<string, object> data = D("display", index + 1, "frame", Convert.ToBase64String(jpeg), "width", width, "height", height, "trigger", reason);
|
|
1545
|
+
if (visible != null) data["check_text"] = visible;
|
|
1546
|
+
else data["check_failed"] = failed;
|
|
1547
|
+
if (redactions.Count > 0) data["redactions"] = Decisions.CapRedactions(redactions);
|
|
1548
|
+
if (isFront)
|
|
1549
|
+
{
|
|
1550
|
+
IntPtr hwnd = frontHwnd;
|
|
1551
|
+
string text = Bounded(delegate () { return WindowText(hwnd); }, 800, "");
|
|
1552
|
+
data["title"] = frontTitle ?? "";
|
|
1553
|
+
if (frontUrl != null) data["url"] = frontUrl;
|
|
1554
|
+
if (text.Length > 0) { data["text"] = text; data["text_source"] = "accessibility"; }
|
|
1555
|
+
}
|
|
1556
|
+
frameCount++;
|
|
1557
|
+
// ENG-4731: a frame that could not be checked (a window just opened answers slowly) is
|
|
1558
|
+
// looked at again on the next tick, not sent again as "unchanged" while the screen is still.
|
|
1559
|
+
if (visible == null) { lastDigest.Remove(screen.DeviceName); lastSent.Remove(screen.DeviceName); }
|
|
1560
|
+
else lastSent[screen.DeviceName] = new KeyValuePair<string[], Dictionary<string, object>>(isFront && frontExe != null ? new string[] { frontName ?? frontExe, frontExe } : null, data);
|
|
1561
|
+
Emit("screen", isFront, data);
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
void CheckDisplays()
|
|
1567
|
+
{
|
|
1568
|
+
Screen[] screens = Screen.AllScreens;
|
|
1569
|
+
if (screens.Length == displayCount) return;
|
|
1570
|
+
List<string> names = new List<string>();
|
|
1571
|
+
foreach (Screen s in screens) names.Add(s.DeviceName.Replace("\\\\.\\", ""));
|
|
1572
|
+
string joined = string.Join(", ", names.ToArray());
|
|
1573
|
+
Emit("displays", false, D("count", screens.Length, "names", joined));
|
|
1574
|
+
if (screens.Length > 1 && screens.Length > displayCount)
|
|
1575
|
+
Emit("flag", false, D("check", "extra_display", "detail", screens.Length + " displays attached: " + joined + "."));
|
|
1576
|
+
displayCount = screens.Length;
|
|
1577
|
+
CaptureScreens("displays changed");
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
bool remoteSeen;
|
|
1581
|
+
|
|
1582
|
+
/// Remote Desktop into this session leaves no process the watcher's scan could
|
|
1583
|
+
/// match, so the session itself is asked (SM_REMOTESESSION, SM_REMOTECONTROL).
|
|
1584
|
+
void CheckRemoteSession()
|
|
1585
|
+
{
|
|
1586
|
+
bool session = Native.GetSystemMetrics(0x1000) != 0, control = Native.GetSystemMetrics(0x2001) != 0;
|
|
1587
|
+
string detail = Decisions.RemoteSessionFlag(remoteSeen, session, control);
|
|
1588
|
+
if (detail != null) Emit("flag", false, D("check", "remote_access", "app", "Remote Desktop", "detail", detail));
|
|
1589
|
+
remoteSeen = session || control;
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
/// Windows that hide themselves from screen sharing (Cluely-style).
|
|
1593
|
+
void CheckHiddenWindows()
|
|
1594
|
+
{
|
|
1595
|
+
uint self = (uint)Process.GetCurrentProcess().Id;
|
|
1596
|
+
Native.EnumWindows(delegate (IntPtr hwnd, IntPtr lParam)
|
|
1597
|
+
{
|
|
1598
|
+
uint affinity;
|
|
1599
|
+
if (!Native.IsWindowVisible(hwnd) || !Native.GetWindowDisplayAffinity(hwnd, out affinity) || affinity == 0) return true;
|
|
1600
|
+
uint pid;
|
|
1601
|
+
Native.GetWindowThreadProcessId(hwnd, out pid);
|
|
1602
|
+
if (pid == self || !flaggedHidden.Add(pid)) return true;
|
|
1603
|
+
string name;
|
|
1604
|
+
string exe = ProcessExe(hwnd, out name) ?? "unknown";
|
|
1605
|
+
Native.RECT r;
|
|
1606
|
+
Native.GetWindowRect(hwnd, out r);
|
|
1607
|
+
Dictionary<string, object> e = D("kind", "flag", "ts", Now(), "app", name ?? exe, "app_id", exe, "data",
|
|
1608
|
+
D("check", "hidden_window", "app", name ?? exe, "detail", (name ?? exe) + " has a " + (r.Right - r.Left) + "x" + (r.Bottom - r.Top) + " window that hides itself from screen sharing (" + WindowTitle(hwnd) + ")."));
|
|
1609
|
+
Send(e);
|
|
1610
|
+
return true;
|
|
1611
|
+
}, IntPtr.Zero);
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
// ── Microphone ───────────────────────────────────────────────────────────
|
|
1615
|
+
|
|
1616
|
+
void StartAudio()
|
|
1617
|
+
{
|
|
1618
|
+
Native.WAVEFORMATEX format = new Native.WAVEFORMATEX();
|
|
1619
|
+
format.wFormatTag = 1; format.nChannels = 1; format.nSamplesPerSec = 16000; format.wBitsPerSample = 16;
|
|
1620
|
+
format.nBlockAlign = 2; format.nAvgBytesPerSec = 32000;
|
|
1621
|
+
if (Native.waveInOpen(out wave, -1, ref format, Handle, IntPtr.Zero, 0x10000) != 0) { wave = IntPtr.Zero; micDenied = true; return; } // CALLBACK_WINDOW
|
|
1622
|
+
for (int i = 0; i < 4; i++)
|
|
1623
|
+
{
|
|
1624
|
+
Native.WAVEHDR header = new Native.WAVEHDR();
|
|
1625
|
+
header.lpData = Marshal.AllocHGlobal(16000);
|
|
1626
|
+
header.dwBufferLength = 16000;
|
|
1627
|
+
IntPtr pointer = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(Native.WAVEHDR)));
|
|
1628
|
+
Marshal.StructureToPtr(header, pointer, false);
|
|
1629
|
+
Native.waveInPrepareHeader(wave, pointer, Marshal.SizeOf(typeof(Native.WAVEHDR)));
|
|
1630
|
+
Native.waveInAddBuffer(wave, pointer, Marshal.SizeOf(typeof(Native.WAVEHDR)));
|
|
1631
|
+
waveHeaders.Add(pointer);
|
|
1632
|
+
}
|
|
1633
|
+
Native.waveInStart(wave);
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
void OnAudio(IntPtr pointer)
|
|
1637
|
+
{
|
|
1638
|
+
if (wave == IntPtr.Zero) return;
|
|
1639
|
+
Native.WAVEHDR header = (Native.WAVEHDR)Marshal.PtrToStructure(pointer, typeof(Native.WAVEHDR));
|
|
1640
|
+
byte[] bytes = new byte[header.dwBytesRecorded];
|
|
1641
|
+
Marshal.Copy(header.lpData, bytes, 0, bytes.Length);
|
|
1642
|
+
audio.Write(bytes, 0, bytes.Length);
|
|
1643
|
+
Native.waveInAddBuffer(wave, pointer, Marshal.SizeOf(typeof(Native.WAVEHDR)));
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
/// The last chunk as a WAV, in memory only.
|
|
1647
|
+
void FlushAudio()
|
|
1648
|
+
{
|
|
1649
|
+
if (audio.Length < 3200) { audio.SetLength(0); return; }
|
|
1650
|
+
byte[] samples = audio.ToArray();
|
|
1651
|
+
audio.SetLength(0);
|
|
1652
|
+
MemoryStream wav = new MemoryStream();
|
|
1653
|
+
BinaryWriter w = new BinaryWriter(wav);
|
|
1654
|
+
w.Write(Encoding.ASCII.GetBytes("RIFF")); w.Write(36 + samples.Length); w.Write(Encoding.ASCII.GetBytes("WAVEfmt "));
|
|
1655
|
+
w.Write(16); w.Write((short)1); w.Write((short)1); w.Write(16000); w.Write(32000); w.Write((short)2); w.Write((short)16);
|
|
1656
|
+
w.Write(Encoding.ASCII.GetBytes("data")); w.Write(samples.Length); w.Write(samples); w.Flush();
|
|
1657
|
+
Emit("audio", false, D("audio", Convert.ToBase64String(wav.ToArray()), "format", "wav", "seconds", samples.Length / 32000.0));
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1660
|
+
/// Close the microphone and free every native buffer it was given.
|
|
1661
|
+
void StopAudio()
|
|
1662
|
+
{
|
|
1663
|
+
if (wave == IntPtr.Zero) return;
|
|
1664
|
+
IntPtr closing = wave;
|
|
1665
|
+
wave = IntPtr.Zero; // MM_WIM_DATA from the reset below is then ignored
|
|
1666
|
+
Native.waveInReset(closing);
|
|
1667
|
+
int size = Marshal.SizeOf(typeof(Native.WAVEHDR));
|
|
1668
|
+
foreach (IntPtr pointer in waveHeaders)
|
|
1669
|
+
{
|
|
1670
|
+
Native.waveInUnprepareHeader(closing, pointer, size);
|
|
1671
|
+
Native.WAVEHDR header = (Native.WAVEHDR)Marshal.PtrToStructure(pointer, typeof(Native.WAVEHDR));
|
|
1672
|
+
Marshal.FreeHGlobal(header.lpData);
|
|
1673
|
+
Marshal.FreeHGlobal(pointer);
|
|
1674
|
+
}
|
|
1675
|
+
waveHeaders.Clear();
|
|
1676
|
+
Native.waveInClose(closing);
|
|
1677
|
+
audio.SetLength(0);
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
public void Finish()
|
|
1681
|
+
{
|
|
1682
|
+
FlushBurst();
|
|
1683
|
+
if (wave != IntPtr.Zero) { FlushAudio(); StopAudio(); }
|
|
1684
|
+
Application.Exit();
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
static class Program
|
|
1689
|
+
{
|
|
1690
|
+
static string Arg(string[] args, string name)
|
|
1691
|
+
{
|
|
1692
|
+
int i = Array.IndexOf(args, name);
|
|
1693
|
+
return i >= 0 && i + 1 < args.Length ? args[i + 1] : null;
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
[STAThread]
|
|
1697
|
+
static int Main(string[] args)
|
|
1698
|
+
{
|
|
1699
|
+
if (Array.IndexOf(args, "--self-test") >= 0) return Decisions.SelfTest();
|
|
1700
|
+
string pipeName = Arg(args, "--pipe"), session = Arg(args, "--session");
|
|
1701
|
+
if (pipeName == null || session == null) return 0;
|
|
1702
|
+
pipeName = pipeName.Replace("\\\\.\\pipe\\", "");
|
|
1703
|
+
NamedPipeClientStream pipe = new NamedPipeClientStream(".", pipeName, PipeDirection.InOut, PipeOptions.Asynchronous);
|
|
1704
|
+
try { pipe.Connect(10000); } catch (Exception) { return 1; }
|
|
1705
|
+
Capture capture = new Capture(pipe);
|
|
1706
|
+
Dictionary<string, object> hello = new Dictionary<string, object>();
|
|
1707
|
+
hello["kind"] = "hello"; hello["session"] = session; hello["pid"] = Process.GetCurrentProcess().Id; hello["os"] = "windows"; hello["version"] = "0.1.0";
|
|
1708
|
+
capture.Send(hello);
|
|
1709
|
+
SynchronizationContext ui = new WindowsFormsSynchronizationContext();
|
|
1710
|
+
SynchronizationContext.SetSynchronizationContext(ui);
|
|
1711
|
+
Thread reader = new Thread(delegate ()
|
|
1712
|
+
{
|
|
1713
|
+
JavaScriptSerializer json = new JavaScriptSerializer { MaxJsonLength = int.MaxValue };
|
|
1714
|
+
StreamReader lines = new StreamReader(pipe, Encoding.UTF8);
|
|
1715
|
+
string line;
|
|
1716
|
+
try
|
|
1717
|
+
{
|
|
1718
|
+
while ((line = lines.ReadLine()) != null)
|
|
1719
|
+
{
|
|
1720
|
+
Dictionary<string, object> message;
|
|
1721
|
+
try { message = json.Deserialize<Dictionary<string, object>>(line); } catch (Exception) { continue; }
|
|
1722
|
+
object cmd;
|
|
1723
|
+
message.TryGetValue("cmd", out cmd);
|
|
1724
|
+
if ("config".Equals(cmd)) ui.Post(delegate { capture.Configure(message); }, null);
|
|
1725
|
+
else if ("stop".Equals(cmd)) break;
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1728
|
+
catch (Exception) { }
|
|
1729
|
+
// The watcher stopped us, or went away: either way, finish.
|
|
1730
|
+
ui.Post(delegate { capture.Finish(); }, null);
|
|
1731
|
+
});
|
|
1732
|
+
reader.IsBackground = true;
|
|
1733
|
+
reader.Start();
|
|
1734
|
+
Application.Run();
|
|
1735
|
+
try { pipe.Dispose(); } catch (Exception) { }
|
|
1736
|
+
return 0;
|
|
1737
|
+
}
|
|
1738
|
+
}
|