dsh-gui-control 0.0.0-stage → 0.1.0

package/lib/scripts/gui-daemon.ps1

@@ -0,0 +1,89 @@
+#requires -Version 5.1
+# Persistent GUI-control daemon.
+# Compiles the native helpers once, then serves ops over loopback TCP so each action
+# costs ~5 ms instead of spawning PowerShell + recompiling C# (~700 ms).
+# ASCII only.
+param(
+    [int]$Port = 38471,
+    [string]$Root = ''
+)
+
+$ErrorActionPreference = 'Stop'
+if ([string]::IsNullOrEmpty($Root)) { $Root = Split-Path -Parent $MyInvocation.MyCommand.Path }
+$Root = (Resolve-Path $Root).Path
+
+. (Join-Path $Root 'gui-ops.ps1')
+
+$logPath = Join-Path $Root 'gui-daemon.log'
+function Write-Log([string]$m) {
+    try { Add-Content -Path $logPath -Value ((Get-Date).ToString('MM-dd HH:mm:ss.fff') + '  ' + $m) -Encoding UTF8 } catch { }
+}
+
+try {
+    Import-DshGui $Root
+}
+catch {
+    Write-Log ('compile failed: ' + $_.Exception.Message)
+    throw
+}
+
+# UIA is optional: some environments cannot load the accessibility assemblies.
+try {
+    Import-DshUia $Root
+    Write-Log 'uia loaded'
+}
+catch {
+    Write-Log ('uia unavailable: ' + $_.Exception.Message)
+}
+
+$listener = New-Object System.Net.Sockets.TcpListener([System.Net.IPAddress]::Loopback, $Port)
+try { $listener.Server.SetSocketOption([System.Net.Sockets.SocketOptionLevel]::Socket, [System.Net.Sockets.SocketOptionName]::ReuseAddress, $true) } catch { }
+$listener.Start()
+Write-Log "listening on $Port (pid $PID)"
+
+$running = $true
+$served = 0
+
+while ($running) {
+    $client = $listener.AcceptTcpClient()
+    $client.NoDelay = $true
+    $client.ReceiveTimeout = 120000
+    try {
+        $stream = $client.GetStream()
+        $reader = New-Object System.IO.StreamReader($stream, (New-Object System.Text.UTF8Encoding($false)))
+        $writer = New-Object System.IO.StreamWriter($stream, (New-Object System.Text.UTF8Encoding($false)))
+        $writer.AutoFlush = $true
+        while ($true) {
+            $line = $reader.ReadLine()
+            if ($null -eq $line) { break }
+            if ($line.Trim().Length -eq 0) { continue }
+            $op = $null
+            $resp = $null
+            try {
+                $op = $line | ConvertFrom-Json
+                $resp = Invoke-GuiOp $op
+            }
+            catch {
+                $resp = @{ ok = $false; error = $_.Exception.Message }
+            }
+            $served++
+            try {
+                $writer.WriteLine((ConvertTo-Json $resp -Compress -Depth 8))
+            }
+            catch {
+                Write-Log ('write failed: ' + $_.Exception.Message)
+                break
+            }
+            if ($null -ne $op -and $op.op -eq 'quit') { $running = $false; break }
+        }
+    }
+    catch {
+        Write-Log ('connection error: ' + $_.Exception.Message)
+    }
+    finally {
+        try { $client.Close() } catch { }
+    }
+}
+
+Write-Log "stopped after $served ops"
+$listener.Stop()

package/lib/scripts/gui-lib.ps1

@@ -0,0 +1,68 @@
+#requires -Version 5.1
+# Client helpers for the persistent daemon. Dot-source this, then call G/GStart.
+# ASCII only.
+#
+#   . C:\path\to\gui-control\gui-lib.ps1
+#   G @{ op = 'screen' }
+#   G @{ op = 'click'; win = 'bilibili'; x = 120; y = 40 }
+#   G @{ op = 'findtext'; win = 'bilibili'; text = '第153话' }
+
+$script:GuiRoot = $PSScriptRoot
+$script:GuiPort = 38471
+
+function Test-GuiDaemon {
+    try {
+        $c = New-Object System.Net.Sockets.TcpClient
+        $c.Connect('127.0.0.1', $script:GuiPort)
+        $c.Close()
+        return $true
+    }
+    catch { return $false }
+}
+
+function Start-GuiDaemon {
+    if (Test-GuiDaemon) { return 'already running' }
+    $daemon = Join-Path $script:GuiRoot 'gui-daemon.ps1'
+    Start-Process -FilePath 'powershell.exe' `
+        -ArgumentList @('-NoProfile', '-STA', '-ExecutionPolicy', 'Bypass', '-WindowStyle', 'Hidden', '-File', $daemon, '-Port', $script:GuiPort) `
+        -WindowStyle Hidden | Out-Null
+    for ($i = 0; $i -lt 100; $i++) {
+        Start-Sleep -Milliseconds 100
+        if (Test-GuiDaemon) { return 'started' }
+    }
+    throw 'daemon did not come up (see gui-daemon.log)'
+}
+
+function Stop-GuiDaemon {
+    try { [void](Send-GuiRaw @{ op = 'quit' }) } catch { }
+}
+
+function Send-GuiRaw($op) {
+    $json = ($op | ConvertTo-Json -Compress -Depth 8)
+    $client = New-Object System.Net.Sockets.TcpClient
+    $client.NoDelay = $true
+    $client.Connect('127.0.0.1', $script:GuiPort)
+    try {
+        $stream = $client.GetStream()
+        $writer = New-Object System.IO.StreamWriter($stream, (New-Object System.Text.UTF8Encoding($false)))
+        $writer.AutoFlush = $true
+        $reader = New-Object System.IO.StreamReader($stream, (New-Object System.Text.UTF8Encoding($false)))
+        $writer.WriteLine($json)
+        $line = $reader.ReadLine()
+        if ($null -eq $line) { throw 'daemon closed the connection' }
+        return ($line | ConvertFrom-Json)
+    }
+    finally { $client.Close() }
+}
+
+# Main entry: runs one op, throws on failure, returns the raw result payload.
+function G($op) {
+    $r = Send-GuiRaw $op
+    if (-not $r.ok) { throw $r.error }
+    return $r.result
+}
+
+# Batch: run an array of ops in ONE round-trip.
+function GBatch([object[]]$ops) {
+    return (G @{ op = 'batch'; ops = $ops })
+}

package/lib/scripts/gui-ocr.ps1

@@ -0,0 +1,71 @@
+#requires -Version 5.1
+# Windows.Media.Ocr wrapper for PowerShell 5.1 (WinRT interop).
+# Gives semantic targeting: find a control by its text instead of guessing pixels.
+# ASCII only.
+
+$script:OcrReady = $false
+$script:OcrEngine = $null
+$script:AsTask = $null
+
+function Initialize-Ocr {
+    if ($script:OcrReady) { return }
+    Add-Type -AssemblyName System.Runtime.WindowsRuntime -ErrorAction Stop
+    $null = [Windows.Media.Ocr.OcrEngine, Windows.Foundation, ContentType = WindowsRuntime]
+    $null = [Windows.Storage.StorageFile, Windows.Foundation, ContentType = WindowsRuntime]
+    $null = [Windows.Graphics.Imaging.BitmapDecoder, Windows.Foundation, ContentType = WindowsRuntime]
+
+    $script:AsTask = ([System.WindowsRuntimeSystemExtensions].GetMethods() | Where-Object {
+            $_.Name -eq 'AsTask' -and $_.GetParameters().Count -eq 1 -and
+            $_.GetParameters()[0].ParameterType.Name -eq 'IAsyncOperation`1'
+        })[0]
+    if ($null -eq $script:AsTask) { throw 'AsTask(IAsyncOperation<T>) not found' }
+
+    $script:OcrEngine = [Windows.Media.Ocr.OcrEngine]::TryCreateFromUserProfileLanguages()
+    if ($null -eq $script:OcrEngine) { throw 'No OCR engine for user profile languages' }
+    $script:OcrReady = $true
+}
+
+function Await-WinRt($op, [Type]$type) {
+    $m = $script:AsTask.MakeGenericMethod($type)
+    $t = $m.Invoke($null, @($op))
+    if (-not $t.Wait(20000)) { throw 'WinRT await timeout' }
+    if ($t.IsFaulted) { throw $t.Exception.GetBaseException().Message }
+    return $t.Result
+}
+
+# Returns an array of @{ text; x; y; w; h } using absolute screen coordinates.
+# $offsetX/$offsetY are the screen position of the captured image's top-left corner.
+function Get-OcrLines([string]$path, [int]$offsetX, [int]$offsetY) {
+    Initialize-Ocr
+    $file = Await-WinRt ([Windows.Storage.StorageFile]::GetFileFromPathAsync($path)) ([Windows.Storage.StorageFile])
+    $stream = Await-WinRt ($file.OpenAsync([Windows.Storage.FileAccessMode]::Read)) ([Windows.Storage.Streams.IRandomAccessStream])
+    $decoder = Await-WinRt ([Windows.Graphics.Imaging.BitmapDecoder]::CreateAsync($stream)) ([Windows.Graphics.Imaging.BitmapDecoder])
+    $bmp = Await-WinRt ($decoder.GetSoftwareBitmapAsync()) ([Windows.Graphics.Imaging.SoftwareBitmap])
+    $res = Await-WinRt ($script:OcrEngine.RecognizeAsync($bmp)) ([Windows.Media.Ocr.OcrResult])
+
+    $lines = @()
+    foreach ($line in @($res.Lines)) {
+        $words = @($line.Words)
+        if ($words.Count -eq 0) { continue }
+        $minX = [double]::MaxValue; $minY = [double]::MaxValue
+        $maxX = 0.0; $maxY = 0.0
+        foreach ($wd in $words) {
+            $r = $wd.BoundingRect
+            if ($r.X -lt $minX) { $minX = $r.X }
+            if ($r.Y -lt $minY) { $minY = $r.Y }
+            if (($r.X + $r.Width) -gt $maxX) { $maxX = $r.X + $r.Width }
+            if (($r.Y + $r.Height) -gt $maxY) { $maxY = $r.Y + $r.Height }
+        }
+        $lines += @{
+            text = $line.Text
+            x    = [int]($offsetX + $minX)
+            y    = [int]($offsetY + $minY)
+            w    = [int]($maxX - $minX)
+            h    = [int]($maxY - $minY)
+            cx   = [int]($offsetX + ($minX + $maxX) / 2)
+            cy   = [int]($offsetY + ($minY + $maxY) / 2)
+        }
+    }
+    $stream.Dispose()
+    return $lines
+}