dsh-spectacles 0.0.0-stage → 1.1.2

package/package.json

@@ -1,6 +1,65 @@
 {
   "name": "dsh-spectacles",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "1.1.2",
+  "description": "Point at one window and let the agent read it clearly. Window scoped screen reading for DeepSeek Harness: look (numbers first), read_text (OCR), see (crop + zoom handed to the host vision).",
+  "license": "Apache-2.0",
+  "type": "module",
+  "main": "index.js",
+  "os": [
+    "win32"
+  ],
+  "exports": {
+    ".": "./index.js",
+    "./client": "./client.js",
+    "./cordis.patch.yml": "./cordis.patch.yml",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "index.js",
+    "client.js",
+    "cordis.patch.yml",
+    "scripts/",
+    "README.md",
+    "README.zh.md",
+    "CHANGELOG.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "deepseek",
+    "deepseek-harness",
+    "dsh",
+    "dsh-plugin",
+    "plugin",
+    "vision",
+    "look",
+    "spectacles",
+    "screenshot",
+    "ocr",
+    "window-capture"
+  ],
+  "dsh": {
+    "manifestVersion": 1,
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    },
+    "client": {
+      "platform": "web",
+      "inject": []
+    }
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/yanming119/dsh-spectacles.git"
+  },
+  "homepage": "https://github.com/yanming119/dsh-spectacles",
+  "bugs": {
+    "url": "https://github.com/yanming119/dsh-spectacles/issues"
+  },
+  "author": "yanming119",
+  "publishConfig": {
+    "access": "public"
+  }
 }

package/scripts/look-pick.ps1

@@ -0,0 +1,83 @@
+param(
+  [int]$TimeoutSec = 30,
+  [string]$Out = ""
+)
+$DataDir = if ($env:DSH_LOOK_DATA) { $env:DSH_LOOK_DATA } else { Join-Path $env:TEMP 'dsh-look' }
+if (-not (Test-Path $DataDir)) { New-Item -ItemType Directory -Force -Path $DataDir | Out-Null }
+if (-not $Out) { $Out = Join-Path $DataDir 'picked.json' }
+try { [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 } catch { }
+$ErrorActionPreference = 'Stop'
+
+Add-Type -TypeDefinition @"
+using System;
+using System.Text;
+using System.Runtime.InteropServices;
+
+public class Pk3 {
+  [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p);
+  [DllImport("user32.dll")] public static extern IntPtr WindowFromPoint(POINT p);
+  [DllImport("user32.dll")] public static extern IntPtr GetAncestor(IntPtr h, uint flags);
+  [DllImport("user32.dll", CharSet=CharSet.Unicode)] public static extern int GetWindowText(IntPtr h, StringBuilder s, int n);
+  [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
+  [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid);
+  [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
+  [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h);
+
+  public struct POINT { public int X, Y; }
+  public struct RECT { public int Left, Top, Right, Bottom; }
+}
+"@
+
+function Get-Title($h) {
+  $sb = New-Object System.Text.StringBuilder 512
+  [Pk3]::GetWindowText($h, $sb, 512) | Out-Null
+  return $sb.ToString()
+}
+
+function Emit($obj) {
+  $json = $obj | ConvertTo-Json -Compress
+  [System.IO.File]::WriteAllText($Out, $json, (New-Object System.Text.UTF8Encoding($false)))
+  Write-Output $json
+}
+
+$start = [Pk3]::GetForegroundWindow()
+$deadline = (Get-Date).AddSeconds($TimeoutSec)
+
+while ((Get-Date) -lt $deadline) {
+  $fg = [Pk3]::GetForegroundWindow()
+  if ($fg -ne [IntPtr]::Zero -and $fg -ne $start -and [Pk3]::IsWindowVisible($fg) -and (Get-Title $fg).Length -gt 0) {
+    # 真正判据：鼠标此刻就在这个窗口上——说明是用户点它，不是别的窗口自己冒出来。
+    $pt = New-Object Pk3+POINT
+    [Pk3]::GetCursorPos([ref]$pt) | Out-Null
+    $under = [Pk3]::WindowFromPoint($pt)
+    $root = if ($under -ne [IntPtr]::Zero) { [Pk3]::GetAncestor($under, 2) } else { [IntPtr]::Zero }
+    if ($root -eq $fg) {
+      # 别把宿主自己（DeepSeek Harness 的窗口）当成目标——它自己会冒到前台
+      $procId = 0
+      [Pk3]::GetWindowThreadProcessId($fg, [ref]$procId) | Out-Null
+      $pname = ''
+      try { $pname = (Get-Process -Id ([int]$procId) -ErrorAction Stop).ProcessName } catch { $pname = '' }
+      if ($pname -like '*Harness*' -or (Get-Title $fg) -like '*DeepSeek Harness*') {
+        Start-Sleep -Milliseconds 60
+        continue
+      }
+      $r = New-Object Pk3+RECT
+      [Pk3]::GetWindowRect($fg, [ref]$r) | Out-Null
+      $procId = 0
+      [Pk3]::GetWindowThreadProcessId($fg, [ref]$procId) | Out-Null
+      Emit ([ordered]@{
+        ok    = $true
+        when  = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss.fff')
+        hwnd  = [int]$fg
+        title = (Get-Title $fg)
+        pid   = [int]$procId
+        rect  = @([int]$r.Left, [int]$r.Top, [int]($r.Right - $r.Left), [int]($r.Bottom - $r.Top))
+      })
+      exit 0
+    }
+  }
+  Start-Sleep -Milliseconds 60
+}
+
+Emit ([ordered]@{ ok = $false; error = 'timeout: no click' })
+exit 1

package/scripts/look-region.ps1

@@ -0,0 +1,195 @@
+param(
+  [int]$TimeoutSec = 60,
+  [string]$Out = ""
+)
+$DataDir = if ($env:DSH_LOOK_DATA) { $env:DSH_LOOK_DATA } else { Join-Path $env:TEMP 'dsh-look' }
+if (-not (Test-Path $DataDir)) { New-Item -ItemType Directory -Force -Path $DataDir | Out-Null }
+if (-not $Out) { $Out = Join-Path $DataDir 'region.json' }
+[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
+$ErrorActionPreference = 'Stop'
+
+Add-Type -AssemblyName System.Windows.Forms
+Add-Type -AssemblyName System.Drawing
+Add-Type -ReferencedAssemblies System.Drawing -TypeDefinition @"
+using System;
+using System.Text;
+using System.Runtime.InteropServices;
+
+public class Rg {
+  [DllImport("user32.dll")] public static extern bool SetProcessDPIAware();
+  [DllImport("user32.dll")] public static extern IntPtr WindowFromPoint(POINT p);
+  [DllImport("user32.dll")] public static extern IntPtr GetAncestor(IntPtr h, uint flags);
+  [DllImport("user32.dll", CharSet=CharSet.Unicode)] public static extern int GetWindowText(IntPtr h, StringBuilder s, int n);
+  [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
+  [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid);
+
+  public struct POINT { public int X, Y; }
+  public struct RECT { public int Left, Top, Right, Bottom; }
+}
+"@
+[void][Rg]::SetProcessDPIAware()
+
+$vs = [System.Windows.Forms.SystemInformation]::VirtualScreen
+
+# 不抓屏幕：这层遮罩只是一块半透明的、能透过它看见下面东西的玻璃，
+# 用来接你的拖动、记下矩形坐标。框选本身不产生任何画面内容。
+$script:W = $vs.Width
+$script:H = $vs.Height
+$script:sel = $null
+$script:dragging = $false
+$script:done = $false
+$script:cancelled = $false
+$script:start = $null
+$script:pen = New-Object System.Drawing.Pen ([System.Drawing.Color]::FromArgb(255, 0, 205, 120)), 2
+$script:penDot = New-Object System.Drawing.Pen ([System.Drawing.Color]::FromArgb(200, 255, 255, 255)), 1
+$script:font = New-Object System.Drawing.Font 'Microsoft YaHei UI', 12, ([System.Drawing.FontStyle]::Bold)
+$script:brText = New-Object System.Drawing.SolidBrush ([System.Drawing.Color]::White)
+$script:brBox = New-Object System.Drawing.SolidBrush ([System.Drawing.Color]::FromArgb(215, 20, 24, 30))
+
+$form = New-Object System.Windows.Forms.Form
+$form.FormBorderStyle = [System.Windows.Forms.FormBorderStyle]::None
+$form.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual
+$form.Bounds = New-Object System.Drawing.Rectangle $vs.Left, $vs.Top, $vs.Width, $vs.Height
+$form.TopMost = $true
+$form.ShowInTaskbar = $false
+$form.Cursor = [System.Windows.Forms.Cursors]::Cross
+$form.KeyPreview = $true
+$form.BackColor = [System.Drawing.Color]::Black
+$form.Opacity = 0.22
+try { $form.GetType().GetProperty('DoubleBuffered', 'Instance,NonPublic').SetValue($form, $true) } catch {}
+
+function Draw-Tag($g, $text, $x, $y) {
+  $sz = $g.MeasureString($text, $script:font)
+  $w = [int]$sz.Width + 16; $h = [int]$sz.Height + 8
+  $bx = [Math]::Max(0, [Math]::Min($x, $script:W - $w))
+  $by = [Math]::Max(0, [Math]::Min($y, $script:H - $h))
+  $g.FillRectangle($script:brBox, $bx, $by, $w, $h)
+  $g.DrawString($text, $script:font, $script:brText, ($bx + 8), ($by + 4))
+}
+
+$form.Add_Paint({
+    param($s, $e)
+    $g = $e.Graphics
+    $r = $script:sel
+    if ($r -and $r.Width -gt 0 -and $r.Height -gt 0) {
+      $g.DrawRectangle($script:pen, $r)
+      Draw-Tag $g ("{0} x {1}" -f $r.Width, $r.Height) ($r.Left + 6) ($r.Top + 6)
+    }
+    else {
+      Draw-Tag $g '拖一个框；Esc 取消' 40 40
+    }
+  })
+
+$form.Add_MouseDown({
+    param($s, $e)
+    if ($e.Button -eq [System.Windows.Forms.MouseButtons]::Left) {
+      $script:start = $e.Location
+      $script:sel = New-Object System.Drawing.Rectangle $e.X, $e.Y, 0, 0
+      $script:dragging = $true
+      $s.Invalidate()
+    }
+    elseif ($e.Button -eq [System.Windows.Forms.MouseButtons]::Right) {
+      $script:cancelled = $true
+      $s.Close()
+    }
+  })
+
+$form.Add_MouseMove({
+    param($s, $e)
+    if (-not $script:dragging) { return }
+    $x1 = [Math]::Min($script:start.X, $e.X); $y1 = [Math]::Min($script:start.Y, $e.Y)
+    $x2 = [Math]::Max($script:start.X, $e.X); $y2 = [Math]::Max($script:start.Y, $e.Y)
+    $script:sel = New-Object System.Drawing.Rectangle $x1, $y1, ($x2 - $x1), ($y2 - $y1)
+    $s.Invalidate()
+  })
+
+$closeTimer = New-Object System.Windows.Forms.Timer
+$closeTimer.Interval = 160
+$closeTimer.Add_Tick({ $closeTimer.Stop(); $form.Close() })
+
+$form.Add_MouseUp({
+    param($s, $e)
+    if (-not $script:dragging) { return }
+    $script:dragging = $false
+    if ($script:sel -and $script:sel.Width -ge 8 -and $script:sel.Height -ge 8) {
+      $script:done = $true
+      # 先把遮罩藏掉（人立刻看见框定住了），窗口晚 160ms 再关：
+      # 松手这一下要是当场关窗，抬键消息会漏给下面的窗口，有的程序会当成一次点击。
+      $s.Visible = $false
+      $closeTimer.Start()
+    }
+  })
+
+$form.Add_KeyDown({
+    param($s, $e)
+    if ($e.KeyCode -eq [System.Windows.Forms.Keys]::Escape) {
+      $script:cancelled = $true
+      $s.Close()
+    }
+  })
+
+$form.Add_Shown({ $form.Activate() })
+
+$timer = New-Object System.Windows.Forms.Timer
+$timer.Interval = [Math]::Max(1000, $TimeoutSec * 1000)
+$timer.Add_Tick({ $script:cancelled = $true; $form.Close() })
+$timer.Start()
+
+[System.Windows.Forms.Application]::Run($form)
+$timer.Stop()
+
+if (-not $script:done -or -not $script:sel) {
+  @{ ok = $false; error = 'cancelled' } | ConvertTo-Json -Compress
+  exit 1
+}
+
+# 等遮罩真的消失，不然 WindowFromPoint 会点到我们自己
+Start-Sleep -Milliseconds 90
+
+$sel = $script:sel
+$sx = $vs.Left + $sel.X
+$sy = $vs.Top + $sel.Y
+
+$pt = New-Object Rg+POINT
+$pt.X = $sx + [int]($sel.Width / 2)
+$pt.Y = $sy + [int]($sel.Height / 2)
+$h = [Rg]::WindowFromPoint($pt)
+if ($h -eq [IntPtr]::Zero) { @{ ok = $false; error = 'no window under the box' } | ConvertTo-Json -Compress; exit 1 }
+$root = [Rg]::GetAncestor($h, 2)
+if ($root -eq [IntPtr]::Zero) { $root = $h }
+
+$sb = New-Object System.Text.StringBuilder 512
+[Rg]::GetWindowText($root, $sb, 512) | Out-Null
+if ($sb.Length -eq 0) {
+  @{ ok = $false; error = '那个位置没有窗口（是桌面空白处），换个地方再框' } | ConvertTo-Json -Compress
+  exit 1
+}
+$wr = New-Object Rg+RECT
+[Rg]::GetWindowRect($root, [ref]$wr) | Out-Null
+$procId = 0
+[Rg]::GetWindowThreadProcessId($root, [ref]$procId) | Out-Null
+
+# 框可能越出窗口：裁到窗口里面，换算成相对窗口左上角
+$ix = [Math]::Max($sx, $wr.Left); $iy = [Math]::Max($sy, $wr.Top)
+$ir = [Math]::Min($sx + $sel.Width, $wr.Right); $ib = [Math]::Min($sy + $sel.Height, $wr.Bottom)
+$cw = [Math]::Max(1, $ir - $ix)
+$ch = [Math]::Max(1, $ib - $iy)
+$cx = $ix - $wr.Left
+$cy = $iy - $wr.Top
+
+$res = [ordered]@{
+  ok     = $true
+  when   = (Get-Date -Format 'yyyy-MM-dd HH:mm:ss.fff')
+  hwnd   = [int64]$root
+  title  = $sb.ToString()
+  pid    = $procId
+  rect   = @($wr.Left, $wr.Top, ($wr.Right - $wr.Left), ($wr.Bottom - $wr.Top))
+  screen = @($sx, $sy, $sel.Width, $sel.Height)
+  crop   = @($cx, $cy, $cw, $ch)
+}
+
+$dir = Split-Path $Out -Parent
+if ($dir -and -not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null }
+[System.IO.File]::WriteAllText($Out, ($res | ConvertTo-Json -Compress), (New-Object System.Text.UTF8Encoding($false)))
+
+$res | ConvertTo-Json -Compress

package/scripts/look.ps1

@@ -0,0 +1,133 @@
+param(
+  [string]$Crop = "",
+  [double]$Scale = 1.0,
+  [switch]$Image,
+  [string]$Out = "",
+  [string]$SaveDir = "",
+  [int]$TimeoutMs = 6000
+)
+$DataDir = if ($env:DSH_LOOK_DATA) { $env:DSH_LOOK_DATA } else { Join-Path $env:TEMP 'dsh-look' }
+if (-not (Test-Path $DataDir)) { New-Item -ItemType Directory -Force -Path $DataDir | Out-Null }
+if (-not $SaveDir) { $SaveDir = $DataDir }
+
+$ErrorActionPreference = 'Stop'
+Add-Type -AssemblyName System.Drawing
+
+# fullscreen capture defaults to GDI: fast (~200ms) and needs no external binary.
+# set DSH_LOOK_FFMPEG only to force the ffmpeg/DXGI path (slower, but sees some hardware overlays).
+$ffmpeg = $env:DSH_LOOK_FFMPEG
+if ($ffmpeg -and -not (Test-Path $ffmpeg)) { $ffmpeg = '' }
+
+if (-not (Test-Path $SaveDir)) { New-Item -ItemType Directory -Path $SaveDir -Force | Out-Null }
+$stamp = Get-Date -Format 'yyyyMMdd-HHmmss-fff'
+$tmp = Join-Path $SaveDir "raw-$stamp.png"
+$stateFile = Join-Path $SaveDir 'last-grid.txt'
+
+$sw = [Diagnostics.Stopwatch]::StartNew()
+$source = 'gdi-copyfromscreen'
+if ($ffmpeg) {
+  $source = 'dxgi-ddagrabb'
+  & $ffmpeg -hide_banner -loglevel error -f lavfi -i "ddagrab=0:framerate=1" -vf "hwdownload,format=bgra" -frames:v 1 -y $tmp 2>$null | Out-Null
+} else {
+  # GDI screen copy (DPI-aware, no external binary)
+  if (-not ('YM.Dpi' -as [type])) {
+    Add-Type -Namespace YM -Name Dpi -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetProcessDPIAware(); [DllImport("user32.dll")] public static extern int GetSystemMetrics(int n);'
+  }
+  [void][YM.Dpi]::SetProcessDPIAware()
+  $sw0 = [YM.Dpi]::GetSystemMetrics(0); $sh0 = [YM.Dpi]::GetSystemMetrics(1)
+  $shot = New-Object System.Drawing.Bitmap $sw0, $sh0
+  $g0 = [System.Drawing.Graphics]::FromImage($shot)
+  $g0.CopyFromScreen(0, 0, 0, 0, (New-Object System.Drawing.Size $sw0, $sh0))
+  $g0.Dispose()
+  $shot.Save($tmp, [System.Drawing.Imaging.ImageFormat]::Png)
+  $shot.Dispose()
+}
+$sw.Stop()
+if (-not (Test-Path $tmp)) { Write-Output '{"ok":false,"error":"capture failed"}'; exit 1 }
+
+$src = [System.Drawing.Bitmap]::FromFile($tmp)
+$screenW = $src.Width
+$screenH = $src.Height
+
+$x = 0; $y = 0; $w = $screenW; $h = $screenH
+if ($Crop) {
+  $p = $Crop -split ','
+  if ($p.Length -eq 4) {
+    $x = [int]$p[0]; $y = [int]$p[1]; $w = [int]$p[2]; $h = [int]$p[3]
+    if ($x -lt 0) { $x = 0 }
+    if ($y -lt 0) { $y = 0 }
+    if (($x + $w) -gt $screenW) { $w = $screenW - $x }
+    if (($y + $h) -gt $screenH) { $h = $screenH - $y }
+  }
+}
+
+function New-StatBitmap($bmp, $tw, $th) {
+  $small = New-Object System.Drawing.Bitmap $tw, $th
+  $g = [System.Drawing.Graphics]::FromImage($small)
+  $g.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBilinear
+  $g.DrawImage($bmp, (New-Object System.Drawing.Rectangle 0, 0, $tw, $th), (New-Object System.Drawing.Rectangle $x, $y, $w, $h), [System.Drawing.GraphicsUnit]::Pixel)
+  $g.Dispose()
+  return $small
+}
+
+$stat = New-StatBitmap $src 64 36
+$gw = $stat.Width; $gh = $stat.Height
+$gray = New-Object 'int[]' ($gw * $gh)
+$sumR = 0; $sumG = 0; $sumB = 0
+for ($yy = 0; $yy -lt $gh; $yy++) {
+  for ($xx = 0; $xx -lt $gw; $xx++) {
+    $c = $stat.GetPixel($xx, $yy)
+    $sumR += $c.R; $sumG += $c.G; $sumB += $c.B
+    $gray[$yy * $gw + $xx] = [int](($c.R * 0.299) + ($c.G * 0.587) + ($c.B * 0.114))
+  }
+}
+$n = $gw * $gh
+$avgR = [int]($sumR / $n); $avgG = [int]($sumG / $n); $avgB = [int]($sumB / $n)
+
+$edges = 0
+for ($yy = 1; $yy -lt $gh - 1; $yy++) {
+  for ($xx = 1; $xx -lt $gw - 1; $xx++) {
+    $i = $yy * $gw + $xx
+    $dx = [Math]::Abs($gray[$i + 1] - $gray[$i - 1])
+    $dy = [Math]::Abs($gray[$i + $gw] - $gray[$i - $gw])
+    if (($dx + $dy) -gt 60) { $edges++ }
+  }
+}
+$edgeDensity = [Math]::Round($edges / $n, 4)
+
+$cur = ($gray | ForEach-Object { if ($_ -lt 10) { '00' } elseif ($_ -lt 100) { '0' + $_ } else { "$_" } }) -join ''
+$changed = $null
+if (Test-Path $stateFile) { $prev = Get-Content $stateFile -Raw; if ($prev.Trim() -eq $cur) { $changed = $false } else { $changed = $true } }
+Set-Content -Path $stateFile -Value $cur -NoNewline
+
+$stat.Dispose()
+
+$pngPath = ''
+if ($Image -or $Out) {
+  $target = New-Object System.Drawing.Bitmap ([int]($w * $Scale)), ([int]($h * $Scale))
+  $g2 = [System.Drawing.Graphics]::FromImage($target)
+  $g2.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic
+  $g2.DrawImage($src, (New-Object System.Drawing.Rectangle 0, 0, $target.Width, $target.Height), (New-Object System.Drawing.Rectangle $x, $y, $w, $h), [System.Drawing.GraphicsUnit]::Pixel)
+  $g2.Dispose()
+  if ($Out) { $pngPath = $Out } else { $pngPath = Join-Path $SaveDir "look-$stamp.png" }
+  $target.Save($pngPath, [System.Drawing.Imaging.ImageFormat]::Png)
+  $target.Dispose()
+}
+
+$src.Dispose()
+Remove-Item $tmp -Force -ErrorAction SilentlyContinue
+
+$obj = [ordered]@{
+  ok         = $true
+  when       = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss.fff')
+  source     = $source
+  screen     = @{ w = $screenW; h = $screenH }
+  region     = @{ x = $x; y = $y; w = $w; h = $h }
+  captureMs  = $sw.ElapsedMilliseconds
+  avgColor   = ('#{0:X2}{1:X2}{2:X2}' -f $avgR, $avgG, $avgB)
+  edgeDensity = $edgeDensity
+  changed    = $changed
+  grid       = "$gw" + 'x' + "$gh"
+  png        = $pngPath
+}
+$obj | ConvertTo-Json -Compress -Depth 4