@geravant/sinain 1.0.18 → 1.1.0

package/sense_client/app_detector.py

@@ -0,0 +1,54 @@
+"""Detect the frontmost application and window title (cross-platform)."""
+
+import subprocess
+import sys
+
+
+class MacAppDetector:
+    """Detects the frontmost application and window title on macOS via AppleScript."""
+
+    def __init__(self):
+        self._last_app: str = ""
+        self._last_window: str = ""
+
+    def get_active_app(self) -> tuple[str, str]:
+        """Returns (app_name, window_title) of the frontmost application."""
+        try:
+            result = subprocess.run(
+                [
+                    "osascript", "-e",
+                    'tell application "System Events"\n'
+                    '  set appProc to first application process whose frontmost is true\n'
+                    '  set appName to name of appProc\n'
+                    '  set winTitle to ""\n'
+                    '  try\n'
+                    '    set winTitle to name of front window of appProc\n'
+                    '  end try\n'
+                    '  return appName & "|||" & winTitle\n'
+                    'end tell',
+                ],
+                capture_output=True, text=True, timeout=2,
+            )
+            parts = result.stdout.strip().split("|||", 1)
+            app_name = parts[0].strip() if parts else ""
+            window_title = parts[1].strip() if len(parts) > 1 else ""
+            return app_name, window_title
+        except Exception:
+            return "", ""
+
+    def detect_change(self) -> tuple[bool, bool, str, str]:
+        """Returns (app_changed, window_changed, app_name, window_title)."""
+        app, window = self.get_active_app()
+        app_changed = app != self._last_app and self._last_app != ""
+        window_changed = window != self._last_window and self._last_window != ""
+        self._last_app = app
+        self._last_window = window
+        return app_changed, window_changed, app, window
+
+
+def AppDetector():
+    """Factory: returns the platform-appropriate app detector."""
+    if sys.platform == "win32":
+        from .app_detector_win import WinAppDetector
+        return WinAppDetector()
+    return MacAppDetector()

package/sense_client/app_detector_win.py

@@ -0,0 +1,83 @@
+"""Detect the foreground application and window title on Windows."""
+from __future__ import annotations
+
+import ctypes
+import ctypes.wintypes
+
+
+class WinAppDetector:
+    """Detects the foreground application and window title on Windows.
+
+    Uses Win32 API: GetForegroundWindow, GetWindowTextW, and
+    psutil/GetWindowThreadProcessId for process name resolution.
+    """
+
+    def __init__(self):
+        self._last_app: str = ""
+        self._last_window: str = ""
+        self._user32 = ctypes.windll.user32
+        self._kernel32 = ctypes.windll.kernel32
+
+    def get_active_app(self) -> tuple[str, str]:
+        """Returns (app_name, window_title) of the foreground window."""
+        try:
+            hwnd = self._user32.GetForegroundWindow()
+            if not hwnd:
+                return "", ""
+
+            # Get window title
+            length = self._user32.GetWindowTextLengthW(hwnd)
+            buf = ctypes.create_unicode_buffer(length + 1)
+            self._user32.GetWindowTextW(hwnd, buf, length + 1)
+            window_title = buf.value
+
+            # Get process name via PID
+            pid = ctypes.wintypes.DWORD()
+            self._user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid))
+            app_name = self._get_process_name(pid.value)
+
+            return app_name, window_title
+        except Exception:
+            return "", ""
+
+    def _get_process_name(self, pid: int) -> str:
+        """Get process executable name from PID."""
+        try:
+            # Try psutil first (more reliable)
+            import psutil
+            proc = psutil.Process(pid)
+            return proc.name().replace(".exe", "")
+        except Exception:
+            pass
+
+        # Fallback: OpenProcess + GetModuleBaseNameW
+        try:
+            PROCESS_QUERY_INFORMATION = 0x0400
+            PROCESS_VM_READ = 0x0010
+            handle = self._kernel32.OpenProcess(
+                PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, False, pid
+            )
+            if not handle:
+                return ""
+
+            try:
+                psapi = ctypes.windll.psapi
+                buf = ctypes.create_unicode_buffer(260)
+                psapi.GetModuleBaseNameW(handle, None, buf, 260)
+                name = buf.value
+                if name.lower().endswith(".exe"):
+                    name = name[:-4]
+                return name
+            finally:
+                self._kernel32.CloseHandle(handle)
+        except Exception:
+            return ""
+
+    def detect_change(self) -> tuple[bool, bool, str, str]:
+        """Returns (app_changed, window_changed, app_name, window_title)."""
+        app, window = self.get_active_app()
+        app_changed = app != self._last_app and self._last_app != ""
+        window_changed = window != self._last_window and self._last_window != ""
+        self._last_app = app
+        self._last_window = window
+        return app_changed, window_changed, app, window

package/sense_client/capture.py

@@ -0,0 +1,215 @@
+"""Screen capture via IPC from sck-capture (preferred) or CoreGraphics fallback.
+
+On Windows, uses mss (DXGI Desktop Duplication) via capture_win.py.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+from typing import Generator
+
+from PIL import Image
+
+# macOS-only imports — deferred so the module loads on Windows
+if sys.platform == "darwin":
+    import Quartz
+
+
+class ScreenCapture:
+    """Captures screen frames via CGDisplayCreateImage (CoreGraphics/IOSurface).
+
+    Uses Quartz CGDisplayCreateImage instead of the screencapture CLI.
+    This avoids CoreMediaIO/ScreenCaptureKit, which blocks camera access
+    for other apps (e.g. Google Meet) on macOS 14+.
+    """
+
+    def __init__(self, mode: str = "screen", target: int = 0,
+                 fps: float = 1, scale: float = 0.5):
+        self.mode = mode
+        self.target = target
+        self.fps = fps
+        self.scale = scale
+        self.stats_ok = 0
+        self.stats_fail = 0
+        self._last_stats_time = time.time()
+        self._stats_interval = 60  # log stats every 60s
+        self._display_id = Quartz.CGMainDisplayID()
+
+    def capture_frame(self) -> tuple[Image.Image, float]:
+        """Returns (PIL Image, timestamp).
+        Uses CGDisplayCreateImage for zero-subprocess, camera-safe capture.
+        Downscales by self.scale factor before returning.
+        """
+        ts = time.time()
+        cg_image = Quartz.CGDisplayCreateImage(self._display_id)
+        if cg_image is None:
+            self.stats_fail += 1
+            raise RuntimeError("CGDisplayCreateImage returned None")
+
+        try:
+            width = Quartz.CGImageGetWidth(cg_image)
+            height = Quartz.CGImageGetHeight(cg_image)
+            bytes_per_row = Quartz.CGImageGetBytesPerRow(cg_image)
+
+            # Get raw pixel data from CGImage
+            data_provider = Quartz.CGImageGetDataProvider(cg_image)
+            raw_data = Quartz.CGDataProviderCopyData(data_provider)
+        finally:
+            # Explicitly release CGImage and its IOSurface handle immediately.
+            # At continuous capture rates, unreleased handles cause GPU/camera
+            # contention because the camera shares IOSurface infrastructure.
+            del cg_image
+
+        # CGDisplayCreateImage returns BGRA (premultiplied alpha, 32Little)
+        img = Image.frombytes("RGBA", (width, height), raw_data,
+                              "raw", "BGRA", bytes_per_row, 1)
+
+        if self.scale != 1.0:
+            new_w = int(width * self.scale)
+            new_h = int(height * self.scale)
+            img = img.resize((new_w, new_h), Image.LANCZOS)
+
+        self.stats_ok += 1
+        return img, ts
+
+    def capture_loop(self) -> Generator[tuple[Image.Image, float], None, None]:
+        """Yields frames at self.fps rate."""
+        interval = 1.0 / self.fps
+        while True:
+            start = time.time()
+            try:
+                yield self.capture_frame()
+            except Exception as e:
+                print(f"[capture] error: {e}")
+            self._maybe_log_stats()
+            elapsed = time.time() - start
+            sleep_time = interval - elapsed
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+
+    def _maybe_log_stats(self):
+        now = time.time()
+        if now - self._last_stats_time >= self._stats_interval:
+            total = self.stats_ok + self.stats_fail
+            rate = (self.stats_ok / total * 100) if total > 0 else 0
+            print(f"[capture] stats: {self.stats_ok} ok, {self.stats_fail} fail"
+                  f" ({rate:.0f}% success, {total} total)")
+            if self.stats_fail > 0 and self.stats_ok == 0:
+                print("[capture] WARNING: all captures failing — check screen recording permissions")
+            self._last_stats_time = now
+
+class ScreenKitCapture:
+    """Reads JPEG frames written by sck-capture via IPC (~/.sinain/capture/)."""
+
+    FRAME_PATH = os.path.expanduser("~/.sinain/capture/frame.jpg")
+    META_PATH = os.path.expanduser("~/.sinain/capture/meta.json")
+    STALE_THRESHOLD = 1.0  # seconds
+
+    def __init__(self, fps: float = 1, scale: float = 1.0, **kwargs):
+        self.fps = fps
+        self.scale = scale
+        self.stats_ok = 0
+        self.stats_fail = 0
+        self._last_frame_ts = 0.0
+        self._last_stats_time = time.time()
+        self._stats_interval = 60
+
+    @classmethod
+    def is_available(cls) -> bool:
+        """Check if fresh frames exist from the overlay app."""
+        try:
+            if not os.path.exists(cls.FRAME_PATH):
+                return False
+            mtime = os.path.getmtime(cls.FRAME_PATH)
+            return (time.time() - mtime) < cls.STALE_THRESHOLD
+        except OSError:
+            return False
+
+    def capture_frame(self) -> tuple[Image.Image, float] | None:
+        """Read the latest frame from IPC.
+
+        Returns (PIL Image, timestamp) or None if frame is stale/duplicate.
+        """
+        try:
+            if not os.path.exists(self.FRAME_PATH):
+                return None
+
+            # Read metadata for precise timestamp
+            ts = time.time()
+            if os.path.exists(self.META_PATH):
+                try:
+                    with open(self.META_PATH) as f:
+                        meta = json.load(f)
+                    ts = meta.get("timestamp", ts)
+                except (json.JSONDecodeError, OSError):
+                    pass
+
+            # Skip duplicate frames
+            if ts == self._last_frame_ts:
+                return None
+
+            img = Image.open(self.FRAME_PATH)
+            img.load()  # Force full read before file can be overwritten
+
+            if self.scale != 1.0:
+                new_w = int(img.width * self.scale)
+                new_h = int(img.height * self.scale)
+                img = img.resize((new_w, new_h), Image.LANCZOS)
+
+            self._last_frame_ts = ts
+            self.stats_ok += 1
+            return img, ts
+
+        except Exception as e:
+            self.stats_fail += 1
+            print(f"[capture-screenkit] error: {e}")
+            return None
+
+    def capture_loop(self) -> Generator[tuple[Image.Image, float], None, None]:
+        """Yields frames at self.fps rate, same interface as ScreenCapture."""
+        interval = 1.0 / self.fps
+        while True:
+            start = time.time()
+            result = self.capture_frame()
+            if result is not None:
+                yield result
+            self._maybe_log_stats()
+            elapsed = time.time() - start
+            sleep_time = interval - elapsed
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+
+    def _maybe_log_stats(self):
+        now = time.time()
+        if now - self._last_stats_time >= self._stats_interval:
+            total = self.stats_ok + self.stats_fail
+            rate = (self.stats_ok / total * 100) if total > 0 else 0
+            print(f"[capture-screenkit] stats: {self.stats_ok} ok, {self.stats_fail} fail"
+                  f" ({rate:.0f}% success, {total} total)")
+            self._last_stats_time = now
+
+
+def create_capture(mode: str = "screen", target: int = 0,
+                   fps: float = 1, scale: float = 0.5):
+    """Factory: platform-dispatched screen capture backend.
+
+    macOS: ScreenKitCapture (IPC from sck-capture) → ScreenCapture (CoreGraphics).
+    Windows: WinScreenCapture (mss / DXGI Desktop Duplication).
+    """
+    if sys.platform == "win32":
+        from .capture_win import WinScreenCapture
+        print("[capture] Using mss (DXGI Desktop Duplication)")
+        return WinScreenCapture(mode=mode, target=target, fps=fps, scale=scale)
+
+    # macOS path
+    # 1. IPC from sck-capture (primary — Swift binary writes JPEG frames)
+    if ScreenKitCapture.is_available():
+        print("[capture] Using ScreenCaptureKit (sck-capture IPC)")
+        return ScreenKitCapture(fps=fps, scale=1.0)  # scale handled by sck-capture
+
+    # 2. CGDisplayCreateImage (legacy fallback for macOS < 13)
+    print("[capture] Using CoreGraphics (CGDisplayCreateImage)")
+    print("[capture] WARNING: CGDisplayCreateImage may cause camera conflicts on macOS 14+")
+    return ScreenCapture(mode=mode, target=target, fps=fps, scale=scale)

package/sense_client/capture_win.py

@@ -0,0 +1,88 @@
+"""Screen capture on Windows via mss (DXGI Desktop Duplication)."""
+from __future__ import annotations
+
+import time
+from typing import Generator
+
+from PIL import Image
+
+try:
+    import mss
+except ImportError:
+    mss = None
+
+
+class WinScreenCapture:
+    """Captures screen frames on Windows using mss (DXGI Desktop Duplication).
+
+    Same interface as ScreenCapture/ScreenKitCapture on macOS:
+    capture_frame() -> (Image, float) and capture_loop() -> Generator.
+    """
+
+    def __init__(self, mode: str = "screen", target: int = 0,
+                 fps: float = 1, scale: float = 0.5):
+        if mss is None:
+            raise RuntimeError("mss library required for Windows capture: pip install mss")
+        self.mode = mode
+        self.target = target  # monitor index (0 = all, 1 = primary, etc.)
+        self.fps = fps
+        self.scale = scale
+        self.stats_ok = 0
+        self.stats_fail = 0
+        self._last_stats_time = time.time()
+        self._stats_interval = 60
+        self._sct = mss.mss()
+
+    def capture_frame(self) -> tuple[Image.Image, float]:
+        """Returns (PIL Image, timestamp).
+        Uses mss for DXGI-based capture. Downscales by self.scale factor.
+        """
+        ts = time.time()
+
+        # mss monitors: index 0 = all monitors combined, 1+ = individual
+        monitor_idx = self.target + 1 if self.target >= 0 else 1
+        if monitor_idx >= len(self._sct.monitors):
+            monitor_idx = 1  # fallback to primary
+
+        monitor = self._sct.monitors[monitor_idx]
+        screenshot = self._sct.grab(monitor)
+
+        # Convert to PIL Image (mss returns BGRA)
+        img = Image.frombytes("RGB", screenshot.size, screenshot.rgb)
+
+        if self.scale != 1.0:
+            new_w = int(img.width * self.scale)
+            new_h = int(img.height * self.scale)
+            img = img.resize((new_w, new_h), Image.LANCZOS)
+
+        self.stats_ok += 1
+        if self.stats_ok == 1:
+            print(f"[capture-win] first frame: {img.width}x{img.height} "
+                  f"(monitor={monitor['width']}x{monitor['height']}, scale={self.scale})",
+                  flush=True)
+        return img, ts
+
+    def capture_loop(self) -> Generator[tuple[Image.Image, float], None, None]:
+        """Yields frames at self.fps rate."""
+        interval = 1.0 / self.fps
+        while True:
+            start = time.time()
+            try:
+                yield self.capture_frame()
+            except Exception as e:
+                self.stats_fail += 1
+                print(f"[capture-win] error: {e}", flush=True)
+            self._maybe_log_stats()
+            elapsed = time.time() - start
+            sleep_time = interval - elapsed
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+
+    def _maybe_log_stats(self):
+        now = time.time()
+        if now - self._last_stats_time >= self._stats_interval:
+            total = self.stats_ok + self.stats_fail
+            rate = (self.stats_ok / total * 100) if total > 0 else 0
+            print(f"[capture-win] stats: {self.stats_ok} ok, {self.stats_fail} fail"
+                  f" ({rate:.0f}% success, {total} total)", flush=True)
+            self._last_stats_time = now

package/sense_client/change_detector.py

@@ -0,0 +1,86 @@
+"""SSIM-based frame change detection."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from PIL import Image
+from skimage.metrics import structural_similarity
+
+
+@dataclass
+class ChangeResult:
+    ssim_score: float
+    diff_image: Image.Image
+    contours: list  # list of (y, x) coordinate arrays
+    bbox: tuple[int, int, int, int]  # (x, y, w, h)
+
+
+class ChangeDetector:
+    """SSIM-based frame change detection."""
+
+    def __init__(self, threshold: float = 0.95, min_area: int = 100):
+        self.threshold = threshold
+        self.min_area = min_area
+        self.prev_frame: np.ndarray | None = None
+
+    def set_threshold(self, threshold: float) -> None:
+        """Dynamically adjust the SSIM change threshold."""
+        self.threshold = threshold
+
+    def detect(self, frame: Image.Image) -> ChangeResult | None:
+        """Compare frame to previous. Returns ChangeResult if significant."""
+        gray = np.array(frame.convert("L"))
+
+        if self.prev_frame is None:
+            self.prev_frame = gray
+            return None
+
+        if gray.shape != self.prev_frame.shape:
+            self.prev_frame = gray
+            return None
+
+        score, diff_map = structural_similarity(
+            self.prev_frame, gray, full=True
+        )
+
+        if score >= self.threshold:
+            return None
+
+        # Keyframe update: only advance prev_frame when change IS detected.
+        # This lets diffs accumulate against the last accepted keyframe,
+        # which is essential at high FPS where consecutive frames differ by <1%.
+        self.prev_frame = gray
+
+        # Convert diff map to binary mask
+        diff_binary = ((1.0 - diff_map) * 255).astype(np.uint8)
+        mask = diff_binary > 30  # threshold for "changed" pixels
+
+        # Find contours via connected components
+        from skimage.measure import label, regionprops
+        labeled = label(mask)
+        regions = regionprops(labeled)
+
+        # Filter by area
+        contours = []
+        for region in regions:
+            if region.area >= self.min_area:
+                contours.append(region.coords)
+
+        if not contours:
+            return None
+
+        # Compute merged bounding box
+        all_coords = np.vstack(contours)
+        min_y, min_x = all_coords.min(axis=0)
+        max_y, max_x = all_coords.max(axis=0)
+        bbox = (int(min_x), int(min_y), int(max_x - min_x), int(max_y - min_y))
+
+        diff_img = Image.fromarray(diff_binary)
+
+        return ChangeResult(
+            ssim_score=score,
+            diff_image=diff_img,
+            contours=contours,
+            bbox=bbox,
+        )

package/sense_client/config.py

@@ -0,0 +1,64 @@
+"""Configuration loader for sense_client."""
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+DEFAULTS = {
+    "capture": {
+        "mode": "screen",
+        "target": 0,
+        "fps": 2.0,
+        "scale": 0.5,
+    },
+    "detection": {
+        "ssimThreshold": 0.92,
+        "minArea": 100,
+        "roiPadding": 20,
+        "cooldownMs": 5000,
+    },
+    "ocr": {
+        "enabled": True,
+        "backend": "auto",
+        "languages": ["en", "ru"],
+        "lang": "eng",
+        "psm": 11,
+        "minConfidence": 50,
+    },
+    "gate": {
+        "minOcrChars": 20,
+        "majorChangeThreshold": 0.85,
+        "cooldownMs": 5000,
+        "adaptiveCooldownMs": 2000,
+        "contextCooldownMs": 10000,
+    },
+    "relay": {
+        "url": "http://localhost:9500",
+        "sendThumbnails": True,
+        "maxImageKB": 500,
+    },
+    "optimization": {
+        "backpressure": False,
+        "textDedup": False,
+        "visionRegionOfInterest": False,
+        "shadowValidation": False,
+    },
+}
+
+
+def load_config(path: str | None = None) -> dict:
+    """Load config from JSON file, merge with defaults."""
+    config = json.loads(json.dumps(DEFAULTS))  # deep copy
+    if path and os.path.exists(path):
+        try:
+            with open(path) as f:
+                user = json.load(f)
+            for section, values in user.items():
+                if section in config and isinstance(values, dict):
+                    config[section].update(values)
+                else:
+                    config[section] = values
+        except (json.JSONDecodeError, ValueError):
+            pass  # use defaults
+    return config