@geravant/sinain 1.0.19 → 1.2.0

package/sense_client/app_detector_win.py

@@ -0,0 +1,83 @@
+"""Detect the foreground application and window title on Windows."""
+from __future__ import annotations
+
+import ctypes
+import ctypes.wintypes
+
+
+class WinAppDetector:
+    """Detects the foreground application and window title on Windows.
+
+    Uses Win32 API: GetForegroundWindow, GetWindowTextW, and
+    psutil/GetWindowThreadProcessId for process name resolution.
+    """
+
+    def __init__(self):
+        self._last_app: str = ""
+        self._last_window: str = ""
+        self._user32 = ctypes.windll.user32
+        self._kernel32 = ctypes.windll.kernel32
+
+    def get_active_app(self) -> tuple[str, str]:
+        """Returns (app_name, window_title) of the foreground window."""
+        try:
+            hwnd = self._user32.GetForegroundWindow()
+            if not hwnd:
+                return "", ""
+
+            # Get window title
+            length = self._user32.GetWindowTextLengthW(hwnd)
+            buf = ctypes.create_unicode_buffer(length + 1)
+            self._user32.GetWindowTextW(hwnd, buf, length + 1)
+            window_title = buf.value
+
+            # Get process name via PID
+            pid = ctypes.wintypes.DWORD()
+            self._user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid))
+            app_name = self._get_process_name(pid.value)
+
+            return app_name, window_title
+        except Exception:
+            return "", ""
+
+    def _get_process_name(self, pid: int) -> str:
+        """Get process executable name from PID."""
+        try:
+            # Try psutil first (more reliable)
+            import psutil
+            proc = psutil.Process(pid)
+            return proc.name().replace(".exe", "")
+        except Exception:
+            pass
+
+        # Fallback: OpenProcess + GetModuleBaseNameW
+        try:
+            PROCESS_QUERY_INFORMATION = 0x0400
+            PROCESS_VM_READ = 0x0010
+            handle = self._kernel32.OpenProcess(
+                PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, False, pid
+            )
+            if not handle:
+                return ""
+
+            try:
+                psapi = ctypes.windll.psapi
+                buf = ctypes.create_unicode_buffer(260)
+                psapi.GetModuleBaseNameW(handle, None, buf, 260)
+                name = buf.value
+                if name.lower().endswith(".exe"):
+                    name = name[:-4]
+                return name
+            finally:
+                self._kernel32.CloseHandle(handle)
+        except Exception:
+            return ""
+
+    def detect_change(self) -> tuple[bool, bool, str, str]:
+        """Returns (app_changed, window_changed, app_name, window_title)."""
+        app, window = self.get_active_app()
+        app_changed = app != self._last_app and self._last_app != ""
+        window_changed = window != self._last_window and self._last_window != ""
+        self._last_app = app
+        self._last_window = window
+        return app_changed, window_changed, app, window

package/sense_client/capture.py

@@ -0,0 +1,215 @@
+"""Screen capture via IPC from sck-capture (preferred) or CoreGraphics fallback.
+
+On Windows, uses mss (DXGI Desktop Duplication) via capture_win.py.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+from typing import Generator
+
+from PIL import Image
+
+# macOS-only imports — deferred so the module loads on Windows
+if sys.platform == "darwin":
+    import Quartz
+
+
+class ScreenCapture:
+    """Captures screen frames via CGDisplayCreateImage (CoreGraphics/IOSurface).
+
+    Uses Quartz CGDisplayCreateImage instead of the screencapture CLI.
+    This avoids CoreMediaIO/ScreenCaptureKit, which blocks camera access
+    for other apps (e.g. Google Meet) on macOS 14+.
+    """
+
+    def __init__(self, mode: str = "screen", target: int = 0,
+                 fps: float = 1, scale: float = 0.5):
+        self.mode = mode
+        self.target = target
+        self.fps = fps
+        self.scale = scale
+        self.stats_ok = 0
+        self.stats_fail = 0
+        self._last_stats_time = time.time()
+        self._stats_interval = 60  # log stats every 60s
+        self._display_id = Quartz.CGMainDisplayID()
+
+    def capture_frame(self) -> tuple[Image.Image, float]:
+        """Returns (PIL Image, timestamp).
+        Uses CGDisplayCreateImage for zero-subprocess, camera-safe capture.
+        Downscales by self.scale factor before returning.
+        """
+        ts = time.time()
+        cg_image = Quartz.CGDisplayCreateImage(self._display_id)
+        if cg_image is None:
+            self.stats_fail += 1
+            raise RuntimeError("CGDisplayCreateImage returned None")
+
+        try:
+            width = Quartz.CGImageGetWidth(cg_image)
+            height = Quartz.CGImageGetHeight(cg_image)
+            bytes_per_row = Quartz.CGImageGetBytesPerRow(cg_image)
+
+            # Get raw pixel data from CGImage
+            data_provider = Quartz.CGImageGetDataProvider(cg_image)
+            raw_data = Quartz.CGDataProviderCopyData(data_provider)
+        finally:
+            # Explicitly release CGImage and its IOSurface handle immediately.
+            # At continuous capture rates, unreleased handles cause GPU/camera
+            # contention because the camera shares IOSurface infrastructure.
+            del cg_image
+
+        # CGDisplayCreateImage returns BGRA (premultiplied alpha, 32Little)
+        img = Image.frombytes("RGBA", (width, height), raw_data,
+                              "raw", "BGRA", bytes_per_row, 1)
+
+        if self.scale != 1.0:
+            new_w = int(width * self.scale)
+            new_h = int(height * self.scale)
+            img = img.resize((new_w, new_h), Image.LANCZOS)
+
+        self.stats_ok += 1
+        return img, ts
+
+    def capture_loop(self) -> Generator[tuple[Image.Image, float], None, None]:
+        """Yields frames at self.fps rate."""
+        interval = 1.0 / self.fps
+        while True:
+            start = time.time()
+            try:
+                yield self.capture_frame()
+            except Exception as e:
+                print(f"[capture] error: {e}")
+            self._maybe_log_stats()
+            elapsed = time.time() - start
+            sleep_time = interval - elapsed
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+
+    def _maybe_log_stats(self):
+        now = time.time()
+        if now - self._last_stats_time >= self._stats_interval:
+            total = self.stats_ok + self.stats_fail
+            rate = (self.stats_ok / total * 100) if total > 0 else 0
+            print(f"[capture] stats: {self.stats_ok} ok, {self.stats_fail} fail"
+                  f" ({rate:.0f}% success, {total} total)")
+            if self.stats_fail > 0 and self.stats_ok == 0:
+                print("[capture] WARNING: all captures failing — check screen recording permissions")
+            self._last_stats_time = now
+
+class ScreenKitCapture:
+    """Reads JPEG frames written by sck-capture via IPC (~/.sinain/capture/)."""
+
+    FRAME_PATH = os.path.expanduser("~/.sinain/capture/frame.jpg")
+    META_PATH = os.path.expanduser("~/.sinain/capture/meta.json")
+    STALE_THRESHOLD = 1.0  # seconds
+
+    def __init__(self, fps: float = 1, scale: float = 1.0, **kwargs):
+        self.fps = fps
+        self.scale = scale
+        self.stats_ok = 0
+        self.stats_fail = 0
+        self._last_frame_ts = 0.0
+        self._last_stats_time = time.time()
+        self._stats_interval = 60
+
+    @classmethod
+    def is_available(cls) -> bool:
+        """Check if fresh frames exist from the overlay app."""
+        try:
+            if not os.path.exists(cls.FRAME_PATH):
+                return False
+            mtime = os.path.getmtime(cls.FRAME_PATH)
+            return (time.time() - mtime) < cls.STALE_THRESHOLD
+        except OSError:
+            return False
+
+    def capture_frame(self) -> tuple[Image.Image, float] | None:
+        """Read the latest frame from IPC.
+
+        Returns (PIL Image, timestamp) or None if frame is stale/duplicate.
+        """
+        try:
+            if not os.path.exists(self.FRAME_PATH):
+                return None
+
+            # Read metadata for precise timestamp
+            ts = time.time()
+            if os.path.exists(self.META_PATH):
+                try:
+                    with open(self.META_PATH) as f:
+                        meta = json.load(f)
+                    ts = meta.get("timestamp", ts)
+                except (json.JSONDecodeError, OSError):
+                    pass
+
+            # Skip duplicate frames
+            if ts == self._last_frame_ts:
+                return None
+
+            img = Image.open(self.FRAME_PATH)
+            img.load()  # Force full read before file can be overwritten
+
+            if self.scale != 1.0:
+                new_w = int(img.width * self.scale)
+                new_h = int(img.height * self.scale)
+                img = img.resize((new_w, new_h), Image.LANCZOS)
+
+            self._last_frame_ts = ts
+            self.stats_ok += 1
+            return img, ts
+
+        except Exception as e:
+            self.stats_fail += 1
+            print(f"[capture-screenkit] error: {e}")
+            return None
+
+    def capture_loop(self) -> Generator[tuple[Image.Image, float], None, None]:
+        """Yields frames at self.fps rate, same interface as ScreenCapture."""
+        interval = 1.0 / self.fps
+        while True:
+            start = time.time()
+            result = self.capture_frame()
+            if result is not None:
+                yield result
+            self._maybe_log_stats()
+            elapsed = time.time() - start
+            sleep_time = interval - elapsed
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+
+    def _maybe_log_stats(self):
+        now = time.time()
+        if now - self._last_stats_time >= self._stats_interval:
+            total = self.stats_ok + self.stats_fail
+            rate = (self.stats_ok / total * 100) if total > 0 else 0
+            print(f"[capture-screenkit] stats: {self.stats_ok} ok, {self.stats_fail} fail"
+                  f" ({rate:.0f}% success, {total} total)")
+            self._last_stats_time = now
+
+
+def create_capture(mode: str = "screen", target: int = 0,
+                   fps: float = 1, scale: float = 0.5):
+    """Factory: platform-dispatched screen capture backend.
+
+    macOS: ScreenKitCapture (IPC from sck-capture) → ScreenCapture (CoreGraphics).
+    Windows: WinScreenCapture (mss / DXGI Desktop Duplication).
+    """
+    if sys.platform == "win32":
+        from .capture_win import WinScreenCapture
+        print("[capture] Using mss (DXGI Desktop Duplication)")
+        return WinScreenCapture(mode=mode, target=target, fps=fps, scale=scale)
+
+    # macOS path
+    # 1. IPC from sck-capture (primary — Swift binary writes JPEG frames)
+    if ScreenKitCapture.is_available():
+        print("[capture] Using ScreenCaptureKit (sck-capture IPC)")
+        return ScreenKitCapture(fps=fps, scale=1.0)  # scale handled by sck-capture
+
+    # 2. CGDisplayCreateImage (legacy fallback for macOS < 13)
+    print("[capture] Using CoreGraphics (CGDisplayCreateImage)")
+    print("[capture] WARNING: CGDisplayCreateImage may cause camera conflicts on macOS 14+")
+    return ScreenCapture(mode=mode, target=target, fps=fps, scale=scale)

package/sense_client/capture_win.py

@@ -0,0 +1,88 @@
+"""Screen capture on Windows via mss (DXGI Desktop Duplication)."""
+from __future__ import annotations
+
+import time
+from typing import Generator
+
+from PIL import Image
+
+try:
+    import mss
+except ImportError:
+    mss = None
+
+
+class WinScreenCapture:
+    """Captures screen frames on Windows using mss (DXGI Desktop Duplication).
+
+    Same interface as ScreenCapture/ScreenKitCapture on macOS:
+    capture_frame() -> (Image, float) and capture_loop() -> Generator.
+    """
+
+    def __init__(self, mode: str = "screen", target: int = 0,
+                 fps: float = 1, scale: float = 0.5):
+        if mss is None:
+            raise RuntimeError("mss library required for Windows capture: pip install mss")
+        self.mode = mode
+        self.target = target  # monitor index (0 = all, 1 = primary, etc.)
+        self.fps = fps
+        self.scale = scale
+        self.stats_ok = 0
+        self.stats_fail = 0
+        self._last_stats_time = time.time()
+        self._stats_interval = 60
+        self._sct = mss.mss()
+
+    def capture_frame(self) -> tuple[Image.Image, float]:
+        """Returns (PIL Image, timestamp).
+        Uses mss for DXGI-based capture. Downscales by self.scale factor.
+        """
+        ts = time.time()
+
+        # mss monitors: index 0 = all monitors combined, 1+ = individual
+        monitor_idx = self.target + 1 if self.target >= 0 else 1
+        if monitor_idx >= len(self._sct.monitors):
+            monitor_idx = 1  # fallback to primary
+
+        monitor = self._sct.monitors[monitor_idx]
+        screenshot = self._sct.grab(monitor)
+
+        # Convert to PIL Image (mss returns BGRA)
+        img = Image.frombytes("RGB", screenshot.size, screenshot.rgb)
+
+        if self.scale != 1.0:
+            new_w = int(img.width * self.scale)
+            new_h = int(img.height * self.scale)
+            img = img.resize((new_w, new_h), Image.LANCZOS)
+
+        self.stats_ok += 1
+        if self.stats_ok == 1:
+            print(f"[capture-win] first frame: {img.width}x{img.height} "
+                  f"(monitor={monitor['width']}x{monitor['height']}, scale={self.scale})",
+                  flush=True)
+        return img, ts
+
+    def capture_loop(self) -> Generator[tuple[Image.Image, float], None, None]:
+        """Yields frames at self.fps rate."""
+        interval = 1.0 / self.fps
+        while True:
+            start = time.time()
+            try:
+                yield self.capture_frame()
+            except Exception as e:
+                self.stats_fail += 1
+                print(f"[capture-win] error: {e}", flush=True)
+            self._maybe_log_stats()
+            elapsed = time.time() - start
+            sleep_time = interval - elapsed
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+
+    def _maybe_log_stats(self):
+        now = time.time()
+        if now - self._last_stats_time >= self._stats_interval:
+            total = self.stats_ok + self.stats_fail
+            rate = (self.stats_ok / total * 100) if total > 0 else 0
+            print(f"[capture-win] stats: {self.stats_ok} ok, {self.stats_fail} fail"
+                  f" ({rate:.0f}% success, {total} total)", flush=True)
+            self._last_stats_time = now

package/sense_client/change_detector.py

@@ -0,0 +1,86 @@
+"""SSIM-based frame change detection."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from PIL import Image
+from skimage.metrics import structural_similarity
+
+
+@dataclass
+class ChangeResult:
+    ssim_score: float
+    diff_image: Image.Image
+    contours: list  # list of (y, x) coordinate arrays
+    bbox: tuple[int, int, int, int]  # (x, y, w, h)
+
+
+class ChangeDetector:
+    """SSIM-based frame change detection."""
+
+    def __init__(self, threshold: float = 0.95, min_area: int = 100):
+        self.threshold = threshold
+        self.min_area = min_area
+        self.prev_frame: np.ndarray | None = None
+
+    def set_threshold(self, threshold: float) -> None:
+        """Dynamically adjust the SSIM change threshold."""
+        self.threshold = threshold
+
+    def detect(self, frame: Image.Image) -> ChangeResult | None:
+        """Compare frame to previous. Returns ChangeResult if significant."""
+        gray = np.array(frame.convert("L"))
+
+        if self.prev_frame is None:
+            self.prev_frame = gray
+            return None
+
+        if gray.shape != self.prev_frame.shape:
+            self.prev_frame = gray
+            return None
+
+        score, diff_map = structural_similarity(
+            self.prev_frame, gray, full=True
+        )
+
+        if score >= self.threshold:
+            return None
+
+        # Keyframe update: only advance prev_frame when change IS detected.
+        # This lets diffs accumulate against the last accepted keyframe,
+        # which is essential at high FPS where consecutive frames differ by <1%.
+        self.prev_frame = gray
+
+        # Convert diff map to binary mask
+        diff_binary = ((1.0 - diff_map) * 255).astype(np.uint8)
+        mask = diff_binary > 30  # threshold for "changed" pixels
+
+        # Find contours via connected components
+        from skimage.measure import label, regionprops
+        labeled = label(mask)
+        regions = regionprops(labeled)
+
+        # Filter by area
+        contours = []
+        for region in regions:
+            if region.area >= self.min_area:
+                contours.append(region.coords)
+
+        if not contours:
+            return None
+
+        # Compute merged bounding box
+        all_coords = np.vstack(contours)
+        min_y, min_x = all_coords.min(axis=0)
+        max_y, max_x = all_coords.max(axis=0)
+        bbox = (int(min_x), int(min_y), int(max_x - min_x), int(max_y - min_y))
+
+        diff_img = Image.fromarray(diff_binary)
+
+        return ChangeResult(
+            ssim_score=score,
+            diff_image=diff_img,
+            contours=contours,
+            bbox=bbox,
+        )

package/sense_client/config.py

@@ -0,0 +1,64 @@
+"""Configuration loader for sense_client."""
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+DEFAULTS = {
+    "capture": {
+        "mode": "screen",
+        "target": 0,
+        "fps": 2.0,
+        "scale": 0.5,
+    },
+    "detection": {
+        "ssimThreshold": 0.92,
+        "minArea": 100,
+        "roiPadding": 20,
+        "cooldownMs": 5000,
+    },
+    "ocr": {
+        "enabled": True,
+        "backend": "auto",
+        "languages": ["en", "ru"],
+        "lang": "eng",
+        "psm": 11,
+        "minConfidence": 50,
+    },
+    "gate": {
+        "minOcrChars": 20,
+        "majorChangeThreshold": 0.85,
+        "cooldownMs": 5000,
+        "adaptiveCooldownMs": 2000,
+        "contextCooldownMs": 10000,
+    },
+    "relay": {
+        "url": "http://localhost:9500",
+        "sendThumbnails": True,
+        "maxImageKB": 500,
+    },
+    "optimization": {
+        "backpressure": False,
+        "textDedup": False,
+        "visionRegionOfInterest": False,
+        "shadowValidation": False,
+    },
+}
+
+
+def load_config(path: str | None = None) -> dict:
+    """Load config from JSON file, merge with defaults."""
+    config = json.loads(json.dumps(DEFAULTS))  # deep copy
+    if path and os.path.exists(path):
+        try:
+            with open(path) as f:
+                user = json.load(f)
+            for section, values in user.items():
+                if section in config and isinstance(values, dict):
+                    config[section].update(values)
+                else:
+                    config[section] = values
+        except (json.JSONDecodeError, ValueError):
+            pass  # use defaults
+    return config

package/sense_client/gate.py

@@ -0,0 +1,145 @@
+"""Decision gate — classifies sense events and decides what to send."""
+from __future__ import annotations
+
+import difflib
+import time
+from collections import deque
+from dataclasses import dataclass, field
+
+from .change_detector import ChangeResult
+from .ocr import OCRResult
+
+
+@dataclass
+class SenseMeta:
+    ssim: float = 0.0
+    app: str = ""
+    window_title: str = ""
+    screen: int = 0
+
+
+@dataclass
+class SenseObservation:
+    """Structured observation fields (claude-mem compatible schema).
+
+    Populated by sinain-core's agent layer, not by sense_client.
+    sense_client sets `title` and `facts` from OCR/app context;
+    sinain-core enriches with `narrative` and `concepts`.
+    """
+    title: str = ""
+    subtitle: str = ""
+    facts: list[str] = field(default_factory=list)
+    narrative: str = ""
+    concepts: list[str] = field(default_factory=list)
+
+
+@dataclass
+class SenseEvent:
+    type: str  # "text" | "visual" | "context"
+    ts: float = 0.0
+    ocr: str = ""
+    roi: dict | None = None
+    diff: dict | None = None
+    meta: SenseMeta = field(default_factory=SenseMeta)
+    observation: SenseObservation = field(default_factory=SenseObservation)
+
+
+class DecisionGate:
+    """Classifies sense events and decides what to send."""
+
+    def __init__(self, min_ocr_chars: int = 20,
+                 major_change_threshold: float = 0.85,
+                 cooldown_ms: int = 5000,
+                 adaptive_cooldown_ms: int = 2000,
+                 context_cooldown_ms: int = 10000):
+        self.min_ocr_chars = min_ocr_chars
+        self.major_change_threshold = major_change_threshold
+        self.cooldown_ms = cooldown_ms
+        self.adaptive_cooldown_ms = adaptive_cooldown_ms
+        self.context_cooldown_ms = context_cooldown_ms
+        self.last_send_ts: float = 0
+        self.last_context_ts: float = 0
+        self.last_app_change_ts: float = 0
+        # Fuzzy dedup: ring buffer of last 5 OCR texts
+        self._recent_texts: deque[str] = deque(maxlen=5)
+        self._last_sent_text: str = ""
+
+    def is_ready(self, app_changed: bool, window_changed: bool) -> bool:
+        """Time-based readiness check without consuming OCR output.
+
+        Used by backpressure scheduling to decide whether to run OCR at all.
+        """
+        if app_changed or window_changed:
+            return True
+        now = time.time() * 1000
+        recent = (now - self.last_app_change_ts) < 10000
+        cooldown = self.adaptive_cooldown_ms if recent else self.cooldown_ms
+        return now - self.last_send_ts >= cooldown
+
+    def _is_duplicate(self, text: str) -> bool:
+        """Check if text is too similar to any recently sent text."""
+        if text == self._last_sent_text:
+            return True
+        for prev in self._recent_texts:
+            ratio = difflib.SequenceMatcher(None, prev, text).ratio()
+            if ratio > 0.7:
+                return True
+        return False
+
+    @staticmethod
+    def _ocr_quality_ok(text: str) -> bool:
+        """Reject garbage OCR: >50% single-char tokens or <50% alphanumeric."""
+        tokens = text.split()
+        if not tokens:
+            return False
+        single_char = sum(1 for t in tokens if len(t) == 1)
+        if single_char / len(tokens) > 0.5:
+            return False
+        alnum = sum(1 for ch in text if ch.isalnum())
+        total = len(text.replace(" ", ""))
+        if total > 0 and alnum / total < 0.5:
+            return False
+        return True
+
+    def classify(self, change: ChangeResult | None,
+                 ocr: OCRResult, app_changed: bool,
+                 window_changed: bool = False) -> SenseEvent | None:
+        """Returns SenseEvent to send, or None to drop."""
+        now = time.time() * 1000
+
+        # Context events (app/window change) bypass normal cooldown
+        if app_changed or window_changed:
+            self.last_app_change_ts = now
+            if now - self.last_context_ts >= self.context_cooldown_ms:
+                self.last_context_ts = now
+                self.last_send_ts = now
+                return SenseEvent(type="context", ts=now)
+
+        # Adaptive cooldown: 2s after recent app switch, 5s otherwise
+        recent_app_change = (now - self.last_app_change_ts) < 10000
+        effective_cooldown = self.adaptive_cooldown_ms if recent_app_change else self.cooldown_ms
+        if now - self.last_send_ts < effective_cooldown:
+            return None
+
+        if change is None:
+            return None
+
+        # OCR text sufficient -> text event
+        if ocr.text and len(ocr.text) >= self.min_ocr_chars:
+            if self._is_duplicate(ocr.text):
+                return None
+            if not self._ocr_quality_ok(ocr.text):
+                return None
+            self._recent_texts.append(ocr.text)
+            self._last_sent_text = ocr.text
+            self.last_send_ts = now
+            return SenseEvent(type="text", ts=now, ocr=ocr.text,
+                              meta=SenseMeta(ssim=change.ssim_score))
+
+        # Major visual change -> visual event
+        if change.ssim_score < self.major_change_threshold:
+            self.last_send_ts = now
+            return SenseEvent(type="visual", ts=now, ocr=ocr.text,
+                              meta=SenseMeta(ssim=change.ssim_score))
+
+        return None