@geravant/sinain 1.6.0 → 1.6.2

package/launcher.js

@@ -127,7 +127,10 @@ async function main() {
         const scDir = path.join(PKG_DIR, "sense_client");
         // Check if key package is importable to skip pip
         try {
-          execSync('python3 -c "import PIL; import skimage"', { stdio: "pipe" });
+          const depCheck = IS_WINDOWS
+            ? 'python3 -c "import PIL; import skimage"'
+            : 'python3 -c "import PIL; import skimage; import Quartz"';
+          execSync(depCheck, { stdio: "pipe" });
         } catch {
           log("Installing sense_client Python dependencies...");
           try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@geravant/sinain",
-  "version": "1.6.0",
+  "version": "1.6.2",
   "description": "Ambient AI overlay invisible to screen capture — real-time insights from audio + screen context",
   "type": "module",
   "bin": {
@@ -16,6 +16,7 @@
     "cli.js",
     "launcher.js",
     "setup-overlay.js",
+    "setup-sck-capture.js",
     "pack-prepare.js",
     "install.js",
     "index.ts",

package/sense_client/__main__.py

@@ -12,6 +12,7 @@ if sys.platform == "win32":
 
 import argparse
 import concurrent.futures
+import copy
 import json
 import os
 import time
@@ -28,7 +29,7 @@ from .capture import ScreenCapture, create_capture
 from .change_detector import ChangeDetector
 from .roi_extractor import ROIExtractor
 from .ocr import OCRResult, create_ocr
-from .gate import DecisionGate, SenseObservation
+from .gate import DecisionGate, SenseEvent, SenseObservation, SenseMeta
 from .sender import SenseSender, package_full_frame, package_roi
 from .app_detector import AppDetector
 from .config import load_config
@@ -128,6 +129,7 @@ def main():
     )
     app_detector = AppDetector()
     ocr_pool = concurrent.futures.ThreadPoolExecutor(max_workers=4)
+    vision_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="vision")
 
     # Vision provider — routes to Ollama (local) or OpenRouter (cloud) based on config/privacy
     vision_cfg = config.get("vision", {})
@@ -355,18 +357,28 @@ def main():
             title=title, subtitle=subtitle, facts=facts,
         )
 
-        # Vision scene analysis (throttled, non-blocking on failure)
+        # Vision scene analysis — async: send text event immediately, vision follows
         if vision_provider and time.time() - last_vision_time >= vision_throttle_s:
-            try:
-                from PIL import Image as PILImage
-                pil_frame = PILImage.fromarray(use_frame) if isinstance(use_frame, np.ndarray) else use_frame
-                scene = vision_provider.describe(pil_frame, prompt=vision_prompt or None)
-                if scene:
-                    event.observation.scene = scene
-                    last_vision_time = time.time()
-                    log(f"vision: {scene[:80]}...")
-            except Exception as e:
-                log(f"vision error: {e}")
+            last_vision_time = time.time()  # claim slot immediately to prevent concurrent calls
+            _v_frame = use_frame.copy() if isinstance(use_frame, np.ndarray) else use_frame.copy()
+            _v_meta = copy.copy(event.meta)
+            _v_ts = event.ts
+            _v_prompt = vision_prompt
+            def _do_vision(frame, meta, ts, prompt):
+                try:
+                    from PIL import Image as PILImage
+                    pil = PILImage.fromarray(frame) if isinstance(frame, np.ndarray) else frame
+                    scene = vision_provider.describe(pil, prompt=prompt or None)
+                    if scene:
+                        log(f"vision: {scene[:80]}...")
+                        ctx_ev = SenseEvent(type="context", ts=ts)
+                        ctx_ev.observation = SenseObservation(scene=scene)
+                        ctx_ev.meta = meta
+                        ctx_ev.roi = package_full_frame(frame)
+                        sender.send(ctx_ev)
+                except Exception as e:
+                    log(f"vision error: {e}")
+            vision_pool.submit(_do_vision, _v_frame, _v_meta, _v_ts, _v_prompt)
 
         # Send small thumbnail for ALL event types (agent uses vision)
         # Privacy matrix: gate image sending based on PRIVACY_IMAGES_OPENROUTER

package/setup-sck-capture.js

@@ -0,0 +1,190 @@
+#!/usr/bin/env node
+// sinain setup-sck-capture — download pre-built sck-capture binary from GitHub Releases
+
+import { execSync } from "child_process";
+import fs from "fs";
+import path from "path";
+import os from "os";
+
+const HOME = os.homedir();
+const SINAIN_DIR = path.join(HOME, ".sinain");
+const INSTALL_DIR = path.join(SINAIN_DIR, "sck-capture");
+const BINARY_PATH = path.join(INSTALL_DIR, "sck-capture");
+const VERSION_FILE = path.join(INSTALL_DIR, "version.json");
+
+const REPO = "anthillnet/sinain-hud";
+const RELEASES_API = `https://api.github.com/repos/${REPO}/releases`;
+const TAG_PREFIX = "sck-capture-v";
+const ASSET_NAME = "sck-capture-macos.zip";
+
+const BOLD  = "\x1b[1m";
+const GREEN = "\x1b[32m";
+const YELLOW = "\x1b[33m";
+const RED   = "\x1b[31m";
+const DIM   = "\x1b[2m";
+const RESET = "\x1b[0m";
+
+function log(msg)  { console.log(`${BOLD}[setup-sck-capture]${RESET} ${msg}`); }
+function ok(msg)   { console.log(`${BOLD}[setup-sck-capture]${RESET} ${GREEN}✓${RESET} ${msg}`); }
+function warn(msg) { console.log(`${BOLD}[setup-sck-capture]${RESET} ${YELLOW}⚠${RESET} ${msg}`); }
+function fail(msg) { console.error(`${BOLD}[setup-sck-capture]${RESET} ${RED}✗${RESET} ${msg}`); process.exit(1); }
+
+// ── Entry point (only when run directly, not when imported) ──────────────────
+
+const isMain = process.argv[1] && (
+  import.meta.url === `file://${process.argv[1]}` ||
+  import.meta.url === new URL(process.argv[1], "file://").href
+);
+
+if (isMain) {
+  const args = process.argv.slice(2);
+  const forceUpdate = args.includes("--update");
+
+  if (os.platform() === "win32") {
+    log("sck-capture is macOS-only (Windows uses win-audio-capture.exe)");
+    process.exit(0);
+  }
+
+  await downloadBinary({ forceUpdate });
+}
+
+// ── Download pre-built binary ────────────────────────────────────────────────
+
+export async function downloadBinary({ silent = false, forceUpdate = false } = {}) {
+  const _log = silent ? () => {} : log;
+  const _ok = silent ? () => {} : ok;
+  const _warn = silent ? () => {} : warn;
+
+  fs.mkdirSync(INSTALL_DIR, { recursive: true });
+
+  // Find latest sck-capture release
+  _log("Checking for latest sck-capture release...");
+  let release;
+  try {
+    const res = await fetch(`${RELEASES_API}?per_page=30`, {
+      signal: AbortSignal.timeout(10000),
+      headers: { "Accept": "application/vnd.github+json" },
+    });
+    if (!res.ok) throw new Error(`GitHub API returned ${res.status}`);
+    const releases = await res.json();
+    release = releases.find(r => r.tag_name?.startsWith(TAG_PREFIX));
+    if (!release) throw new Error("No sck-capture release found");
+  } catch (e) {
+    if (silent) {
+      _warn(`Failed to fetch sck-capture release: ${e.message}`);
+      return false;
+    }
+    fail(`Failed to fetch releases: ${e.message}`);
+  }
+
+  const tag = release.tag_name;
+  const version = tag.replace(TAG_PREFIX, "");
+
+  // Check if already up-to-date
+  if (!forceUpdate && fs.existsSync(VERSION_FILE) && fs.existsSync(BINARY_PATH)) {
+    try {
+      const local = JSON.parse(fs.readFileSync(VERSION_FILE, "utf-8"));
+      if (local.tag === tag) {
+        _ok(`sck-capture already up-to-date (${version})`);
+        return true;
+      }
+      _log(`Updating: ${local.tag} → ${tag}`);
+    } catch { /* corrupt version file — re-download */ }
+  }
+
+  // Find the .zip asset
+  const zipAsset = release.assets?.find(a => a.name === ASSET_NAME);
+  if (!zipAsset) {
+    if (silent) {
+      _warn(`Release ${tag} has no ${ASSET_NAME} asset`);
+      return false;
+    }
+    fail(`Release ${tag} has no ${ASSET_NAME} asset`);
+  }
+
+  // Download
+  _log(`Downloading sck-capture ${version} (${formatBytes(zipAsset.size)})...`);
+  const zipPath = path.join(INSTALL_DIR, ASSET_NAME);
+
+  try {
+    const res = await fetch(zipAsset.browser_download_url, {
+      signal: AbortSignal.timeout(60000),
+      redirect: "follow",
+    });
+    if (!res.ok) throw new Error(`Download failed: ${res.status}`);
+
+    const total = parseInt(res.headers.get("content-length") || "0");
+    const chunks = [];
+    let downloaded = 0;
+
+    const reader = res.body.getReader();
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+      downloaded += value.length;
+      if (!silent && total > 0) {
+        const pct = Math.round((downloaded / total) * 100);
+        process.stdout.write(`\r${BOLD}[setup-sck-capture]${RESET} ${DIM}${pct}% (${formatBytes(downloaded)} / ${formatBytes(total)})${RESET}`);
+      }
+    }
+    if (!silent) process.stdout.write("\n");
+
+    const buffer = Buffer.concat(chunks);
+    fs.writeFileSync(zipPath, buffer);
+    _ok(`Downloaded ${formatBytes(buffer.length)}`);
+  } catch (e) {
+    if (silent) {
+      _warn(`Download failed: ${e.message}`);
+      return false;
+    }
+    fail(`Download failed: ${e.message}`);
+  }
+
+  // Remove old binary if present
+  if (fs.existsSync(BINARY_PATH)) {
+    fs.unlinkSync(BINARY_PATH);
+  }
+
+  // Extract
+  _log("Extracting...");
+  try {
+    execSync(`ditto -x -k "${zipPath}" "${INSTALL_DIR}"`, { stdio: "pipe" });
+  } catch {
+    try {
+      execSync(`unzip -o -q "${zipPath}" -d "${INSTALL_DIR}"`, { stdio: "pipe" });
+    } catch (e) {
+      if (silent) {
+        _warn(`Extraction failed: ${e.message}`);
+        return false;
+      }
+      fail(`Extraction failed: ${e.message}`);
+    }
+  }
+
+  // Make executable
+  try {
+    fs.chmodSync(BINARY_PATH, 0o755);
+  } catch { /* may not exist if zip structure differs */ }
+
+  // Write version marker
+  fs.writeFileSync(VERSION_FILE, JSON.stringify({
+    tag,
+    version,
+    installedAt: new Date().toISOString(),
+  }, null, 2));
+
+  // Clean up zip
+  fs.unlinkSync(zipPath);
+
+  _ok(`sck-capture ${version} installed → ${BINARY_PATH}`);
+  return true;
+}
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function formatBytes(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}

package/sinain-core/src/agent/loop.ts

@@ -1,9 +1,10 @@
 import { EventEmitter } from "node:events";
+import fs from "node:fs";
 import type { FeedBuffer } from "../buffers/feed-buffer.js";
 import type { SenseBuffer } from "../buffers/sense-buffer.js";
-import type { AgentConfig, AgentEntry, ContextWindow, EscalationMode, ContextRichness, RecorderStatus } from "../types.js";
+import type { AgentConfig, AgentEntry, ContextWindow, EscalationMode, ContextRichness, RecorderStatus, SenseEvent, FeedbackRecord } from "../types.js";
 import type { Profiler } from "../profiler.js";
-import { buildContextWindow } from "./context-window.js";
+import { buildContextWindow, RICHNESS_PRESETS } from "./context-window.js";
 import { analyzeContext } from "./analyzer.js";
 import { writeSituationMd } from "./situation-writer.js";
 import { calculateEscalationScore } from "../escalation/scorer.js";
@@ -35,6 +36,10 @@ export interface AgentLoopDeps {
   traitEngine?: TraitEngine;
   /** Directory to write per-day trait log JSONL files. */
   traitLogDir?: string;
+  /** Optional: path to sinain-knowledge.md for startup recap. */
+  getKnowledgeDocPath?: () => string | null;
+  /** Optional: feedback store for startup recap context. */
+  feedbackStore?: { queryRecent(n: number): FeedbackRecord[] };
 }
 
 export interface TraceContext {
@@ -69,6 +74,7 @@ export class AgentLoop extends EventEmitter {
   private lastRunTs = 0;
   private running = false;
   private started = false;
+  private firstTick = true;
 
   private lastPushedHud = "";
   private agentNextId = 1;
@@ -112,6 +118,9 @@ export class AgentLoop extends EventEmitter {
     }, this.deps.agentConfig.maxIntervalMs);
 
     log(TAG, `loop started (debounce=${this.deps.agentConfig.debounceMs}ms, max=${this.deps.agentConfig.maxIntervalMs}ms, cooldown=${this.deps.agentConfig.cooldownMs}ms, model=${this.deps.agentConfig.model})`);
+
+    // Fire recap tick: immediate HUD from persistent knowledge (no sense data needed)
+    this.fireRecapTick().catch(e => debug(TAG, "recap skipped:", String(e)));
   }
 
   /** Stop the agent loop. */
@@ -131,12 +140,13 @@ export class AgentLoop extends EventEmitter {
   onNewContext(): void {
     if (!this.started) return;
 
-    // Debounce: wait N ms after last event before running
+    // Fast first tick: 500ms debounce on startup, normal debounce after
+    const delay = this.firstTick ? 500 : this.deps.agentConfig.debounceMs;
     if (this.debounceTimer) clearTimeout(this.debounceTimer);
     this.debounceTimer = setTimeout(() => {
       this.debounceTimer = null;
       this.run().catch(err => error(TAG, "debounce tick error:", err.message));
-    }, this.deps.agentConfig.debounceMs);
+    }, delay);
   }
 
   /** Get agent results history (newest first). */
@@ -398,7 +408,80 @@ export class AgentLoop extends EventEmitter {
       traceCtx?.finish({ totalLatencyMs: Date.now() - Date.now(), llmLatencyMs: 0, llmInputTokens: 0, llmOutputTokens: 0, llmCost: 0, escalated: false, escalationScore: 0, contextScreenEvents: 0, contextAudioEntries: 0, contextRichness: richness, digestLength: 0, hudChanged: false });
     } finally {
       this.running = false;
+      this.firstTick = false;
       this.lastRunTs = Date.now();
     }
   }
+
+  // ── Private: startup recap tick from persistent knowledge ──
+
+  private async fireRecapTick(): Promise<void> {
+    if (this.running) return;
+    this.running = true;
+
+    try {
+      const sections: string[] = [];
+      const startTs = Date.now();
+
+      // 1. sinain-knowledge.md (established patterns, user preferences)
+      const knowledgePath = this.deps.getKnowledgeDocPath?.();
+      if (knowledgePath) {
+        const content = await fs.promises.readFile(knowledgePath, "utf-8").catch(() => "");
+        if (content.length > 50) sections.push(content.slice(0, 2000));
+      }
+
+      // 2. SITUATION.md digest (if fresh — less than 5 minutes old)
+      try {
+        const stat = await fs.promises.stat(this.deps.situationMdPath);
+        if (Date.now() - stat.mtimeMs < 5 * 60_000) {
+          const sit = await fs.promises.readFile(this.deps.situationMdPath, "utf-8");
+          const digestMatch = sit.match(/## Digest\n([\s\S]*?)(?=\n##|$)/);
+          if (digestMatch?.[1]?.trim()) {
+            sections.push(`Last session digest:\n${digestMatch[1].trim()}`);
+          }
+        }
+      } catch { /* SITUATION.md missing — fine */ }
+
+      // 3. Recent feedback records (last 5 escalation summaries)
+      const records = this.deps.feedbackStore?.queryRecent(5) ?? [];
+      if (records.length > 0) {
+        const recaps = records.slice(0, 5).map(r => `- ${r.currentApp}: ${r.hud}`).join("\n");
+        sections.push(`Recent activity:\n${recaps}`);
+      }
+
+      if (sections.length === 0) { return; }
+
+      const recapContext = sections.join("\n\n");
+
+      // Build synthetic ContextWindow with knowledge as screen entry
+      const recapWindow: ContextWindow = {
+        audio: [],
+        screen: [{
+          ts: Date.now(),
+          ocr: recapContext,
+          meta: { app: "sinain-recap", windowTitle: "startup" },
+          type: "context",
+        } as unknown as SenseEvent],
+        images: [],
+        currentApp: "sinain-recap",
+        appHistory: [],
+        audioCount: 0,
+        screenCount: 1,
+        windowMs: 0,
+        newestEventTs: Date.now(),
+        preset: RICHNESS_PRESETS.lean,
+      };
+
+      const result = await analyzeContext(recapWindow, this.deps.agentConfig, null);
+      if (result?.hud && result.hud !== "—" && result.hud !== "Idle") {
+        this.deps.onHudUpdate(result.hud);
+        log(TAG, `recap tick (${Date.now() - startTs}ms, ${result.tokensIn}in+${result.tokensOut}out tok) hud="${result.hud}"`);
+      }
+    } catch (err: any) {
+      debug(TAG, "recap tick error:", err.message || err);
+    } finally {
+      this.running = false;
+      // Do NOT update lastRunTs — normal cooldown should not be affected by recap
+    }
+  }
 }

package/sinain-core/src/audio/pipeline.ts

@@ -99,7 +99,7 @@ export class AudioPipeline extends EventEmitter {
       return;
     }
 
-    log(TAG, `starting capture: device=${this.config.device} cmd=${this.config.captureCommand} rate=${this.config.sampleRate}`);
+    log(TAG, `starting capture: device=${this.config.device} cmd=${this.config.captureCommand} rate=${this.config.sampleRate}${this.config.gainDb ? ` gain=${this.config.gainDb}dB` : ""}`);
 
     try {
       this.spawnCaptureProcess();
@@ -295,6 +295,16 @@ export class AudioPipeline extends EventEmitter {
 
     if (alignedPcm.length === 0) return;
 
+    // Apply gain (amplify quiet ScreenCaptureKit audio before VAD + transcription)
+    if (this.config.gainDb !== 0) {
+      const multiplier = Math.pow(10, this.config.gainDb / 20);
+      for (let i = 0; i < alignedPcm.length - 1; i += 2) {
+        const sample = alignedPcm.readInt16LE(i);
+        const amplified = Math.max(-32768, Math.min(32767, Math.round(sample * multiplier)));
+        alignedPcm.writeInt16LE(amplified, i);
+      }
+    }
+
     const energy = calculateRmsEnergy(alignedPcm);
     this.profiler?.gauge("audio.lastChunkKb", Math.round(alignedPcm.length / 1024));
 

package/sinain-core/src/escalation/escalator.ts

@@ -492,7 +492,6 @@ ${recentLines.join("\n")}`;
 
     // Generate a unique child session key — bypasses the main agent entirely
     const childSessionKey = `agent:main:subagent:${randomUUID()}`;
-    const mainSessionKey = this.deps.openclawConfig.sessionKey;
 
     this.outboundBytes += Buffer.byteLength(task);
     this.deps.profiler?.gauge("network.escalationOutBytes", this.outboundBytes);
@@ -523,7 +522,6 @@ ${recentLines.join("\n")}`;
         lane: "subagent",
         extraSystemPrompt: this.buildChildSystemPrompt(task, label),
         deliver: false,
-        spawnedBy: mainSessionKey,
         idempotencyKey: idemKey,
         label: label || undefined,
       }, 45_000, { expectFinal: true });

package/sinain-core/src/index.ts

@@ -25,6 +25,12 @@ import { initPrivacy, levelFor, applyLevel } from "./privacy/index.js";
 
 const TAG = "core";
 
+/** Resolve workspace path, expanding leading ~ to HOME. */
+function resolveWorkspace(): string {
+  const raw = process.env.SINAIN_WORKSPACE || `${process.env.HOME}/.openclaw/workspace`;
+  return raw.startsWith("~") ? raw.replace("~", process.env.HOME || "") : raw;
+}
+
 async function main() {
   log(TAG, "sinain-core starting...");
 
@@ -80,7 +86,7 @@ async function main() {
     profiler,
     feedbackStore: feedbackStore ?? undefined,
     queryKnowledgeFacts: async (entities: string[], maxFacts: number) => {
-      const workspace = process.env.SINAIN_WORKSPACE || `${process.env.HOME}/.openclaw/workspace`;
+      const workspace = resolveWorkspace();
       const dbPath = `${workspace}/memory/knowledge-graph.db`;
       const scriptPath = `${workspace}/sinain-memory/graph_query.py`;
       try {
@@ -156,6 +162,13 @@ async function main() {
     } : undefined,
     traitEngine,
     traitLogDir: config.traitConfig.logDir,
+    getKnowledgeDocPath: () => {
+      const workspace = resolveWorkspace();
+      const p = `${workspace}/memory/sinain-knowledge.md`;
+      try { if (existsSync(p)) return p; } catch {}
+      return null;
+    },
+    feedbackStore: feedbackStore ?? undefined,
   });
 
   // ── Wire learning signal collector (needs agentLoop) ──
@@ -400,13 +413,13 @@ async function main() {
 
     // Knowledge graph integration
     getKnowledgeDocPath: () => {
-      const workspace = process.env.SINAIN_WORKSPACE || `${process.env.HOME}/.openclaw/workspace`;
+      const workspace = resolveWorkspace();
       const p = `${workspace}/memory/sinain-knowledge.md`;
       try { if (existsSync(p)) return p; } catch {}
       return null;
     },
     queryKnowledgeFacts: async (entities: string[], maxFacts: number) => {
-      const workspace = process.env.SINAIN_WORKSPACE || `${process.env.HOME}/.openclaw/workspace`;
+      const workspace = resolveWorkspace();
       const dbPath = `${workspace}/memory/knowledge-graph.db`;
       const scriptPath = `${workspace}/sinain-memory/graph_query.py`;
       try {