@geravant/sinain 1.6.8 → 1.6.9

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@geravant/sinain",
-  "version": "1.6.8",
-  "description": "Ambient AI overlay invisible to screen capture — real-time insights from audio + screen context",
+  "version": "1.6.9",
+  "description": "Ambient intelligence that sees what you see, hears what you hear, and acts on your behalf",
   "type": "module",
   "bin": {
     "sinain": "./cli.js",

package/sense_client/__main__.py

@@ -12,6 +12,7 @@ if sys.platform == "win32":
 
 import argparse
 import concurrent.futures
+import copy
 import json
 import os
 import time
@@ -28,7 +29,7 @@ from .capture import ScreenCapture, create_capture
 from .change_detector import ChangeDetector
 from .roi_extractor import ROIExtractor
 from .ocr import OCRResult, create_ocr
-from .gate import DecisionGate, SenseObservation
+from .gate import DecisionGate, SenseEvent, SenseObservation, SenseMeta
 from .sender import SenseSender, package_full_frame, package_roi
 from .app_detector import AppDetector
 from .config import load_config
@@ -128,6 +129,7 @@ def main():
     )
     app_detector = AppDetector()
     ocr_pool = concurrent.futures.ThreadPoolExecutor(max_workers=4)
+    vision_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="vision")
 
     # Vision provider — routes to Ollama (local) or OpenRouter (cloud) based on config/privacy
     vision_cfg = config.get("vision", {})
@@ -196,13 +198,16 @@ def main():
             time.sleep(1)
             continue
 
-        # First-frame log
+        # First-frame log + force initial context event
+        _is_first_frame = not _logged_first_frame
         if not _logged_first_frame:
             log(f"first frame: {frame.size[0]}x{frame.size[1]} (scale={config['capture']['scale']})")
             _logged_first_frame = True
 
-        # 1. Check app/window change
+        # 1. Check app/window change (first frame always treated as app change)
         app_changed, window_changed, app_name, window_title = app_detector.detect_change()
+        if _is_first_frame:
+            app_changed = True  # force context event on startup
 
         # Adaptive SSIM threshold
         now_sec = time.time()
@@ -355,18 +360,28 @@ def main():
             title=title, subtitle=subtitle, facts=facts,
         )
 
-        # Vision scene analysis (throttled, non-blocking on failure)
+        # Vision scene analysis — async: send text event immediately, vision follows
         if vision_provider and time.time() - last_vision_time >= vision_throttle_s:
-            try:
-                from PIL import Image as PILImage
-                pil_frame = PILImage.fromarray(use_frame) if isinstance(use_frame, np.ndarray) else use_frame
-                scene = vision_provider.describe(pil_frame, prompt=vision_prompt or None)
-                if scene:
-                    event.observation.scene = scene
-                    last_vision_time = time.time()
-                    log(f"vision: {scene[:80]}...")
-            except Exception as e:
-                log(f"vision error: {e}")
+            last_vision_time = time.time()  # claim slot immediately to prevent concurrent calls
+            _v_frame = use_frame.copy() if isinstance(use_frame, np.ndarray) else use_frame.copy()
+            _v_meta = copy.copy(event.meta)
+            _v_ts = event.ts
+            _v_prompt = vision_prompt
+            def _do_vision(frame, meta, ts, prompt):
+                try:
+                    from PIL import Image as PILImage
+                    pil = PILImage.fromarray(frame) if isinstance(frame, np.ndarray) else frame
+                    scene = vision_provider.describe(pil, prompt=prompt or None)
+                    if scene:
+                        log(f"vision: {scene[:80]}...")
+                        ctx_ev = SenseEvent(type="context", ts=ts)
+                        ctx_ev.observation = SenseObservation(scene=scene)
+                        ctx_ev.meta = meta
+                        ctx_ev.roi = package_full_frame(frame)
+                        sender.send(ctx_ev)
+                except Exception as e:
+                    log(f"vision error: {e}")
+            vision_pool.submit(_do_vision, _v_frame, _v_meta, _v_ts, _v_prompt)
 
         # Send small thumbnail for ALL event types (agent uses vision)
         # Privacy matrix: gate image sending based on PRIVACY_IMAGES_OPENROUTER

package/sinain-agent/run.sh

@@ -49,20 +49,20 @@ invoke_agent() {
   local prompt="$1"
   case "$AGENT" in
     claude)
-      claude --dangerously-skip-permissions \
+      claude --enable-auto-mode \
         --mcp-config "$MCP_CONFIG" \
         --max-turns 5 --output-format text \
-        -p "$prompt" 2>/dev/null
+        -p "$prompt"
       ;;
     codex)
       codex exec -s danger-full-access \
-        "$prompt" 2>/dev/null
+        "$prompt"
       ;;
     junie)
       if $JUNIE_HAS_MCP; then
         junie --output-format text \
           --mcp-location "$JUNIE_MCP_DIR" \
-          --task "$prompt" 2>/dev/null
+          --task "$prompt"
       else
         return 1
       fi
@@ -70,7 +70,7 @@ invoke_agent() {
     goose)
       goose run --text "$prompt" \
         --output-format text \
-        --max-turns 10 2>/dev/null
+        --max-turns 10
       ;;
     aider)
       # No MCP support — signal pipe mode
@@ -104,10 +104,10 @@ invoke_pipe() {
   local msg="$1"
   case "$AGENT" in
     junie)
-      junie --output-format text --task "$msg" 2>/dev/null
+      junie --output-format text --task "$msg"
       ;;
     aider)
-      aider --yes -m "$msg" 2>/dev/null
+      aider --yes -m "$msg"
       ;;
     *)
       # Generic: pipe message to stdin
@@ -131,7 +131,9 @@ if [ "$AGENT" = "junie" ]; then
   if junie --help 2>&1 | grep -q "mcp-location"; then
     JUNIE_HAS_MCP=true
     mkdir -p "$JUNIE_MCP_DIR"
-    cp "$MCP_CONFIG" "$JUNIE_MCP_DIR/mcp.json"
+    # Junie expects relative paths from the config file location.
+    # Since we moved the config into a sub-directory, we need to adjust ../ to ../../
+    sed 's|"\.\./|"../../|g' "$MCP_CONFIG" > "$JUNIE_MCP_DIR/mcp.json"
   else
     echo "NOTE: junie $(junie --version 2>&1 | grep -oE '[0-9.]+' | head -1) lacks --mcp-location, using pipe mode"
     echo "  Upgrade junie for MCP support: brew upgrade junie"

package/sinain-core/src/agent/loop.ts

@@ -1,9 +1,10 @@
 import { EventEmitter } from "node:events";
+import fs from "node:fs";
 import type { FeedBuffer } from "../buffers/feed-buffer.js";
 import type { SenseBuffer } from "../buffers/sense-buffer.js";
-import type { AgentConfig, AgentEntry, ContextWindow, EscalationMode, ContextRichness, RecorderStatus } from "../types.js";
+import type { AgentConfig, AgentEntry, ContextWindow, EscalationMode, ContextRichness, RecorderStatus, SenseEvent, FeedbackRecord } from "../types.js";
 import type { Profiler } from "../profiler.js";
-import { buildContextWindow } from "./context-window.js";
+import { buildContextWindow, RICHNESS_PRESETS } from "./context-window.js";
 import { analyzeContext } from "./analyzer.js";
 import { writeSituationMd } from "./situation-writer.js";
 import { calculateEscalationScore } from "../escalation/scorer.js";
@@ -35,6 +36,10 @@ export interface AgentLoopDeps {
   traitEngine?: TraitEngine;
   /** Directory to write per-day trait log JSONL files. */
   traitLogDir?: string;
+  /** Optional: path to sinain-knowledge.md for startup recap. */
+  getKnowledgeDocPath?: () => string | null;
+  /** Optional: feedback store for startup recap context. */
+  feedbackStore?: { queryRecent(n: number): FeedbackRecord[] };
 }
 
 export interface TraceContext {
@@ -69,6 +74,7 @@ export class AgentLoop extends EventEmitter {
   private lastRunTs = 0;
   private running = false;
   private started = false;
+  private firstTick = true;
 
   private lastPushedHud = "";
   private agentNextId = 1;
@@ -112,6 +118,9 @@ export class AgentLoop extends EventEmitter {
     }, this.deps.agentConfig.maxIntervalMs);
 
     log(TAG, `loop started (debounce=${this.deps.agentConfig.debounceMs}ms, max=${this.deps.agentConfig.maxIntervalMs}ms, cooldown=${this.deps.agentConfig.cooldownMs}ms, model=${this.deps.agentConfig.model})`);
+
+    // Fire recap tick: immediate HUD from persistent knowledge (no sense data needed)
+    this.fireRecapTick().catch(e => debug(TAG, "recap skipped:", String(e)));
   }
 
   /** Stop the agent loop. */
@@ -131,12 +140,13 @@ export class AgentLoop extends EventEmitter {
   onNewContext(): void {
     if (!this.started) return;
 
-    // Debounce: wait N ms after last event before running
+    // Fast first tick: 500ms debounce on startup, normal debounce after
+    const delay = this.firstTick ? 500 : this.deps.agentConfig.debounceMs;
     if (this.debounceTimer) clearTimeout(this.debounceTimer);
     this.debounceTimer = setTimeout(() => {
       this.debounceTimer = null;
       this.run().catch(err => error(TAG, "debounce tick error:", err.message));
-    }, this.deps.agentConfig.debounceMs);
+    }, delay);
   }
 
   /** Get agent results history (newest first). */
@@ -398,7 +408,80 @@ export class AgentLoop extends EventEmitter {
       traceCtx?.finish({ totalLatencyMs: Date.now() - Date.now(), llmLatencyMs: 0, llmInputTokens: 0, llmOutputTokens: 0, llmCost: 0, escalated: false, escalationScore: 0, contextScreenEvents: 0, contextAudioEntries: 0, contextRichness: richness, digestLength: 0, hudChanged: false });
     } finally {
       this.running = false;
+      this.firstTick = false;
       this.lastRunTs = Date.now();
     }
   }
+
+  // ── Private: startup recap tick from persistent knowledge ──
+
+  private async fireRecapTick(): Promise<void> {
+    if (this.running) return;
+    this.running = true;
+
+    try {
+      const sections: string[] = [];
+      const startTs = Date.now();
+
+      // 1. sinain-knowledge.md (established patterns, user preferences)
+      const knowledgePath = this.deps.getKnowledgeDocPath?.();
+      if (knowledgePath) {
+        const content = await fs.promises.readFile(knowledgePath, "utf-8").catch(() => "");
+        if (content.length > 50) sections.push(content.slice(0, 2000));
+      }
+
+      // 2. SITUATION.md digest (if fresh — less than 5 minutes old)
+      try {
+        const stat = await fs.promises.stat(this.deps.situationMdPath);
+        if (Date.now() - stat.mtimeMs < 5 * 60_000) {
+          const sit = await fs.promises.readFile(this.deps.situationMdPath, "utf-8");
+          const digestMatch = sit.match(/## Digest\n([\s\S]*?)(?=\n##|$)/);
+          if (digestMatch?.[1]?.trim()) {
+            sections.push(`Last session digest:\n${digestMatch[1].trim()}`);
+          }
+        }
+      } catch { /* SITUATION.md missing — fine */ }
+
+      // 3. Recent feedback records (last 5 escalation summaries)
+      const records = this.deps.feedbackStore?.queryRecent(5) ?? [];
+      if (records.length > 0) {
+        const recaps = records.slice(0, 5).map(r => `- ${r.currentApp}: ${r.hud}`).join("\n");
+        sections.push(`Recent activity:\n${recaps}`);
+      }
+
+      if (sections.length === 0) { return; }
+
+      const recapContext = sections.join("\n\n");
+
+      // Build synthetic ContextWindow with knowledge as screen entry
+      const recapWindow: ContextWindow = {
+        audio: [],
+        screen: [{
+          ts: Date.now(),
+          ocr: recapContext,
+          meta: { app: "sinain-recap", windowTitle: "startup" },
+          type: "context",
+        } as unknown as SenseEvent],
+        images: [],
+        currentApp: "sinain-recap",
+        appHistory: [],
+        audioCount: 0,
+        screenCount: 1,
+        windowMs: 0,
+        newestEventTs: Date.now(),
+        preset: RICHNESS_PRESETS.lean,
+      };
+
+      const result = await analyzeContext(recapWindow, this.deps.agentConfig, null);
+      if (result?.hud && result.hud !== "—" && result.hud !== "Idle") {
+        this.deps.onHudUpdate(result.hud);
+        log(TAG, `recap tick (${Date.now() - startTs}ms, ${result.tokensIn}in+${result.tokensOut}out tok) hud="${result.hud}"`);
+      }
+    } catch (err: any) {
+      debug(TAG, "recap tick error:", err.message || err);
+    } finally {
+      this.running = false;
+      // Do NOT update lastRunTs — normal cooldown should not be affected by recap
+    }
+  }
 }

package/sinain-core/src/config.ts

@@ -7,6 +7,9 @@ import { PRESETS } from "./privacy/presets.js";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
+/** The .env file path that was actually loaded (if any). */
+export let loadedEnvPath: string | undefined;
+
 function loadDotEnv(): void {
   // Try sinain-core/.env first, then project root .env
   const candidates = [
@@ -35,6 +38,7 @@ function loadDotEnv(): void {
           process.env[key] = val;
         }
       }
+      loadedEnvPath = envPath;
       console.log(`[config] loaded ${envPath}`);
       return;
     } catch { /* ignore */ }

package/sinain-core/src/escalation/escalator.ts

@@ -52,7 +52,7 @@ export class Escalator {
   private slot: EscalationSlot;
   private httpPending: HttpPendingEscalation | null = null;
 
-  private lastEscalationTs = Date.now();
+  private lastEscalationTs = 0;
   private lastEscalatedDigest = "";
 
   // Spawn deduplication state

package/sinain-core/src/index.ts

@@ -25,6 +25,12 @@ import { initPrivacy, levelFor, applyLevel } from "./privacy/index.js";
 
 const TAG = "core";
 
+/** Resolve workspace path, expanding leading ~ to HOME. */
+function resolveWorkspace(): string {
+  const raw = process.env.SINAIN_WORKSPACE || `${process.env.HOME}/.openclaw/workspace`;
+  return raw.startsWith("~") ? raw.replace("~", process.env.HOME || "") : raw;
+}
+
 async function main() {
   log(TAG, "sinain-core starting...");
 
@@ -80,7 +86,7 @@ async function main() {
     profiler,
     feedbackStore: feedbackStore ?? undefined,
     queryKnowledgeFacts: async (entities: string[], maxFacts: number) => {
-      const workspace = process.env.SINAIN_WORKSPACE || `${process.env.HOME}/.openclaw/workspace`;
+      const workspace = resolveWorkspace();
       const dbPath = `${workspace}/memory/knowledge-graph.db`;
       const scriptPath = `${workspace}/sinain-memory/graph_query.py`;
       try {
@@ -139,6 +145,7 @@ async function main() {
       escalator.pushSituationMd(content);
     },
     onHudUpdate: (text) => {
+      wsHandler.broadcastRaw({ type: "thinking", active: false } as any);
       wsHandler.broadcast(text, "normal", "stream");
     },
     onTraceStart: tracer ? (tickId) => {
@@ -156,6 +163,13 @@ async function main() {
     } : undefined,
     traitEngine,
     traitLogDir: config.traitConfig.logDir,
+    getKnowledgeDocPath: () => {
+      const workspace = resolveWorkspace();
+      const p = `${workspace}/memory/sinain-knowledge.md`;
+      try { if (existsSync(p)) return p; } catch {}
+      return null;
+    },
+    feedbackStore: feedbackStore ?? undefined,
   });
 
   // ── Wire learning signal collector (needs agentLoop) ──
@@ -400,13 +414,13 @@ async function main() {
 
     // Knowledge graph integration
     getKnowledgeDocPath: () => {
-      const workspace = process.env.SINAIN_WORKSPACE || `${process.env.HOME}/.openclaw/workspace`;
+      const workspace = resolveWorkspace();
       const p = `${workspace}/memory/sinain-knowledge.md`;
       try { if (existsSync(p)) return p; } catch {}
       return null;
     },
     queryKnowledgeFacts: async (entities: string[], maxFacts: number) => {
-      const workspace = process.env.SINAIN_WORKSPACE || `${process.env.HOME}/.openclaw/workspace`;
+      const workspace = resolveWorkspace();
       const dbPath = `${workspace}/memory/knowledge-graph.db`;
       const scriptPath = `${workspace}/sinain-memory/graph_query.py`;
       try {

package/sinain-core/src/overlay/commands.ts

@@ -43,6 +43,17 @@ export function setupCommands(deps: CommandDeps): void {
       }
       case "user_command": {
         log(TAG, `user command received: "${msg.text.slice(0, 60)}"`);
+        // Echo user message to all overlay clients as a feed item
+        wsHandler.broadcastRaw({
+          type: "feed",
+          text: msg.text,
+          priority: "normal",
+          ts: Date.now(),
+          channel: "agent",
+          sender: "user",
+        } as any);
+        // Show thinking indicator
+        wsHandler.broadcastRaw({ type: "thinking", active: true } as any);
         deps.onUserCommand(msg.text);
         break;
       }

package/sinain-core/src/overlay/ws-handler.ts

@@ -11,6 +11,7 @@ import type {
   FeedChannel,
 } from "../types.js";
 import { log, warn } from "../log.js";
+import { loadedEnvPath } from "../config.js";
 
 const TAG = "ws";
 const HEARTBEAT_INTERVAL_MS = 10_000;
@@ -135,13 +136,14 @@ export class WsHandler {
 
   /** Send a status update to all connected overlays. */
   broadcastStatus(): void {
-    const msg: StatusMessage = {
+    const msg: StatusMessage & { envPath?: string } = {
       type: "status",
       audio: this.state.audio,
       mic: this.state.mic,
       screen: this.state.screen,
       connection: this.state.connection,
     };
+    if (loadedEnvPath) msg.envPath = loadedEnvPath;
     this.broadcastMessage(msg);
   }