@geravant/sinain 1.2.0 → 1.3.0

package/cli.js

@@ -164,13 +164,18 @@ Usage:
   sinain start [options]       Launch sinain services
   sinain stop                  Stop all sinain services
   sinain status                Check what's running
-  sinain setup-overlay         Clone and build the overlay app
+  sinain setup-overlay         Download pre-built overlay app
   sinain install               Install OpenClaw plugin (server-side)
 
 Start options:
   --no-sense                   Skip screen capture (sense_client)
-  --no-overlay                 Skip Flutter overlay
+  --no-overlay                 Skip overlay
   --no-agent                   Skip agent poll loop
   --agent=<name>               Agent to use: claude, codex, goose, aider (default: claude)
+
+Setup-overlay options:
+  --from-source                Build from Flutter source instead of downloading
+  --update                     Force re-download even if version matches
 `);
+
 }

package/install.js

@@ -279,10 +279,11 @@ async function installLocal() {
     }
   }
 
+  const token = cfg?.gateway?.auth?.token ?? "(see openclaw.json)";
   console.log("\n✓ sinain installed successfully.");
   console.log("  Plugin config: ~/.openclaw/openclaw.json");
-  console.log(`  Auth token:    ${authToken}`);
-  console.log("  Next: run 'openclaw gateway' in a new terminal, then run ./setup-nemoclaw.sh on your Mac.\n");
+  console.log(`  Auth token:    ${token}`);
+  console.log("  Next: run 'openclaw gateway' if not running, or restart to pick up changes.\n");
 }
 
 // ── Knowledge snapshot repo setup (shared) ──────────────────────────────────

package/launcher.js

@@ -139,14 +139,19 @@ async function main() {
   // Start overlay
   let overlayStatus = "skipped";
   if (!skipOverlay) {
-    const overlayDir = findOverlayDir();
-    const hasFlutter = commandExists("flutter");
-    if (overlayDir && hasFlutter) {
-      log("Starting overlay...");
-      startProcess("overlay", "flutter", ["run", "-d", "macos"], {
-        cwd: overlayDir,
-        color: MAGENTA,
-      });
+    const overlay = findOverlay();
+    if (overlay?.type === "prebuilt") {
+      // Remove quarantine if present (ad-hoc signed app)
+      try {
+        const xattrs = execSync(`xattr "${overlay.path}"`, { encoding: "utf-8" });
+        if (xattrs.includes("com.apple.quarantine")) {
+          execSync(`xattr -dr com.apple.quarantine "${overlay.path}"`, { stdio: "pipe" });
+        }
+      } catch { /* no quarantine or xattr failed — try launching anyway */ }
+
+      log("Starting overlay (pre-built)...");
+      const binary = path.join(overlay.path, "Contents/MacOS/sinain_hud");
+      startProcess("overlay", binary, [], { color: MAGENTA });
       await sleep(2000);
       const overlayChild = children.find(c => c.name === "overlay");
       if (overlayChild && !overlayChild.proc.killed && overlayChild.proc.exitCode === null) {
@@ -156,10 +161,28 @@ async function main() {
         warn("overlay exited early — check logs above");
         overlayStatus = "failed";
       }
-    } else if (!overlayDir) {
-      warn("overlay not found — run: sinain setup-overlay");
+    } else if (overlay?.type === "source") {
+      const hasFlutter = commandExists("flutter");
+      if (hasFlutter) {
+        log("Starting overlay (flutter run)...");
+        startProcess("overlay", "flutter", ["run", "-d", "macos"], {
+          cwd: overlay.path,
+          color: MAGENTA,
+        });
+        await sleep(2000);
+        const overlayChild = children.find(c => c.name === "overlay");
+        if (overlayChild && !overlayChild.proc.killed && overlayChild.proc.exitCode === null) {
+          ok(`overlay running (pid:${overlayChild.pid})`);
+          overlayStatus = "running";
+        } else {
+          warn("overlay exited early — check logs above");
+          overlayStatus = "failed";
+        }
+      } else {
+        warn("flutter not found — overlay source found but can't build");
+      }
     } else {
-      warn("flutter not found — overlay skipped");
+      warn("overlay not found — run: sinain setup-overlay");
     }
   }
 
@@ -227,7 +250,7 @@ async function preflight() {
     skipSense = true;
   }
 
-  // Flutter (optional)
+  // Flutter (optional — only needed if no pre-built overlay)
   if (commandExists("flutter")) {
     try {
       const flutterVer = execSync("flutter --version 2>&1", { encoding: "utf-8" }).split("\n")[0].split(" ")[1];
@@ -236,8 +259,13 @@ async function preflight() {
       ok("flutter (version unknown)");
     }
   } else {
-    warn("flutter not found — overlay will be skipped");
-    skipOverlay = true;
+    const prebuiltApp = path.join(SINAIN_DIR, "overlay-app", "sinain_hud.app");
+    if (fs.existsSync(prebuiltApp)) {
+      ok("overlay: pre-built app");
+    } else {
+      warn("no overlay available — run: sinain setup-overlay");
+      skipOverlay = true;
+    }
   }
 
   // Port 9500
@@ -461,17 +489,23 @@ function generateMcpConfig() {
 
 // ── Overlay discovery ───────────────────────────────────────────────────────
 
-function findOverlayDir() {
-  // 1. Sibling overlay/ (running from cloned repo)
+function findOverlay() {
+  // 1. Dev monorepo: sibling overlay/ with pubspec.yaml (Flutter source)
   const siblingOverlay = path.join(PKG_DIR, "..", "overlay");
   if (fs.existsSync(path.join(siblingOverlay, "pubspec.yaml"))) {
-    return siblingOverlay;
+    return { type: "source", path: siblingOverlay };
+  }
+
+  // 2. Pre-built .app bundle (downloaded by setup-overlay)
+  const prebuiltApp = path.join(SINAIN_DIR, "overlay-app", "sinain_hud.app");
+  if (fs.existsSync(prebuiltApp)) {
+    return { type: "prebuilt", path: prebuiltApp };
   }
 
-  // 2. ~/.sinain/overlay/ (installed via setup-overlay)
+  // 3. Legacy: ~/.sinain/overlay/ source install (setup-overlay --from-source)
   const installedOverlay = path.join(SINAIN_DIR, "overlay");
   if (fs.existsSync(path.join(installedOverlay, "pubspec.yaml"))) {
-    return installedOverlay;
+    return { type: "source", path: installedOverlay };
   }
 
   return null;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@geravant/sinain",
-  "version": "1.2.0",
-  "description": "sinain — AI overlay system for macOS (npx @geravant/sinain start)",
+  "version": "1.3.0",
+  "description": "Ambient AI overlay invisible to screen capture — real-time insights from audio + screen context",
   "type": "module",
   "bin": {
     "sinain": "./cli.js",

package/setup-overlay.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-// sinain setup-overlay — clone and build the Flutter overlay app
+// sinain setup-overlay — download pre-built overlay app (or build from source)
 
 import { execSync } from "child_process";
 import fs from "fs";
@@ -8,75 +8,227 @@ import os from "os";
 
 const HOME = os.homedir();
 const SINAIN_DIR = path.join(HOME, ".sinain");
+const APP_DIR = path.join(SINAIN_DIR, "overlay-app");
+const APP_PATH = path.join(APP_DIR, "sinain_hud.app");
+const VERSION_FILE = path.join(APP_DIR, "version.json");
+
+const REPO = "anthillnet/sinain-hud";
+const RELEASES_API = `https://api.github.com/repos/${REPO}/releases`;
+
+// Legacy source-build paths
 const REPO_DIR = path.join(SINAIN_DIR, "overlay-repo");
 const OVERLAY_LINK = path.join(SINAIN_DIR, "overlay");
 
 const BOLD  = "\x1b[1m";
 const GREEN = "\x1b[32m";
+const YELLOW = "\x1b[33m";
 const RED   = "\x1b[31m";
+const DIM   = "\x1b[2m";
 const RESET = "\x1b[0m";
 
-function log(msg) { console.log(`${BOLD}[setup-overlay]${RESET} ${msg}`); }
-function ok(msg)  { console.log(`${BOLD}[setup-overlay]${RESET} ${GREEN}✓${RESET} ${msg}`); }
+function log(msg)  { console.log(`${BOLD}[setup-overlay]${RESET} ${msg}`); }
+function ok(msg)   { console.log(`${BOLD}[setup-overlay]${RESET} ${GREEN}✓${RESET} ${msg}`); }
+function warn(msg) { console.log(`${BOLD}[setup-overlay]${RESET} ${YELLOW}⚠${RESET} ${msg}`); }
 function fail(msg) { console.error(`${BOLD}[setup-overlay]${RESET} ${RED}✗${RESET} ${msg}`); process.exit(1); }
 
-// Check flutter
-try {
-  execSync("which flutter", { stdio: "pipe" });
-} catch {
-  fail("flutter not found. Install it: https://docs.flutter.dev/get-started/install");
+// ── Parse flags ──────────────────────────────────────────────────────────────
+
+const args = process.argv.slice(2);
+const fromSource = args.includes("--from-source");
+const forceUpdate = args.includes("--update");
+
+if (fromSource) {
+  await buildFromSource();
+} else {
+  await downloadPrebuilt();
 }
 
-const flutterVer = execSync("flutter --version 2>&1", { encoding: "utf-8" }).split("\n")[0];
-ok(`flutter: ${flutterVer}`);
+// ── Download pre-built .app ──────────────────────────────────────────────────
 
-fs.mkdirSync(SINAIN_DIR, { recursive: true });
+async function downloadPrebuilt() {
+  fs.mkdirSync(APP_DIR, { recursive: true });
 
-// Clone or update
-if (fs.existsSync(path.join(REPO_DIR, ".git"))) {
-  log("Updating existing overlay repo...");
-  execSync("git pull --ff-only", { cwd: REPO_DIR, stdio: "inherit" });
-  ok("Repository updated");
-} else {
-  log("Cloning overlay (sparse checkout — only overlay/ directory)...");
-  if (fs.existsSync(REPO_DIR)) {
-    fs.rmSync(REPO_DIR, { recursive: true, force: true });
+  // Find latest overlay release
+  log("Checking for latest overlay release...");
+  let release;
+  try {
+    const res = await fetch(`${RELEASES_API}?per_page=20`, {
+      signal: AbortSignal.timeout(10000),
+      headers: { "Accept": "application/vnd.github+json" },
+    });
+    if (!res.ok) throw new Error(`GitHub API returned ${res.status}`);
+    const releases = await res.json();
+    release = releases.find(r => r.tag_name?.startsWith("overlay-v"));
+    if (!release) throw new Error("No overlay release found");
+  } catch (e) {
+    fail(`Failed to fetch releases: ${e.message}\n  Try: sinain setup-overlay --from-source`);
   }
-  execSync(
-    `git clone --depth 1 --filter=blob:none --sparse https://github.com/anthillnet/sinain-hud.git "${REPO_DIR}"`,
-    { stdio: "inherit" }
-  );
-  execSync("git sparse-checkout set overlay", { cwd: REPO_DIR, stdio: "inherit" });
-  ok("Repository cloned");
-}
 
-// Build
-const overlayDir = path.join(REPO_DIR, "overlay");
-if (!fs.existsSync(path.join(overlayDir, "pubspec.yaml"))) {
-  fail("overlay/pubspec.yaml not found — sparse checkout may have failed");
-}
+  const tag = release.tag_name;
+  const version = tag.replace("overlay-v", "");
 
-log("Installing Flutter dependencies...");
-execSync("flutter pub get", { cwd: overlayDir, stdio: "inherit" });
+  // Check if already up-to-date
+  if (!forceUpdate && fs.existsSync(VERSION_FILE) && fs.existsSync(APP_PATH)) {
+    try {
+      const local = JSON.parse(fs.readFileSync(VERSION_FILE, "utf-8"));
+      if (local.tag === tag) {
+        ok(`Overlay already up-to-date (${version})`);
+        return;
+      }
+      log(`Updating: ${local.tag} → ${tag}`);
+    } catch { /* corrupt version file — re-download */ }
+  }
+
+  // Find the .zip asset
+  const zipAsset = release.assets?.find(a => a.name === "sinain_hud.app.zip");
+  if (!zipAsset) {
+    fail(`Release ${tag} has no sinain_hud.app.zip asset.\n  Try: sinain setup-overlay --from-source`);
+  }
 
-log("Building overlay (this may take a few minutes)...");
-execSync("flutter build macos", { cwd: overlayDir, stdio: "inherit" });
-ok("Overlay built successfully");
+  // Download with progress
+  log(`Downloading overlay ${version} (${formatBytes(zipAsset.size)})...`);
+  const zipPath = path.join(APP_DIR, "sinain_hud.app.zip");
 
-// Symlink ~/.sinain/overlay → the overlay source dir
-try {
-  if (fs.existsSync(OVERLAY_LINK)) {
-    fs.unlinkSync(OVERLAY_LINK);
+  try {
+    const res = await fetch(zipAsset.browser_download_url, {
+      signal: AbortSignal.timeout(120000),
+      redirect: "follow",
+    });
+    if (!res.ok) throw new Error(`Download failed: ${res.status}`);
+
+    const total = parseInt(res.headers.get("content-length") || "0");
+    const chunks = [];
+    let downloaded = 0;
+
+    const reader = res.body.getReader();
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+      downloaded += value.length;
+      if (total > 0) {
+        const pct = Math.round((downloaded / total) * 100);
+        process.stdout.write(`\r${BOLD}[setup-overlay]${RESET} ${DIM}${pct}% (${formatBytes(downloaded)} / ${formatBytes(total)})${RESET}`);
+      }
+    }
+    process.stdout.write("\n");
+
+    const buffer = Buffer.concat(chunks);
+    fs.writeFileSync(zipPath, buffer);
+    ok(`Downloaded ${formatBytes(buffer.length)}`);
+  } catch (e) {
+    fail(`Download failed: ${e.message}`);
+  }
+
+  // Remove old app if present
+  if (fs.existsSync(APP_PATH)) {
+    fs.rmSync(APP_PATH, { recursive: true, force: true });
   }
-  fs.symlinkSync(overlayDir, OVERLAY_LINK);
-  ok(`Symlinked: ${OVERLAY_LINK} → ${overlayDir}`);
-} catch (e) {
-  // Symlink may fail on some systems — fall back to just noting the path
-  log(`Overlay built at: ${overlayDir}`);
+
+  // Extract — ditto preserves macOS extended attributes (critical for code signing)
+  log("Extracting...");
+  try {
+    execSync(`ditto -x -k "${zipPath}" "${APP_DIR}"`, { stdio: "pipe" });
+  } catch {
+    // Fallback to unzip
+    try {
+      execSync(`unzip -o -q "${zipPath}" -d "${APP_DIR}"`, { stdio: "pipe" });
+    } catch (e) {
+      fail(`Extraction failed: ${e.message}`);
+    }
+  }
+
+  // Remove quarantine attribute (ad-hoc signed app downloaded from internet)
+  try {
+    execSync(`xattr -cr "${APP_PATH}"`, { stdio: "pipe" });
+  } catch { /* xattr may not be needed */ }
+
+  // Write version marker
+  fs.writeFileSync(VERSION_FILE, JSON.stringify({
+    tag,
+    version,
+    installedAt: new Date().toISOString(),
+  }, null, 2));
+
+  // Clean up zip
+  fs.unlinkSync(zipPath);
+
+  ok(`Overlay ${version} installed`);
+  console.log(`
+${GREEN}✓${RESET} Overlay ready!
+  Location: ${APP_PATH}
+  The overlay will auto-start with: ${BOLD}sinain start${RESET}
+`);
 }
 
-console.log(`
+// ── Build from source (legacy) ───────────────────────────────────────────────
+
+async function buildFromSource() {
+  // Check flutter
+  try {
+    execSync("which flutter", { stdio: "pipe" });
+  } catch {
+    fail("flutter not found. Install it: https://docs.flutter.dev/get-started/install");
+  }
+
+  const flutterVer = execSync("flutter --version 2>&1", { encoding: "utf-8" }).split("\n")[0];
+  ok(`flutter: ${flutterVer}`);
+
+  fs.mkdirSync(SINAIN_DIR, { recursive: true });
+
+  // Clone or update
+  if (fs.existsSync(path.join(REPO_DIR, ".git"))) {
+    log("Updating existing overlay repo...");
+    execSync("git pull --ff-only", { cwd: REPO_DIR, stdio: "inherit" });
+    ok("Repository updated");
+  } else {
+    log("Cloning overlay (sparse checkout — only overlay/ directory)...");
+    if (fs.existsSync(REPO_DIR)) {
+      fs.rmSync(REPO_DIR, { recursive: true, force: true });
+    }
+    execSync(
+      `git clone --depth 1 --filter=blob:none --sparse https://github.com/${REPO}.git "${REPO_DIR}"`,
+      { stdio: "inherit" }
+    );
+    execSync("git sparse-checkout set overlay", { cwd: REPO_DIR, stdio: "inherit" });
+    ok("Repository cloned");
+  }
+
+  // Build
+  const overlayDir = path.join(REPO_DIR, "overlay");
+  if (!fs.existsSync(path.join(overlayDir, "pubspec.yaml"))) {
+    fail("overlay/pubspec.yaml not found — sparse checkout may have failed");
+  }
+
+  log("Installing Flutter dependencies...");
+  execSync("flutter pub get", { cwd: overlayDir, stdio: "inherit" });
+
+  log("Building overlay (this may take a few minutes)...");
+  execSync("flutter build macos", { cwd: overlayDir, stdio: "inherit" });
+  ok("Overlay built successfully");
+
+  // Symlink ~/.sinain/overlay → the overlay source dir
+  try {
+    if (fs.existsSync(OVERLAY_LINK)) {
+      fs.unlinkSync(OVERLAY_LINK);
+    }
+    fs.symlinkSync(overlayDir, OVERLAY_LINK);
+    ok(`Symlinked: ${OVERLAY_LINK} → ${overlayDir}`);
+  } catch (e) {
+    log(`Overlay built at: ${overlayDir}`);
+  }
+
+  console.log(`
 ${GREEN}✓${RESET} Overlay setup complete!
   The overlay will auto-start with: ${BOLD}sinain start${RESET}
   Or run manually: cd ${overlayDir} && flutter run -d macos
 `);
+}
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function formatBytes(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}

package/sinain-core/.env.example

@@ -14,7 +14,6 @@ AUDIO_VAD_ENABLED=true
 AUDIO_VAD_THRESHOLD=0.003
 AUDIO_AUTO_START=true
 AUDIO_GAIN_DB=20
-# AUDIO_ALT_DEVICE=                # alternate device for switch_device command
 
 # ── Microphone (opt-in for privacy) ──
 MIC_ENABLED=false                  # set true to capture user's microphone

package/sinain-core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "sinain-core",
   "version": "1.0.0",
-  "description": "Unified HUD-sense-audio-bridge-relay — replaces relay + bridge with a single process",
+  "description": "SinainHUD core — audio transcription, agent analysis loop, escalation orchestration, WebSocket feed",
   "type": "module",
   "main": "dist/index.js",
   "scripts": {

package/sinain-core/src/config.ts

@@ -228,7 +228,6 @@ export function loadConfig(): CoreConfig {
   return {
     port: intEnv("PORT", 9500),
     audioConfig,
-    audioAltDevice: env("AUDIO_ALT_DEVICE", "BlackHole 2ch"),
     micConfig,
     micEnabled,
     transcriptionConfig,

package/sinain-core/src/escalation/escalator.ts

@@ -1,4 +1,4 @@
-import type { AgentEntry, ContextWindow, EscalationConfig, OpenClawConfig, FeedItem, SpawnTaskMessage, SpawnTaskStatus } from "../types.js";
+import type { AgentEntry, ContextWindow, EscalationConfig, OpenClawConfig, FeedItem, SpawnTaskMessage, SpawnTaskStatus, UserCommand } from "../types.js";
 import type { FeedBuffer } from "../buffers/feed-buffer.js";
 import type { WsHandler } from "../overlay/ws-handler.js";
 import type { Profiler } from "../profiler.js";
@@ -74,6 +74,10 @@ export class Escalator {
   // Store context from last escalation for response handling
   private lastEscalationContext: ContextWindow | null = null;
 
+  // User command to inject into the next escalation
+  private pendingUserCommand: UserCommand | null = null;
+  private static readonly USER_COMMAND_EXPIRY_MS = 120_000; // 2 minutes
+
   private stats = {
     totalEscalations: 0,
     totalResponses: 0,
@@ -123,6 +127,15 @@ export class Escalator {
     this.deps.signalCollector = sc;
   }
 
+  /** Queue a user command to inject into the next escalation. */
+  setUserCommand(text: string, source: "text" | "voice" = "text"): void {
+    this.pendingUserCommand = { text, ts: Date.now(), source };
+    const preview = text.length > 60 ? text.slice(0, 60) + "…" : text;
+    this.deps.feedBuffer.push(`⌘ Command queued: ${preview}`, "normal", "system", "stream");
+    this.deps.wsHandler.broadcast(`⌘ Command queued: ${preview}`, "normal");
+    log(TAG, `user command set: "${preview}"`);
+  }
+
   /** Start the WS connection to OpenClaw (skipped when transport=http). */
   start(): void {
     if (this.deps.escalationConfig.mode !== "off" && this.deps.escalationConfig.transport !== "http") {
@@ -161,6 +174,14 @@ export class Escalator {
    * Decides whether to escalate and enqueues the message for delivery.
    */
   async onAgentAnalysis(entry: AgentEntry, contextWindow: ContextWindow): Promise<void> {
+    // Expire stale user commands (safety net — 120s is generous)
+    if (this.pendingUserCommand && Date.now() - this.pendingUserCommand.ts > Escalator.USER_COMMAND_EXPIRY_MS) {
+      warn(TAG, `user command expired after ${Escalator.USER_COMMAND_EXPIRY_MS / 1000}s — no escalation occurred`);
+      this.deps.feedBuffer.push("⚠ Command expired — no escalation occurred", "normal", "system", "stream");
+      this.deps.wsHandler.broadcast("⚠ Command expired — no escalation occurred", "normal");
+      this.pendingUserCommand = null;
+    }
+
     // Skip WS escalations when circuit is open (HTTP transport bypasses this)
     const transport = this.deps.escalationConfig.transport;
     if (this.wsClient.isCircuitOpen && transport !== "http") {
@@ -168,6 +189,9 @@ export class Escalator {
       return;
     }
 
+    // If user command is pending, force escalation (bypass score + cooldown)
+    const hasUserCommand = this.pendingUserCommand !== null;
+
     const { escalate, score, stale } = shouldEscalate(
       entry.digest,
       entry.hud,
@@ -179,7 +203,7 @@ export class Escalator {
       this.deps.escalationConfig.staleMs,
     );
 
-    if (!escalate) {
+    if (!escalate && !hasUserCommand) {
       log(TAG, `tick #${entry.id}: not escalating (mode=${this.deps.escalationConfig.mode}, score=${score.total}, hud="${entry.hud.slice(0, 40)}")`);
       return;
     }
@@ -192,21 +216,29 @@ export class Escalator {
     this.lastEscalatedDigest = entry.digest;
 
     const staleTag = stale ? ", STALE" : "";
+    const cmdTag = hasUserCommand ? ", USER_CMD" : "";
     const wsState = this.wsClient.isConnected ? "ws=connected" : "ws=disconnected";
-    log(TAG, `escalating tick #${entry.id} (score=${score.total}, reasons=[${score.reasons.join(",")}]${staleTag}, ${wsState})`);
+    log(TAG, `escalating tick #${entry.id} (score=${score.total}, reasons=[${score.reasons.join(",")}]${staleTag}${cmdTag}, ${wsState})`);
 
     // Store context for response handling (used in pushResponse for coding-context max-length)
     this.lastEscalationContext = contextWindow;
 
-    const escalationReason = score.reasons.join(", ");
+    const escalationReason = hasUserCommand
+      ? `user_command: ${this.pendingUserCommand!.text.slice(0, 80)}`
+      : score.reasons.join(", ");
     let message = buildEscalationMessage(
       entry.digest,
       contextWindow,
       entry,
       this.deps.escalationConfig.mode,
       escalationReason,
+      undefined,
+      this.pendingUserCommand ?? undefined,
     );
 
+    // Clear user command after building the message (consumed once)
+    this.pendingUserCommand = null;
+
     // Enrich with long-term knowledge facts (best-effort, 5s max)
     if (this.deps.queryKnowledgeFacts) {
       try {
@@ -389,6 +421,7 @@ ${recentLines.join("\n")}`;
       cooldownMs: this.deps.escalationConfig.cooldownMs,
       staleMs: this.deps.escalationConfig.staleMs,
       pendingSpawnTasks: this.pendingSpawnTasks.size,
+      pendingUserCommand: this.pendingUserCommand ? this.pendingUserCommand.text.slice(0, 80) : null,
       ...this.stats,
     };
   }

package/sinain-core/src/escalation/message-builder.ts

@@ -1,4 +1,4 @@
-import type { ContextWindow, AgentEntry, EscalationMode, FeedbackRecord } from "../types.js";
+import type { ContextWindow, AgentEntry, EscalationMode, FeedbackRecord, UserCommand } from "../types.js";
 import { normalizeAppName } from "../agent/context-window.js";
 import { levelFor, applyLevel } from "../privacy/index.js";
 
@@ -142,12 +142,18 @@ export function buildEscalationMessage(
   mode: EscalationMode,
   escalationReason?: string,
   recentFeedback?: FeedbackRecord[],
+  userCommand?: UserCommand,
 ): string {
   const sections: string[] = [];
 
   // Header with tick metadata
   sections.push(`[sinain-hud live context — tick #${entry.id}]`);
 
+  // User command — placed at the top so the agent sees it first
+  if (userCommand) {
+    sections.push(`## User Command\n> ${userCommand.text}\n\nThe user has explicitly asked you to address the above command. Prioritize it in your response.`);
+  }
+
   // Digest (always full)
   sections.push(`## Digest\n${digest}`);
 

package/sinain-core/src/index.ts

@@ -1,3 +1,4 @@
+import { existsSync } from "node:fs";
 import { loadConfig } from "./config.js";
 import { FeedBuffer } from "./buffers/feed-buffer.js";
 import { SenseBuffer } from "./buffers/sense-buffer.js";
@@ -390,6 +391,9 @@ async function main() {
     getTraces: (after, limit) => tracer ? tracer.getTraces(after, limit) : [],
     reconnectGateway: () => escalator.reconnectGateway(),
 
+    // User command injection (bare agent / HTTP)
+    setUserCommand: (text: string) => escalator.setUserCommand(text),
+
     // Bare agent HTTP escalation bridge
     getEscalationPending: () => escalator.getPendingHttp(),
     respondEscalation: (id: string, response: string) => escalator.respondHttp(id, response),
@@ -398,7 +402,7 @@ async function main() {
     getKnowledgeDocPath: () => {
       const workspace = process.env.SINAIN_WORKSPACE || `${process.env.HOME}/.openclaw/workspace`;
       const p = `${workspace}/memory/sinain-knowledge.md`;
-      try { if (require("node:fs").existsSync(p)) return p; } catch {}
+      try { if (existsSync(p)) return p; } catch {}
       return null;
     },
     queryKnowledgeFacts: async (entities: string[], maxFacts: number) => {
@@ -428,6 +432,9 @@ async function main() {
     onUserMessage: async (text) => {
       await escalator.sendDirect(text);
     },
+    onUserCommand: (text) => {
+      escalator.setUserCommand(text);
+    },
     onToggleScreen: () => {
       screenActive = !screenActive;
       if (!screenActive) {

package/sinain-core/src/overlay/commands.ts

@@ -13,6 +13,8 @@ export interface CommandDeps {
   micPipeline: AudioPipeline | null;
   config: CoreConfig;
   onUserMessage: (text: string) => Promise<void>;
+  /** Queue a user command to augment the next escalation */
+  onUserCommand: (text: string) => void;
   /** Toggle screen capture — returns new state */
   onToggleScreen: () => boolean;
   /** Toggle trait voices — returns new enabled state */
@@ -37,6 +39,11 @@ export function setupCommands(deps: CommandDeps): void {
         }
         break;
       }
+      case "user_command": {
+        log(TAG, `user command received: "${msg.text.slice(0, 60)}"`);
+        deps.onUserCommand(msg.text);
+        break;
+      }
       case "command": {
         handleCommand(msg.action, deps);
         log(TAG, `command processed: ${msg.action}`);
@@ -47,7 +54,7 @@ export function setupCommands(deps: CommandDeps): void {
 }
 
 function handleCommand(action: string, deps: CommandDeps): void {
-  const { wsHandler, systemAudioPipeline, micPipeline, config } = deps;
+  const { wsHandler, systemAudioPipeline, micPipeline } = deps;
 
   switch (action) {
     case "toggle_audio": {
@@ -100,15 +107,6 @@ function handleCommand(action: string, deps: CommandDeps): void {
       log(TAG, `screen toggled ${nowActive ? "ON" : "OFF"}`);
       break;
     }
-    case "switch_device": {
-      const current = systemAudioPipeline.getDevice();
-      const alt = config.audioAltDevice;
-      const next = current === config.audioConfig.device ? alt : config.audioConfig.device;
-      systemAudioPipeline.switchDevice(next);
-      wsHandler.broadcast(`Audio device \u2192 ${next}`, "normal");
-      log(TAG, `audio device switched: ${current} \u2192 ${next}`);
-      break;
-    }
     case "toggle_traits": {
       if (!deps.onToggleTraits) {
         wsHandler.broadcast("Trait voices not configured", "normal");

package/sinain-core/src/overlay/ws-handler.ts

@@ -194,6 +194,9 @@ export class WsHandler {
       case "command":
         log(TAG, `\u2190 command: ${msg.action}`);
         break;
+      case "user_command":
+        log(TAG, `\u2190 user command: ${msg.text.slice(0, 100)}`);
+        break;
       case "profiling":
         if (this.onProfilingCb) this.onProfilingCb(msg);
         return;

package/sinain-core/src/server.ts

@@ -34,6 +34,7 @@ export interface ServerDeps {
   getTraces: (after: number, limit: number) => unknown[];
   reconnectGateway: () => void;
   feedbackStore?: FeedbackStore;
+  setUserCommand?: (text: string) => void;
   getEscalationPending?: () => any;
   respondEscalation?: (id: string, response: string) => any;
   getKnowledgeDocPath?: () => string | null;
@@ -305,6 +306,20 @@ export function createAppServer(deps: ServerDeps) {
         return;
       }
 
+      // ── /user/command ──
+      if (req.method === "POST" && url.pathname === "/user/command") {
+        const body = await readBody(req, 4096);
+        const { text } = JSON.parse(body);
+        if (!text) {
+          res.writeHead(400);
+          res.end(JSON.stringify({ ok: false, error: "missing text" }));
+          return;
+        }
+        deps.setUserCommand?.(text);
+        res.end(JSON.stringify({ ok: true, message: "Command queued for next escalation" }));
+        return;
+      }
+
       // ── /escalation/pending ──
       if (req.method === "GET" && url.pathname === "/escalation/pending") {
         const pending = deps.getEscalationPending?.();

package/sinain-core/src/types.ts

@@ -66,8 +66,21 @@ export interface ProfilingMessage {
   ts: number;
 }
 
+/** Overlay → sinain-core: user command to augment next escalation */
+export interface UserCommandMessage {
+  type: "user_command";
+  text: string;
+}
+
 export type OutboundMessage = FeedMessage | StatusMessage | PingMessage | SpawnTaskMessage;
-export type InboundMessage = UserMessage | CommandMessage | PongMessage | ProfilingMessage;
+export type InboundMessage = UserMessage | CommandMessage | PongMessage | ProfilingMessage | UserCommandMessage;
+
+/** Abstraction for user commands (text now, voice later). */
+export interface UserCommand {
+  text: string;
+  ts: number;
+  source: "text" | "voice";
+}
 
 // ── Feed buffer types ──
 
@@ -411,7 +424,6 @@ export interface PrivacyConfig {
 export interface CoreConfig {
   port: number;
   audioConfig: AudioPipelineConfig;
-  audioAltDevice: string;
   micConfig: AudioPipelineConfig;
   micEnabled: boolean;
   transcriptionConfig: TranscriptionConfig;

package/sinain-mcp-server/index.ts

@@ -367,7 +367,22 @@ server.tool(
   },
 );
 
-// 10. sinain_module_guidance
+// 10. sinain_user_command
+server.tool(
+  "sinain_user_command",
+  "Queue a user command to augment the next escalation context (forces escalation on next agent tick)",
+  { text: z.string().describe("The command text to inject into the next escalation") },
+  async ({ text }) => {
+    try {
+      const data = await coreRequest("POST", "/user/command", { text });
+      return textResult(JSON.stringify(data, null, 2));
+    } catch (err: any) {
+      return textResult(`Error queuing user command: ${err.message}`);
+    }
+  },
+);
+
+// 11. sinain_module_guidance
 server.tool(
   "sinain_module_guidance",
   "Read guidance from all active modules in the workspace",

package/sinain-memory/graph_query.py

@@ -21,7 +21,12 @@ def query_facts_by_entities(
     entities: list[str],
     max_facts: int = 5,
 ) -> list[dict]:
-    """Query knowledge graph for facts related to specified entities/domains."""
+    """Query knowledge graph for facts matching keywords via tag index.
+
+    Uses auto-extracted 'tag' attributes for discovery. Results ranked by
+    number of matching tags (more matches = more relevant). Falls back to
+    domain/entity_id matching for untagged facts.
+    """
     if not Path(db_path).exists():
         return []
 
@@ -29,36 +34,56 @@ def query_facts_by_entities(
         from triplestore import TripleStore
         store = TripleStore(db_path)
 
-        # Find fact entity_ids that match the requested domains or entity names
-        placeholders = ",".join(["?" for _ in entities])
-        # Match by domain attribute OR by entity name substring
-        like_clauses = " OR ".join([f"entity_id LIKE ?" for _ in entities])
-        entity_likes = [f"fact:{e}%" for e in entities]
+        # Normalize keywords for tag matching
+        keywords = [e.lower().replace(" ", "-") for e in entities]
+        placeholders = ",".join(["?" for _ in keywords])
 
+        # Primary: tag-based ranked search (AVET index)
         rows = store._conn.execute(
-            f"""SELECT DISTINCT entity_id FROM triples
-                WHERE NOT retracted AND (
-                    (attribute = 'domain' AND value IN ({placeholders}))
-                    OR ({like_clauses})
-                )
+            f"""SELECT entity_id, COUNT(*) as matches
+                FROM triples
+                WHERE attribute = 'tag' AND NOT retracted
+                AND value IN ({placeholders})
+                GROUP BY entity_id
+                ORDER BY matches DESC
                 LIMIT ?""",
-            (*entities, *entity_likes, max_facts * 3),
+            (*keywords, max_facts * 3),
         ).fetchall()
 
         fact_ids = [r["entity_id"] for r in rows]
 
-        # Load full attributes for each fact, sorted by confidence
+        # Fallback: if tags found < max_facts, also search domain/entity_id (for untagged facts)
+        if len(fact_ids) < max_facts:
+            domain_placeholders = ",".join(["?" for _ in keywords])
+            like_clauses = " OR ".join([f"entity_id LIKE ?" for _ in keywords])
+            entity_likes = [f"fact:{kw}%" for kw in keywords]
+
+            fallback_rows = store._conn.execute(
+                f"""SELECT DISTINCT entity_id FROM triples
+                    WHERE NOT retracted AND entity_id NOT IN ({','.join(['?' for _ in fact_ids]) or "''"})
+                    AND (
+                        (attribute = 'domain' AND value IN ({domain_placeholders}))
+                        OR ({like_clauses})
+                    )
+                    LIMIT ?""",
+                (*fact_ids, *keywords, *entity_likes, max_facts - len(fact_ids)),
+            ).fetchall()
+            fact_ids.extend(r["entity_id"] for r in fallback_rows)
+
+        # Load full attributes for each fact
         facts = []
         for fid in fact_ids:
             attrs = store.entity(fid)
             if not attrs:
                 continue
             fact = {"entityId": fid}
-            for a in attrs:
-                fact[a["attribute"]] = a["value"]
+            for attr_name, values in attrs.items():
+                if attr_name == "tag":
+                    continue  # Don't include tags in output (noise)
+                fact[attr_name] = values[0] if len(values) == 1 else values
             facts.append(fact)
 
-        # Sort by confidence descending
+        # Sort by confidence descending (tag ranking already done in SQL)
         facts.sort(key=lambda f: float(f.get("confidence", "0")), reverse=True)
         store.close()
         return facts[:max_facts]
@@ -93,8 +118,8 @@ def query_top_facts(db_path: str, limit: int = 30) -> list[dict]:
             if not attrs:
                 continue
             fact = {"entityId": fid}
-            for a in attrs:
-                fact[a["attribute"]] = a["value"]
+            for attr_name, values in attrs.items():
+                fact[attr_name] = values[0] if len(values) == 1 else values
             facts.append(fact)
 
         store.close()

package/sinain-memory/knowledge_integrator.py

@@ -18,6 +18,7 @@ Usage:
 import argparse
 import hashlib
 import json
+import re
 import shutil
 import sys
 from datetime import datetime, timezone
@@ -80,6 +81,34 @@ Respond with ONLY a JSON object:
 }"""
 
 
+_STOPWORDS = frozenset({
+    "the", "and", "for", "when", "with", "that", "this", "from", "into",
+    "after", "before", "during", "should", "would", "could", "been", "have",
+    "will", "also", "then", "than", "not", "but", "are", "was", "were",
+    "can", "may", "use", "run", "set", "get", "try", "all", "any", "new",
+    "score", "seen",
+})
+
+
+def _extract_tags(value: str) -> list[str]:
+    """Extract searchable keyword tags from fact value text.
+
+    Returns up to 10 deduplicated lowercase tags suitable for AVET-indexed lookup.
+    """
+    # Lowercase words (including hyphenated compounds like "react-native")
+    words = re.findall(r"[a-z][a-z0-9-]+", value.lower())
+    tags = [w for w in words if len(w) > 2 and w not in _STOPWORDS]
+    # Detect compound terms from CamelCase or "Title Case" patterns
+    compounds = re.findall(r"[A-Z][a-z]+ [A-Z][a-z]+", value)
+    for c in compounds:
+        tags.append(c.lower().replace(" ", "-"))
+    # Numeric tokens that look meaningful (error codes, port numbers)
+    nums = re.findall(r"\b\d{3,5}\b", value)
+    tags.extend(nums)
+    # Deduplicate preserving order, cap at 10
+    return list(dict.fromkeys(tags))[:10]
+
+
 def _fact_id(entity: str, attribute: str, value: str) -> str:
     """Generate a deterministic fact entity ID from entity+attribute+value."""
     content = f"{entity}:{attribute}:{value}"
@@ -99,14 +128,19 @@ def _load_graph_facts(db_path: str, entities: list[str] | None = None, limit: in
 
         # Get all non-retracted fact entities with their attributes
         if entities:
-            # Entity-scoped query: find facts related to specified domains
-            domain_clause = " OR ".join([f"value = ?" for _ in entities])
+            # Tag-based search: find facts whose tags match any of the keywords
+            # Normalize keywords to lowercase for tag matching
+            keywords = [e.lower().replace(" ", "-") for e in entities]
+            placeholders = ",".join(["?" for _ in keywords])
             rows = store._conn.execute(
-                f"""SELECT DISTINCT entity_id FROM triples
-                    WHERE attribute = 'domain' AND NOT retracted
-                    AND ({domain_clause})
+                f"""SELECT entity_id, COUNT(*) as matches
+                    FROM triples
+                    WHERE attribute = 'tag' AND NOT retracted
+                    AND value IN ({placeholders})
+                    GROUP BY entity_id
+                    ORDER BY matches DESC
                     LIMIT ?""",
-                (*entities, limit),
+                (*keywords, limit),
             ).fetchall()
             fact_ids = [r["entity_id"] for r in rows]
         else:
@@ -127,8 +161,8 @@ def _load_graph_facts(db_path: str, entities: list[str] | None = None, limit: in
             attrs = store.entity(fid)
             if attrs:
                 fact = {"entityId": fid}
-                for a in attrs:
-                    fact[a["attribute"]] = a["value"]
+                for attr_name, values in attrs.items():
+                    fact[attr_name] = values[0] if len(values) == 1 else values
                 facts.append(fact)
 
         store.close()
@@ -172,6 +206,9 @@ def _execute_graph_ops(db_path: str, ops: list[dict], digest_ts: str) -> dict:
                 store.assert_triple(tx, entity_id, "reinforce_count", "1")
                 if domain:
                     store.assert_triple(tx, entity_id, "domain", domain)
+                # Auto-tag for keyword-based discovery
+                for tag in _extract_tags(value):
+                    store.assert_triple(tx, entity_id, "tag", tag)
                 stats["asserted"] += 1
 
             elif op == "reinforce":
@@ -186,16 +223,15 @@ def _execute_graph_ops(db_path: str, ops: list[dict], digest_ts: str) -> dict:
 
                 cur_conf = 0.5
                 cur_count = 0
-                for a in attrs:
-                    if a["attribute"] == "confidence":
-                        try:
-                            cur_conf = float(a["value"])
-                        except ValueError:
-                            pass
-                    elif a["attribute"] == "reinforce_count":
-                        try:
-                            cur_count = int(a["value"])
-                        except ValueError:
+                if "confidence" in attrs:
+                    try:
+                        cur_conf = float(attrs["confidence"][0])
+                    except (ValueError, IndexError):
+                        pass
+                if "reinforce_count" in attrs:
+                    try:
+                        cur_count = int(attrs["reinforce_count"][0])
+                    except (ValueError, IndexError):
                             pass
 
                 new_conf = min(1.0, cur_conf + 0.15)
@@ -209,7 +245,10 @@ def _execute_graph_ops(db_path: str, ops: list[dict], digest_ts: str) -> dict:
                 store.assert_triple(tx, entity_id, "confidence", str(round(new_conf, 2)))
                 store.retract_triple(tx, entity_id, "reinforce_count", str(cur_count))
                 store.assert_triple(tx, entity_id, "reinforce_count", str(new_count))
-                store.retract_triple(tx, entity_id, "last_reinforced", "")  # retract any
+                # Retract old last_reinforced if present
+                old_reinforced = attrs.get("last_reinforced", [])
+                for val in old_reinforced:
+                    store.retract_triple(tx, entity_id, "last_reinforced", val)
                 store.assert_triple(tx, entity_id, "last_reinforced", digest_ts)
                 stats["reinforced"] += 1
 
@@ -224,8 +263,9 @@ def _execute_graph_ops(db_path: str, ops: list[dict], digest_ts: str) -> dict:
                 }))
                 # Retract all attributes of this entity
                 attrs = store.entity(entity_id)
-                for a in attrs:
-                    store.retract_triple(tx, entity_id, a["attribute"], a["value"])
+                for attr_name, values in attrs.items():
+                    for val in values:
+                        store.retract_triple(tx, entity_id, attr_name, val)
                 stats["retracted"] += 1
 
         store.close()
@@ -335,6 +375,7 @@ def main() -> None:
     parser.add_argument("--memory-dir", required=True, help="Path to memory/ directory")
     parser.add_argument("--digest", default=None, help="SessionDigest JSON string")
     parser.add_argument("--bootstrap", action="store_true", help="One-time: seed graph from playbook")
+    parser.add_argument("--retag", action="store_true", help="Re-extract tags for all existing facts")
     args = parser.parse_args()
 
     memory_dir = args.memory_dir
@@ -346,9 +387,37 @@ def main() -> None:
         output_json(result)
         return
 
+    # Retag mode: extract tags for all existing facts
+    if args.retag:
+        if not Path(db_path).exists():
+            output_json({"error": "knowledge-graph.db not found"})
+            return
+        from triplestore import TripleStore
+        store = TripleStore(db_path)
+        # Get all fact entities that have a 'value' attribute
+        rows = store._conn.execute(
+            "SELECT DISTINCT entity_id FROM triples WHERE attribute = 'value' AND NOT retracted AND entity_id LIKE 'fact:%'"
+        ).fetchall()
+        tagged = 0
+        for row in rows:
+            fid = row["entity_id"]
+            attrs = store.entity(fid)
+            value_text = attrs.get("value", [""])[0] if attrs else ""
+            existing_tags = set(attrs.get("tag", [])) if attrs else set()
+            new_tags = _extract_tags(value_text)
+            missing = [t for t in new_tags if t not in existing_tags]
+            if missing:
+                tx = store.begin_tx("retag", metadata=json.dumps({"entity_id": fid}))
+                for tag in missing:
+                    store.assert_triple(tx, fid, "tag", tag)
+                tagged += 1
+        store.close()
+        output_json({"retagged": tagged, "total_facts": len(rows)})
+        return
+
     # Normal mode: integrate session digest
     if not args.digest:
-        print("--digest is required (unless --bootstrap)", file=sys.stderr)
+        print("--digest is required (unless --bootstrap or --retag)", file=sys.stderr)
         output_json({"error": "--digest required"})
         return