@geravant/sinain 1.0.19 → 1.1.0

package/sinain-core/src/learning/feedback-store.ts

@@ -0,0 +1,253 @@
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import type { FeedbackRecord, FeedbackSignals } from "../types.js";
+import { log, error } from "../log.js";
+
+const TAG = "feedback-store";
+
+/**
+ * Persistent JSONL feedback log — one file per day.
+ * Follows TraceStore pattern: daily rotation, WriteStream append.
+ *
+ * Storage: ~/.sinain-core/feedback/2025-02-03.jsonl
+ *
+ * Records are written at escalation time with null signals, then
+ * patched in-place by SignalCollector once deferred feedback arrives.
+ */
+export class FeedbackStore {
+  private dir: string;
+  private currentDate = "";
+  private currentStream: fs.WriteStream | null = null;
+  private retentionDays: number;
+
+  constructor(dir: string, retentionDays = 30) {
+    this.dir = dir;
+    this.retentionDays = retentionDays;
+    try {
+      fs.mkdirSync(dir, { recursive: true });
+    } catch (err: any) {
+      if (err.code !== "EEXIST") {
+        error(TAG, "failed to create feedback dir:", err.message);
+      }
+    }
+  }
+
+  /** Create a new FeedbackRecord with null signals. */
+  createRecord(params: {
+    tickId: number;
+    digest: string;
+    hud: string;
+    currentApp: string;
+    escalationScore: number;
+    escalationReasons: string[];
+    codingContext: boolean;
+    escalationMessage: string;
+    openclawResponse: string;
+    responseLatencyMs: number;
+  }): FeedbackRecord {
+    const record: FeedbackRecord = {
+      id: crypto.randomUUID(),
+      ts: Date.now(),
+      tickId: params.tickId,
+      digest: params.digest.slice(0, 2048),
+      hud: params.hud,
+      currentApp: params.currentApp,
+      escalationScore: params.escalationScore,
+      escalationReasons: params.escalationReasons,
+      codingContext: params.codingContext,
+      escalationMessage: params.escalationMessage.slice(0, 2048),
+      openclawResponse: params.openclawResponse.slice(0, 2048),
+      responseLatencyMs: params.responseLatencyMs,
+      signals: {
+        errorCleared: null,
+        noReEscalation: null,
+        dwellTimeMs: null,
+        quickAppSwitch: null,
+        compositeScore: 0,
+      },
+      tags: this.deriveTags(params),
+    };
+    return record;
+  }
+
+  /** Append a record to today's JSONL file. */
+  append(record: FeedbackRecord): void {
+    try {
+      this.rotateIfNeeded();
+      if (this.currentStream) {
+        this.currentStream.write(JSON.stringify(record) + "\n");
+      }
+    } catch (err: any) {
+      error(TAG, "append failed:", err.message);
+    }
+  }
+
+  /** Update signals for a record by ID. Reads + rewrites the day's file. */
+  updateSignals(recordId: string, date: string, signals: FeedbackSignals): boolean {
+    const filePath = path.join(this.dir, `${date}.jsonl`);
+    try {
+      const content = fs.readFileSync(filePath, "utf-8");
+      const lines = content.split("\n");
+      let updated = false;
+
+      const newLines = lines.map(line => {
+        if (!line.trim()) return line;
+        try {
+          const rec = JSON.parse(line) as FeedbackRecord;
+          if (rec.id === recordId) {
+            rec.signals = signals;
+            updated = true;
+            return JSON.stringify(rec);
+          }
+        } catch { /* skip malformed lines */ }
+        return line;
+      });
+
+      if (updated) {
+        fs.writeFileSync(filePath, newLines.join("\n"));
+        // If the stream points to this file, re-open it
+        if (date === this.currentDate) {
+          this.currentStream?.end();
+          this.currentStream = fs.createWriteStream(filePath, { flags: "a" });
+        }
+      }
+      return updated;
+    } catch {
+      return false;
+    }
+  }
+
+  /** Read all records for a given date. */
+  queryDay(date: string): FeedbackRecord[] {
+    const filePath = path.join(this.dir, `${date}.jsonl`);
+    try {
+      const content = fs.readFileSync(filePath, "utf-8");
+      return content.split("\n")
+        .filter(line => line.trim())
+        .map(line => JSON.parse(line) as FeedbackRecord);
+    } catch {
+      return [];
+    }
+  }
+
+  /** Read recent records across today and yesterday. */
+  queryRecent(limit = 20): FeedbackRecord[] {
+    const results: FeedbackRecord[] = [];
+    const today = new Date();
+
+    // Check today and up to 6 previous days to fill the limit
+    for (let d = 0; d < 7 && results.length < limit; d++) {
+      const date = new Date(today);
+      date.setDate(date.getDate() - d);
+      const dateStr = date.toISOString().slice(0, 10);
+      const dayRecords = this.queryDay(dateStr);
+      results.push(...dayRecords);
+    }
+
+    // Sort newest first, truncate
+    return results.sort((a, b) => b.ts - a.ts).slice(0, limit);
+  }
+
+  /** Aggregate stats across recent records. */
+  getStats(): Record<string, unknown> {
+    const records = this.queryRecent(100);
+    if (records.length === 0) {
+      return { totalRecords: 0, withSignals: 0, avgCompositeScore: null };
+    }
+
+    const withSignals = records.filter(r => r.signals.compositeScore !== 0 || r.signals.errorCleared !== null);
+    const scores = withSignals.map(r => r.signals.compositeScore).filter(s => s !== 0);
+    const avgScore = scores.length > 0
+      ? scores.reduce((a, b) => a + b, 0) / scores.length
+      : null;
+
+    // Tag distribution
+    const tagCounts: Record<string, number> = {};
+    for (const r of records) {
+      for (const t of r.tags) {
+        tagCounts[t] = (tagCounts[t] || 0) + 1;
+      }
+    }
+
+    return {
+      totalRecords: records.length,
+      withSignals: withSignals.length,
+      avgCompositeScore: avgScore !== null ? Math.round(avgScore * 1000) / 1000 : null,
+      avgLatencyMs: Math.round(records.reduce((s, r) => s + r.responseLatencyMs, 0) / records.length),
+      topTags: Object.entries(tagCounts).sort((a, b) => b[1] - a[1]).slice(0, 10),
+    };
+  }
+
+  /** Close the write stream. */
+  destroy(): void {
+    if (this.currentStream) {
+      this.currentStream.end();
+      this.currentStream = null;
+    }
+  }
+
+  /** Prune files older than retentionDays. */
+  prune(): number {
+    const cutoff = Date.now() - this.retentionDays * 86_400_000;
+    let pruned = 0;
+    try {
+      for (const file of fs.readdirSync(this.dir)) {
+        if (!file.endsWith(".jsonl")) continue;
+        const dateStr = file.replace(".jsonl", "");
+        const fileDate = new Date(dateStr).getTime();
+        if (fileDate && fileDate < cutoff) {
+          fs.unlinkSync(path.join(this.dir, file));
+          pruned++;
+        }
+      }
+      if (pruned > 0) log(TAG, `pruned ${pruned} old feedback files`);
+    } catch (err: any) {
+      error(TAG, "prune failed:", err.message);
+    }
+    return pruned;
+  }
+
+  // ── Private ──
+
+  private rotateIfNeeded(): void {
+    const date = new Date().toISOString().slice(0, 10);
+    if (date !== this.currentDate) {
+      if (this.currentStream) {
+        this.currentStream.end();
+      }
+      const filePath = path.join(this.dir, `${date}.jsonl`);
+      this.currentStream = fs.createWriteStream(filePath, { flags: "a" });
+      this.currentDate = date;
+      log(TAG, `writing to ${filePath}`);
+    }
+  }
+
+  private deriveTags(params: {
+    escalationReasons: string[];
+    currentApp: string;
+    codingContext: boolean;
+  }): string[] {
+    const tags: string[] = [];
+
+    // From escalation reasons
+    for (const r of params.escalationReasons) {
+      const category = r.split(":")[0];
+      if (category && !tags.includes(category)) {
+        tags.push(category);
+      }
+    }
+
+    // App category
+    if (params.currentApp) {
+      tags.push(`app:${params.currentApp.toLowerCase().slice(0, 30)}`);
+    }
+
+    // Coding context
+    if (params.codingContext) {
+      tags.push("coding");
+    }
+
+    return tags;
+  }
+}

package/sinain-core/src/learning/signal-collector.ts

@@ -0,0 +1,218 @@
+import type { FeedbackSignals, FeedbackRecord } from "../types.js";
+import type { FeedbackStore } from "./feedback-store.js";
+import type { AgentLoop } from "../agent/loop.js";
+import type { SenseBuffer } from "../buffers/sense-buffer.js";
+import { log, warn } from "../log.js";
+
+const TAG = "signal-collector";
+
+/** Error patterns matching scorer.ts */
+const ERROR_PATTERNS = [
+  "error", "failed", "failure", "exception", "crash", "traceback",
+  "typeerror", "referenceerror", "syntaxerror", "cannot read", "undefined is not",
+  "exit code", "segfault", "panic", "fatal", "enoent",
+];
+
+function hasErrorPattern(text: string): boolean {
+  const lower = text.toLowerCase();
+  return ERROR_PATTERNS.some(p => lower.includes(p));
+}
+
+interface PendingCollection {
+  recordId: string;
+  recordTs: number;
+  recordDate: string;       // YYYY-MM-DD for file lookup
+  escalationReasons: string[];
+  digestAtEscalation: string;
+  timers: ReturnType<typeof setTimeout>[];
+}
+
+/**
+ * Deferred signal backfill for feedback records.
+ *
+ * After each escalation, schedules checks at 60s, 120s, and 300s
+ * to read from existing buffers and compute feedback signals.
+ * At 300s (the final check), writes the composite score and persists.
+ */
+export class SignalCollector {
+  private pending = new Map<string, PendingCollection>();
+
+  constructor(
+    private feedbackStore: FeedbackStore,
+    private agentLoop: AgentLoop,
+    private senseBuffer: SenseBuffer,
+  ) {}
+
+  /** Schedule signal collection for a feedback record. */
+  schedule(record: FeedbackRecord): void {
+    const date = new Date(record.ts).toISOString().slice(0, 10);
+    const entry: PendingCollection = {
+      recordId: record.id,
+      recordTs: record.ts,
+      recordDate: date,
+      escalationReasons: record.escalationReasons,
+      digestAtEscalation: record.digest,
+      timers: [],
+    };
+
+    // Schedule partial collections at 60s and 120s, final at 300s
+    entry.timers.push(setTimeout(() => this.collect(entry, "partial"), 60_000));
+    entry.timers.push(setTimeout(() => this.collect(entry, "partial"), 120_000));
+    entry.timers.push(setTimeout(() => this.collect(entry, "final"), 300_000));
+
+    this.pending.set(record.id, entry);
+    log(TAG, `scheduled signal collection for record ${record.id} (tick #${record.tickId})`);
+  }
+
+  /** Cancel all pending collections. Called on shutdown. */
+  destroy(): void {
+    for (const entry of this.pending.values()) {
+      for (const t of entry.timers) clearTimeout(t);
+    }
+    this.pending.clear();
+  }
+
+  get pendingCount(): number {
+    return this.pending.size;
+  }
+
+  // ── Private ──
+
+  private collect(entry: PendingCollection, phase: "partial" | "final"): void {
+    try {
+      const signals = this.computeSignals(entry);
+
+      const updated = this.feedbackStore.updateSignals(
+        entry.recordId,
+        entry.recordDate,
+        signals,
+      );
+
+      if (phase === "final") {
+        this.pending.delete(entry.recordId);
+        log(TAG, `final signals for ${entry.recordId}: score=${signals.compositeScore.toFixed(2)}, err=${signals.errorCleared}, reesc=${signals.noReEscalation}`);
+      }
+
+      if (!updated && phase === "final") {
+        warn(TAG, `could not update signals for ${entry.recordId} — record not found in ${entry.recordDate}.jsonl`);
+      }
+    } catch (err: any) {
+      warn(TAG, `signal collection error for ${entry.recordId}: ${err.message}`);
+      if (phase === "final") {
+        this.pending.delete(entry.recordId);
+      }
+    }
+  }
+
+  private computeSignals(entry: PendingCollection): FeedbackSignals {
+    const now = Date.now();
+    const elapsedMs = now - entry.recordTs;
+
+    // ── errorCleared: check if error patterns are absent in recent digests ──
+    let errorCleared: boolean | null = null;
+    const hadError = entry.escalationReasons.some(r => r.startsWith("error:"));
+    if (hadError) {
+      // Look at the 3 most recent agent entries
+      const recentEntries = this.agentLoop.getHistory(3);
+      if (recentEntries.length > 0) {
+        // All recent entries should be free of error patterns
+        errorCleared = recentEntries.every(e => !hasErrorPattern(e.digest));
+      }
+    }
+
+    // ── noReEscalation: same reasons haven't fired within 5 min ──
+    // We check by looking at recent feedback records for overlapping reasons
+    let noReEscalation: boolean | null = null;
+    if (elapsedMs >= 60_000) {
+      const recentRecords = this.feedbackStore.queryRecent(10);
+      const reEscalated = recentRecords.some(r =>
+        r.id !== entry.recordId &&
+        r.ts > entry.recordTs &&
+        r.ts <= entry.recordTs + 300_000 &&
+        r.escalationReasons.some(reason => entry.escalationReasons.includes(reason))
+      );
+      noReEscalation = !reEscalated;
+    }
+
+    // ── dwellTimeMs: time from escalation until the next HUD push ──
+    let dwellTimeMs: number | null = null;
+    const historyEntries = this.agentLoop.getHistory(20);
+    for (const e of historyEntries) {
+      if (e.ts > entry.recordTs && e.pushed) {
+        dwellTimeMs = e.ts - entry.recordTs;
+        break;
+      }
+    }
+
+    // ── quickAppSwitch: app changed within 10s of escalation ──
+    let quickAppSwitch: boolean | null = null;
+    const appHistory = this.senseBuffer.appHistory(entry.recordTs);
+    if (appHistory.length >= 2) {
+      // Check if there was an app switch within 10s of escalation
+      const earlySwitch = appHistory.find(a =>
+        a.ts > entry.recordTs && a.ts <= entry.recordTs + 10_000
+      );
+      quickAppSwitch = earlySwitch !== undefined;
+    }
+
+    // ── compositeScore: weighted combination ──
+    const compositeScore = this.computeComposite({
+      errorCleared,
+      noReEscalation,
+      dwellTimeMs,
+      quickAppSwitch,
+    });
+
+    return {
+      errorCleared,
+      noReEscalation,
+      dwellTimeMs,
+      quickAppSwitch,
+      compositeScore,
+    };
+  }
+
+  private computeComposite(signals: {
+    errorCleared: boolean | null;
+    noReEscalation: boolean | null;
+    dwellTimeMs: number | null;
+    quickAppSwitch: boolean | null;
+  }): number {
+    let score = 0;
+    let weight = 0;
+
+    // Error cleared: strong positive (+0.5)
+    if (signals.errorCleared !== null) {
+      score += signals.errorCleared ? 0.5 : -0.3;
+      weight += 0.5;
+    }
+
+    // No re-escalation: positive (+0.3)
+    if (signals.noReEscalation !== null) {
+      score += signals.noReEscalation ? 0.3 : -0.2;
+      weight += 0.3;
+    }
+
+    // Dwell time: weak positive if > 60s
+    if (signals.dwellTimeMs !== null) {
+      if (signals.dwellTimeMs > 60_000) {
+        score += 0.15;
+      } else if (signals.dwellTimeMs < 10_000) {
+        score -= 0.1;
+      }
+      weight += 0.15;
+    }
+
+    // Quick app switch: weak negative
+    if (signals.quickAppSwitch !== null) {
+      score += signals.quickAppSwitch ? -0.15 : 0.05;
+      weight += 0.1;
+    }
+
+    // Normalize if we have signals, otherwise return 0
+    if (weight === 0) return 0;
+
+    // Clamp to [-1, 1]
+    return Math.max(-1, Math.min(1, score));
+  }
+}

package/sinain-core/src/log.ts

@@ -0,0 +1,24 @@
+/** Timestamped structured logger — writes to stderr for easy piping */
+
+const DEBUG = process.env.DEBUG === "true" || process.env.LOG_LEVEL === "debug";
+
+function ts(): string {
+  return new Date().toISOString();
+}
+
+export function debug(tag: string, ...args: unknown[]): void {
+  if (!DEBUG) return;
+  console.log(`[${ts()}] [${tag}] 🐛`, ...args);
+}
+
+export function log(tag: string, ...args: unknown[]): void {
+  console.log(`[${ts()}] [${tag}]`, ...args);
+}
+
+export function warn(tag: string, ...args: unknown[]): void {
+  console.warn(`[${ts()}] [${tag}] \u26a0`, ...args);
+}
+
+export function error(tag: string, ...args: unknown[]): void {
+  console.error(`[${ts()}] [${tag}] \u2718`, ...args);
+}

package/sinain-core/src/overlay/commands.ts

@@ -0,0 +1,126 @@
+import type { InboundMessage } from "../types.js";
+import type { WsHandler } from "./ws-handler.js";
+import type { AudioPipeline } from "../audio/pipeline.js";
+import type { CoreConfig } from "../types.js";
+import { WebSocket } from "ws";
+import { log } from "../log.js";
+
+const TAG = "cmd";
+
+export interface CommandDeps {
+  wsHandler: WsHandler;
+  systemAudioPipeline: AudioPipeline;
+  micPipeline: AudioPipeline | null;
+  config: CoreConfig;
+  onUserMessage: (text: string) => Promise<void>;
+  /** Toggle screen capture — returns new state */
+  onToggleScreen: () => boolean;
+  /** Toggle trait voices — returns new enabled state */
+  onToggleTraits?: () => boolean;
+}
+
+/**
+ * Handle overlay commands and user messages.
+ * Registers as the WS handler's onIncoming callback.
+ */
+export function setupCommands(deps: CommandDeps): void {
+  const { wsHandler } = deps;
+
+  wsHandler.onIncoming(async (msg: InboundMessage, _client: WebSocket) => {
+    switch (msg.type) {
+      case "message": {
+        log(TAG, `routing user message to OpenClaw`);
+        try {
+          await deps.onUserMessage(msg.text);
+        } catch {
+          wsHandler.broadcast("\u26a0 Failed to reach Sinain. Check gateway connection.", "high");
+        }
+        break;
+      }
+      case "command": {
+        handleCommand(msg.action, deps);
+        log(TAG, `command processed: ${msg.action}`);
+        break;
+      }
+    }
+  });
+}
+
+function handleCommand(action: string, deps: CommandDeps): void {
+  const { wsHandler, systemAudioPipeline, micPipeline, config } = deps;
+
+  switch (action) {
+    case "toggle_audio": {
+      const isSck = systemAudioPipeline.getCaptureCommand() === "screencapturekit";
+      if (systemAudioPipeline.isRunning() && !systemAudioPipeline.isMuted()) {
+        if (isSck) {
+          // sck-capture also captures screen — keep process alive, just mute audio
+          systemAudioPipeline.mute();
+          log(TAG, "system audio muted (sck-capture still running for screen)");
+        } else {
+          // sox/ffmpeg are audio-only — full stop
+          systemAudioPipeline.stop();
+          log(TAG, "system audio stopped");
+        }
+        wsHandler.broadcast("System audio muted", "normal");
+      } else if (systemAudioPipeline.isRunning() && systemAudioPipeline.isMuted()) {
+        systemAudioPipeline.unmute();
+        wsHandler.broadcast("System audio unmuted", "normal");
+        log(TAG, "system audio unmuted");
+      } else {
+        systemAudioPipeline.start();
+        wsHandler.broadcast("System audio capture started", "normal");
+        log(TAG, "system audio started (was not running)");
+      }
+      break;
+    }
+    case "toggle_mic": {
+      if (!micPipeline) {
+        wsHandler.broadcast("\u26a0 Mic not enabled (set MIC_ENABLED=true)", "normal");
+        log(TAG, "toggle_mic: mic not enabled");
+        break;
+      }
+      if (micPipeline.isRunning()) {
+        micPipeline.stop();
+        wsHandler.broadcast("Mic capture stopped", "normal");
+        log(TAG, "mic toggled OFF");
+      } else {
+        micPipeline.start();
+        wsHandler.broadcast("Mic capture started", "normal");
+        log(TAG, "mic toggled ON");
+      }
+      break;
+    }
+    case "toggle_screen": {
+      const nowActive = deps.onToggleScreen();
+      wsHandler.broadcast(
+        nowActive ? "Screen capture started" : "Screen capture stopped",
+        "normal"
+      );
+      log(TAG, `screen toggled ${nowActive ? "ON" : "OFF"}`);
+      break;
+    }
+    case "switch_device": {
+      const current = systemAudioPipeline.getDevice();
+      const alt = config.audioAltDevice;
+      const next = current === config.audioConfig.device ? alt : config.audioConfig.device;
+      systemAudioPipeline.switchDevice(next);
+      wsHandler.broadcast(`Audio device \u2192 ${next}`, "normal");
+      log(TAG, `audio device switched: ${current} \u2192 ${next}`);
+      break;
+    }
+    case "toggle_traits": {
+      if (!deps.onToggleTraits) {
+        wsHandler.broadcast("Trait voices not configured", "normal");
+        break;
+      }
+      const nowEnabled = deps.onToggleTraits();
+      wsHandler.updateState({ traits: nowEnabled ? "active" : "off" });
+      wsHandler.broadcast(`Trait voices ${nowEnabled ? "on" : "off"}`, "normal");
+      log(TAG, `traits toggled ${nowEnabled ? "ON" : "OFF"}`);
+      break;
+    }
+    default:
+      log(TAG, `unhandled command: ${action}`);
+  }
+}