@geravant/sinain 1.0.19 → 1.2.0

package/sinain-core/src/recorder.ts

@@ -0,0 +1,186 @@
+import type { FeedItem, SenseEvent, RecordCommand, RecorderStatus, StopResult } from "./types.js";
+import { log, warn } from "./log.js";
+
+const TAG = "recorder";
+
+/**
+ * Recorder collects audio transcripts during a recording session.
+ *
+ * Usage:
+ * - handleCommand({command: "start", label: "Meeting"}) → starts recording
+ * - onFeedItem() → called for each audio transcript, collects if recording
+ * - onSenseEvent() → tracks app context for title detection
+ * - handleCommand({command: "stop"}) → stops recording, returns StopResult with transcript
+ * - getStatus() → returns current RecorderStatus for prompt injection
+ */
+export class Recorder {
+  private recording = false;
+  private label: string | null = null;
+  private startedAt: number | null = null;
+  private segments: { text: string; ts: number }[] = [];
+  private lastApp: string = "";
+  private lastWindowTitle: string = "";
+
+  /**
+   * Handle a record command from the analyzer.
+   * Returns StopResult on stop if there are segments, otherwise undefined.
+   */
+  handleCommand(cmd: RecordCommand | undefined): StopResult | undefined {
+    if (!cmd) return undefined;
+
+    if (cmd.command === "start") {
+      return this.start(cmd.label);
+    } else if (cmd.command === "stop") {
+      return this.stop();
+    }
+    return undefined;
+  }
+
+  /**
+   * Start a new recording session.
+   */
+  private start(label?: string): undefined {
+    if (this.recording) {
+      log(TAG, `already recording "${this.label}" — ignoring start`);
+      return undefined;
+    }
+
+    this.recording = true;
+    this.label = label || null;
+    this.startedAt = Date.now();
+    this.segments = [];
+
+    const labelStr = label ? ` "${label}"` : "";
+    log(TAG, `started recording${labelStr}`);
+    return undefined;
+  }
+
+  /**
+   * Stop recording and return the collected transcript.
+   */
+  private stop(): StopResult | undefined {
+    if (!this.recording) {
+      log(TAG, "not recording — ignoring stop");
+      return undefined;
+    }
+
+    const title = this.buildTitle();
+    const transcript = this.buildTranscript();
+    const segments = this.segments.length;
+    const durationS = this.startedAt ? Math.round((Date.now() - this.startedAt) / 1000) : 0;
+
+    // Reset state
+    this.recording = false;
+    this.label = null;
+    this.startedAt = null;
+    this.segments = [];
+
+    log(TAG, `stopped recording: "${title}" (${segments} segments, ${durationS}s)`);
+
+    if (segments === 0) {
+      log(TAG, "no segments captured — returning undefined");
+      return undefined;
+    }
+
+    return { title, transcript, segments, durationS };
+  }
+
+  /**
+   * Called for each audio FeedItem. Collects if recording.
+   */
+  onFeedItem(item: FeedItem): void {
+    if (!this.recording) return;
+    if (item.source !== "audio") return;
+    if (!item.text || item.text.trim().length === 0) return;
+
+    // Strip audio source prefixes if present ([📝], [🔊], [🎤])
+    let text = item.text;
+    if (text.startsWith("[📝] ")) {
+      text = text.slice(5);
+    } else if (text.startsWith("[🔊] ")) {
+      text = text.slice(5);
+    } else if (text.startsWith("[🎤] ")) {
+      text = text.slice(5);
+    }
+
+    this.segments.push({ text: text.trim(), ts: item.ts });
+  }
+
+  /**
+   * Called for each SenseEvent. Tracks app context for title.
+   */
+  onSenseEvent(event: SenseEvent): void {
+    if (event.meta.app) {
+      this.lastApp = event.meta.app;
+    }
+    if (event.meta.windowTitle) {
+      this.lastWindowTitle = event.meta.windowTitle;
+    }
+  }
+
+  /**
+   * Get current recorder status for prompt injection.
+   */
+  getStatus(): RecorderStatus {
+    return {
+      recording: this.recording,
+      label: this.label,
+      startedAt: this.startedAt,
+      segments: this.segments.length,
+      durationMs: this.startedAt ? Date.now() - this.startedAt : 0,
+    };
+  }
+
+  /**
+   * Build a title from label or app context.
+   */
+  private buildTitle(): string {
+    if (this.label) return this.label;
+
+    // Try to build from app context
+    const app = this.lastApp.replace(/\.app$/i, "").trim();
+    if (app) {
+      // Common meeting apps
+      if (/zoom|meet|teams|slack|discord/i.test(app)) {
+        return `${app} call`;
+      }
+      return `Recording in ${app}`;
+    }
+
+    // Fallback with timestamp
+    const now = new Date();
+    return `Recording ${now.toLocaleDateString()} ${now.toLocaleTimeString()}`;
+  }
+
+  /**
+   * Build transcript from collected segments.
+   * Format: [MM:SS] text
+   */
+  private buildTranscript(): string {
+    if (this.segments.length === 0) return "";
+    if (!this.startedAt) return this.segments.map(s => s.text).join("\n");
+
+    const baseTs = this.startedAt;
+    return this.segments
+      .map(s => {
+        const offsetMs = s.ts - baseTs;
+        const offsetSec = Math.max(0, Math.floor(offsetMs / 1000));
+        const min = Math.floor(offsetSec / 60);
+        const sec = offsetSec % 60;
+        const timestamp = `[${String(min).padStart(2, "0")}:${String(sec).padStart(2, "0")}]`;
+        return `${timestamp} ${s.text}`;
+      })
+      .join("\n");
+  }
+
+  /**
+   * Force stop recording (e.g., on shutdown).
+   */
+  forceStop(): StopResult | undefined {
+    if (this.recording) {
+      warn(TAG, "force stopping recording");
+      return this.stop();
+    }
+    return undefined;
+  }
+}

package/sinain-core/src/server.ts

@@ -0,0 +1,456 @@
+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
+import { WebSocketServer, WebSocket } from "ws";
+import type { CoreConfig, SenseEvent } from "./types.js";
+import type { Profiler } from "./profiler.js";
+import type { FeedbackStore } from "./learning/feedback-store.js";
+import { FeedBuffer } from "./buffers/feed-buffer.js";
+import { SenseBuffer, type SemanticSenseEvent, type TextDelta } from "./buffers/sense-buffer.js";
+import { WsHandler } from "./overlay/ws-handler.js";
+import { log, error } from "./log.js";
+
+const TAG = "server";
+const MAX_SENSE_BODY = 2 * 1024 * 1024;
+
+/** Server epoch — lets clients detect restarts. */
+const serverEpoch = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
+
+export interface ServerDeps {
+  config: CoreConfig;
+  feedBuffer: FeedBuffer;
+  senseBuffer: SenseBuffer;
+  wsHandler: WsHandler;
+  profiler?: Profiler;
+  onSenseEvent: (event: SenseEvent) => void;
+  onSenseDelta?: (data: { app: string; activity: string; changes: TextDelta[]; priority?: string; ts: number }) => void;
+  onFeedPost: (text: string, priority: string) => void;
+  isScreenActive: () => boolean;
+  onSenseProfile: (snapshot: any) => void;
+  getHealthPayload: () => Record<string, unknown>;
+  getAgentDigest: () => unknown;
+  getAgentHistory: (limit: number) => unknown[];
+  getAgentContext: () => unknown;
+  getAgentConfig: () => unknown;
+  updateAgentConfig: (updates: Record<string, unknown>) => unknown;
+  getTraces: (after: number, limit: number) => unknown[];
+  reconnectGateway: () => void;
+  feedbackStore?: FeedbackStore;
+  getEscalationPending?: () => any;
+  respondEscalation?: (id: string, response: string) => any;
+  getKnowledgeDocPath?: () => string | null;
+  queryKnowledgeFacts?: (entities: string[], maxFacts: number) => Promise<string>;
+}
+
+function readBody(req: IncomingMessage, maxBytes: number): Promise<string> {
+  return new Promise((resolve, reject) => {
+    let body = "";
+    let bytes = 0;
+    req.on("data", (chunk: Buffer) => {
+      bytes += chunk.length;
+      if (bytes > maxBytes) {
+        reject(new Error("body too large"));
+        req.destroy();
+        return;
+      }
+      body += chunk;
+    });
+    req.on("end", () => resolve(body));
+    req.on("error", reject);
+  });
+}
+
+export function createAppServer(deps: ServerDeps) {
+  const { config, feedBuffer, senseBuffer, wsHandler } = deps;
+  let senseInBytes = 0;
+
+  const httpServer = createServer(async (req: IncomingMessage, res: ServerResponse) => {
+    res.setHeader("Access-Control-Allow-Origin", "*");
+    res.setHeader("Content-Type", "application/json");
+
+    if (req.method === "OPTIONS") {
+      res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+      res.setHeader("Access-Control-Allow-Headers", "Content-Type");
+      res.writeHead(204);
+      res.end();
+      return;
+    }
+
+    const url = new URL(req.url || "/", `http://localhost:${config.port}`);
+
+    try {
+      // ── /sense ──
+      if (req.method === "POST" && url.pathname === "/sense") {
+        if (!deps.isScreenActive()) {
+          res.end(JSON.stringify({ ok: true, gated: true }));
+          return;
+        }
+        const body = await readBody(req, MAX_SENSE_BODY);
+        senseInBytes += Buffer.byteLength(body);
+        deps.profiler?.gauge("network.senseInBytes", senseInBytes);
+        const data = JSON.parse(body);
+        if (!data.type || data.ts === undefined) {
+          res.writeHead(400);
+          res.end(JSON.stringify({ ok: false, error: "missing type or ts" }));
+          return;
+        }
+        // Extract image data from ROI if present
+        const imageData = data.roi?.data || undefined;
+        const imageBbox = data.roi?.bbox || undefined;
+
+        const event = senseBuffer.push({
+          type: data.type,
+          ts: data.ts,
+          ocr: data.ocr || "",
+          imageData,
+          imageBbox,
+          meta: {
+            ssim: data.meta?.ssim ?? 0,
+            app: data.meta?.app || "unknown",
+            windowTitle: data.meta?.windowTitle,
+            screen: data.meta?.screen ?? 0,
+          },
+        });
+        if (event) {
+          log(TAG, `[sense] #${event.id} (${event.type}): app=${event.meta.app} ssim=${event.meta.ssim?.toFixed(3)}`);
+          deps.onSenseEvent(event);
+          res.end(JSON.stringify({ ok: true, id: event.id }));
+        } else {
+          // Event was deduplicated
+          res.end(JSON.stringify({ ok: true, deduplicated: true }));
+        }
+        return;
+      }
+
+      if (req.method === "GET" && url.pathname === "/sense") {
+        const after = parseInt(url.searchParams.get("after") || "0");
+        const metaOnly = url.searchParams.get("meta_only") === "true";
+        const events = senseBuffer.query(after, metaOnly);
+        res.end(JSON.stringify({ events, epoch: serverEpoch }));
+        return;
+      }
+
+      // ── /sense/context (structured semantic context) ──
+      if (req.method === "GET" && url.pathname === "/sense/context") {
+        const limit = Math.min(parseInt(url.searchParams.get("limit") || "10"), 50);
+        const includeDeltas = url.searchParams.get("include_deltas") === "true";
+        const includeSummary = url.searchParams.get("include_summary") !== "false";
+        const context = senseBuffer.getStructuredContext({
+          limit,
+          includeDeltas,
+          includeSummary,
+        });
+        res.end(JSON.stringify({ ok: true, context, epoch: serverEpoch }));
+        return;
+      }
+
+      // ── /sense/activity (activity breakdown) ──
+      if (req.method === "GET" && url.pathname === "/sense/activity") {
+        const since = parseInt(url.searchParams.get("since") || "0");
+        const breakdown = senseBuffer.getActivityBreakdown(since);
+        res.end(JSON.stringify({
+          ok: true,
+          activity: senseBuffer.latestActivity(),
+          breakdown,
+          epoch: serverEpoch,
+        }));
+        return;
+      }
+
+      // ── /sense/deltas (accumulated deltas) ──
+      if (req.method === "GET" && url.pathname === "/sense/deltas") {
+        const flush = url.searchParams.get("flush") === "true";
+        const deltas = senseBuffer.getAccumulatedDeltas(flush);
+        res.end(JSON.stringify({ ok: true, deltas, count: deltas.length }));
+        return;
+      }
+
+      // ── /feed ──
+      if (req.method === "GET" && url.pathname === "/feed") {
+        const after = parseInt(url.searchParams.get("after") || "0");
+        const items = feedBuffer.query(after);
+        res.end(JSON.stringify({ messages: items, epoch: serverEpoch }));
+        return;
+      }
+
+      if (req.method === "POST" && url.pathname === "/feed") {
+        const body = await readBody(req, 65536);
+        const { text, priority } = JSON.parse(body);
+        deps.onFeedPost(text, priority || "normal");
+        res.end(JSON.stringify({ ok: true }));
+        return;
+      }
+
+      // ── /agent ──
+      if (req.method === "GET" && url.pathname === "/agent/digest") {
+        res.end(JSON.stringify({ ok: true, digest: deps.getAgentDigest() }));
+        return;
+      }
+
+      if (req.method === "GET" && url.pathname === "/agent/history") {
+        const limit = Math.min(parseInt(url.searchParams.get("limit") || "10"), 50);
+        res.end(JSON.stringify({ ok: true, results: deps.getAgentHistory(limit) }));
+        return;
+      }
+
+      if (req.method === "GET" && url.pathname === "/agent/context") {
+        res.end(JSON.stringify({ ok: true, context: deps.getAgentContext() }));
+        return;
+      }
+
+      if (req.method === "GET" && url.pathname === "/agent/config") {
+        res.end(JSON.stringify({ ok: true, config: deps.getAgentConfig() }));
+        return;
+      }
+
+      if (req.method === "POST" && url.pathname === "/agent/config") {
+        const body = await readBody(req, 4096);
+        const updates = JSON.parse(body);
+        const result = deps.updateAgentConfig(updates);
+        res.end(JSON.stringify({ ok: true, config: result }));
+        return;
+      }
+
+      // ── /knowledge ──
+      if (req.method === "GET" && url.pathname === "/knowledge") {
+        // Return portable knowledge document
+        const knowledgePath = deps.getKnowledgeDocPath?.();
+        if (knowledgePath) {
+          try {
+            const { readFileSync } = await import("node:fs");
+            const content = readFileSync(knowledgePath, "utf-8");
+            res.end(JSON.stringify({ ok: true, content }));
+          } catch {
+            res.end(JSON.stringify({ ok: true, content: "" }));
+          }
+        } else {
+          res.end(JSON.stringify({ ok: true, content: "" }));
+        }
+        return;
+      }
+
+      if (req.method === "GET" && url.pathname === "/knowledge/facts") {
+        // Query knowledge graph for entity-matched facts
+        const entitiesParam = url.searchParams.get("entities") || "";
+        const maxFacts = Math.min(parseInt(url.searchParams.get("max") || "5"), 20);
+        const entities = entitiesParam.split(",").map(e => e.trim()).filter(Boolean);
+
+        if (deps.queryKnowledgeFacts) {
+          try {
+            const facts = await deps.queryKnowledgeFacts(entities, maxFacts);
+            res.end(JSON.stringify({ ok: true, facts }));
+          } catch (err) {
+            res.end(JSON.stringify({ ok: true, facts: [], error: String(err) }));
+          }
+        } else {
+          res.end(JSON.stringify({ ok: true, facts: [] }));
+        }
+        return;
+      }
+
+      // ── /traces ──
+      if (req.method === "GET" && url.pathname === "/traces") {
+        const after = parseInt(url.searchParams.get("after") || "0");
+        const limit = Math.min(parseInt(url.searchParams.get("limit") || "50"), 500);
+        res.end(JSON.stringify({ traces: deps.getTraces(after, limit) }));
+        return;
+      }
+
+      // ── /profiling/sense ──
+      if (req.method === "POST" && url.pathname === "/profiling/sense") {
+        const body = await readBody(req, 4096);
+        deps.onSenseProfile(JSON.parse(body));
+        res.end(JSON.stringify({ ok: true }));
+        return;
+      }
+
+      // ── /learning/feedback ──
+      if (req.method === "GET" && url.pathname === "/learning/feedback") {
+        if (!deps.feedbackStore) {
+          res.end(JSON.stringify({ ok: false, error: "learning disabled" }));
+          return;
+        }
+        const limit = Math.min(parseInt(url.searchParams.get("limit") || "20"), 100);
+        const records = deps.feedbackStore.queryRecent(limit);
+        res.end(JSON.stringify({ ok: true, records, count: records.length }));
+        return;
+      }
+
+      // ── /learning/stats ──
+      if (req.method === "GET" && url.pathname === "/learning/stats") {
+        if (!deps.feedbackStore) {
+          res.end(JSON.stringify({ ok: false, error: "learning disabled" }));
+          return;
+        }
+        const stats = deps.feedbackStore.getStats();
+        res.end(JSON.stringify({ ok: true, ...stats }));
+        return;
+      }
+
+      // ── /reconnect-gateway ──
+      if (req.method === "POST" && url.pathname === "/reconnect-gateway") {
+        deps.reconnectGateway();
+        res.end(JSON.stringify({ ok: true, message: "gateway reconnection initiated" }));
+        return;
+      }
+
+      // ── /health ──
+      if (req.method === "GET" && url.pathname === "/health") {
+        res.end(JSON.stringify({
+          ok: true,
+          epoch: serverEpoch,
+          messages: feedBuffer.size,
+          senseEvents: senseBuffer.size,
+          overlayClients: wsHandler.clientCount,
+          ...deps.getHealthPayload(),
+        }));
+        return;
+      }
+
+      // ── /escalation/pending ──
+      if (req.method === "GET" && url.pathname === "/escalation/pending") {
+        const pending = deps.getEscalationPending?.();
+        res.end(JSON.stringify({ ok: true, escalation: pending ?? null }));
+        return;
+      }
+
+      // ── /escalation/respond ──
+      if (req.method === "POST" && url.pathname === "/escalation/respond") {
+        const body = await readBody(req, 65536);
+        const { id, response } = JSON.parse(body);
+        if (!id || !response) {
+          res.writeHead(400);
+          res.end(JSON.stringify({ ok: false, error: "missing id or response" }));
+          return;
+        }
+        const result = deps.respondEscalation?.(id, response) ?? { ok: false, error: "escalation not configured" };
+        res.end(JSON.stringify(result));
+        return;
+      }
+
+      res.writeHead(404);
+      res.end(JSON.stringify({ error: "not found" }));
+    } catch (err: any) {
+      const status = err.message === "body too large" ? 413 : 400;
+      res.writeHead(status);
+      res.end(JSON.stringify({ ok: false, error: err.message }));
+    }
+  });
+
+  // Attach WS server on the same HTTP server
+  const wss = new WebSocketServer({ server: httpServer });
+  wss.on("connection", (ws, req) => {
+    const pathname = new URL(req.url || "/", `http://localhost:${config.port}`).pathname;
+
+    // Sense WebSocket endpoint for low-latency event streaming
+    if (pathname === "/sense/ws") {
+      log(TAG, "[sense/ws] client connected");
+
+      // Backpressure tracking
+      let pendingAcks = 0;
+      const MAX_PENDING = 5;
+
+      ws.on("message", (data) => {
+        try {
+          const msg = JSON.parse(data.toString());
+
+          // Gate sense ingestion when screen is toggled off
+          if (!deps.isScreenActive()) {
+            ws.send(JSON.stringify({ type: "ack", gated: true }));
+            return;
+          }
+
+          // Handle different message types
+          if (msg.type === "delta") {
+            // Delta-only update (new semantic format)
+            senseBuffer.pushDelta({
+              app: msg.app || "unknown",
+              activity: msg.activity || "unknown",
+              changes: msg.changes || [],
+              priority: msg.priority,
+              ts: msg.ts || Date.now(),
+            });
+
+            // Trigger immediate context update for urgent priority
+            if (msg.priority === "urgent") {
+              deps.onSenseDelta?.(msg);
+            }
+
+            // Send ack with backpressure signal
+            pendingAcks++;
+            const backpressure = pendingAcks > MAX_PENDING ? 100 : 0;
+            ws.send(JSON.stringify({ type: "ack", backpressure }));
+            pendingAcks = Math.max(0, pendingAcks - 1);
+
+          } else {
+            // Full event (backwards compatible)
+            const imageData = msg.roi?.data || undefined;
+            const imageBbox = msg.roi?.bbox || undefined;
+
+            const event = senseBuffer.push({
+              type: msg.type,
+              ts: msg.ts,
+              ocr: msg.ocr || "",
+              imageData,
+              imageBbox,
+              meta: {
+                ssim: msg.meta?.ssim ?? 0,
+                app: msg.meta?.app || "unknown",
+                windowTitle: msg.meta?.windowTitle,
+                screen: msg.meta?.screen ?? 0,
+              },
+              semantic: msg.semantic,
+              priority: msg.priority,
+            });
+
+            if (event) {
+              deps.onSenseEvent(event);
+
+              // Send ack with event ID
+              pendingAcks++;
+              const backpressure = pendingAcks > MAX_PENDING ? 100 : 0;
+              ws.send(JSON.stringify({ type: "ack", id: event.id, backpressure }));
+              pendingAcks = Math.max(0, pendingAcks - 1);
+            } else {
+              // Deduplicated
+              ws.send(JSON.stringify({ type: "ack", deduplicated: true }));
+            }
+          }
+        } catch (err: any) {
+          ws.send(JSON.stringify({ type: "error", message: err.message }));
+        }
+      });
+
+      ws.on("close", () => {
+        log(TAG, "[sense/ws] client disconnected");
+      });
+
+      ws.on("error", (err) => {
+        error(TAG, `[sense/ws] error: ${err.message}`);
+      });
+
+      return;
+    }
+
+    // Default: overlay WebSocket handler
+    wsHandler.handleConnection(ws, req);
+  });
+
+  return {
+    httpServer,
+    wss,
+    start(): Promise<void> {
+      return new Promise((resolve, reject) => {
+        httpServer.on("error", reject);
+        httpServer.listen(config.port, "0.0.0.0", () => {
+          log(TAG, `listening on http://0.0.0.0:${config.port} (HTTP + WS, epoch=${serverEpoch})`);
+          resolve();
+        });
+      });
+    },
+    async destroy(): Promise<void> {
+      wsHandler.destroy();
+      wss.close();
+      await new Promise<void>((resolve) => httpServer.close(() => resolve()));
+      log(TAG, "server closed");
+    },
+  };
+}