@usejarvis/brain 0.1.0

package/src/awareness/awareness.test.ts

@@ -0,0 +1,332 @@
+import { test, expect, describe, beforeEach } from 'bun:test';
+import { initDatabase } from '../vault/schema.ts';
+import { ContextTracker } from './context-tracker.ts';
+import { SuggestionEngine } from './suggestion-engine.ts';
+import { ContextGraph } from './context-graph.ts';
+import type { AwarenessConfig } from '../config/types.ts';
+import type { AwarenessEvent, ScreenContext } from './types.ts';
+import {
+  createCapture,
+  getCapture,
+  getRecentCaptures,
+  getCapturesInRange,
+  getAppUsageStats,
+  createSession,
+  getSession,
+  updateSession,
+  endSession,
+  getRecentSessions,
+  incrementSessionCaptureCount,
+  createSuggestion,
+  getRecentSuggestions,
+  markSuggestionDismissed,
+  markSuggestionActedOn,
+  getSuggestionStats,
+  getSuggestionCountSince,
+} from '../vault/awareness.ts';
+
+const testConfig: AwarenessConfig = {
+  enabled: true,
+  capture_interval_ms: 5000,
+  min_change_threshold: 0.02,
+  cloud_vision_enabled: false,
+  cloud_vision_cooldown_ms: 30000,
+  stuck_threshold_ms: 5000, // 5s for tests
+  suggestion_rate_limit_ms: 100, // fast for tests
+  retention: { full_hours: 1, key_moment_hours: 24 },
+  capture_dir: '/tmp/jarvis-test-captures',
+  struggle_grace_ms: 120000,
+  struggle_cooldown_ms: 180000,
+  overlay_autolaunch: false,
+};
+
+describe('Vault — Screen Captures', () => {
+  beforeEach(() => initDatabase(':memory:'));
+
+  test('createCapture + getCapture', () => {
+    const row = createCapture({
+      timestamp: Date.now(),
+      pixelChangePct: 0.15,
+      appName: 'VS Code',
+      windowTitle: 'index.ts - jarvis - Visual Studio Code',
+      ocrText: 'function hello() { return "world"; }',
+    });
+    expect(row.id).toBeTruthy();
+    expect(row.app_name).toBe('VS Code');
+    expect(row.pixel_change_pct).toBe(0.15);
+
+    const fetched = getCapture(row.id);
+    expect(fetched).not.toBeNull();
+    expect(fetched!.ocr_text).toContain('hello');
+  });
+
+  test('getRecentCaptures with app filter', () => {
+    createCapture({ timestamp: Date.now() - 2000, pixelChangePct: 0.1, appName: 'Chrome' });
+    createCapture({ timestamp: Date.now() - 1000, pixelChangePct: 0.2, appName: 'VS Code' });
+    createCapture({ timestamp: Date.now(), pixelChangePct: 0.3, appName: 'Chrome' });
+
+    const all = getRecentCaptures(10);
+    expect(all.length).toBe(3);
+
+    const chromeOnly = getRecentCaptures(10, 'Chrome');
+    expect(chromeOnly.length).toBe(2);
+    expect(chromeOnly.every(c => c.app_name === 'Chrome')).toBe(true);
+  });
+
+  test('getCapturesInRange', () => {
+    const now = Date.now();
+    createCapture({ timestamp: now - 60000, pixelChangePct: 0.1, appName: 'A' });
+    createCapture({ timestamp: now - 30000, pixelChangePct: 0.2, appName: 'B' });
+    createCapture({ timestamp: now, pixelChangePct: 0.3, appName: 'C' });
+
+    const range = getCapturesInRange(now - 40000, now - 10000);
+    expect(range.length).toBe(1);
+    expect(range[0]!.app_name).toBe('B');
+  });
+
+  test('getAppUsageStats', () => {
+    const now = Date.now();
+    for (let i = 0; i < 5; i++) {
+      createCapture({ timestamp: now - i * 1000, pixelChangePct: 0.1, appName: 'Chrome' });
+    }
+    for (let i = 0; i < 3; i++) {
+      createCapture({ timestamp: now - i * 1000, pixelChangePct: 0.1, appName: 'VS Code' });
+    }
+
+    const stats = getAppUsageStats(now - 10000, now + 1000);
+    expect(stats.length).toBe(2);
+    expect(stats[0]!.app).toBe('Chrome');
+    expect(stats[0]!.captureCount).toBe(5);
+    expect(stats[1]!.app).toBe('VS Code');
+  });
+});
+
+describe('Vault — Awareness Sessions', () => {
+  beforeEach(() => initDatabase(':memory:'));
+
+  test('create + get + update + end session', () => {
+    const session = createSession({ startedAt: Date.now(), apps: ['Chrome', 'VS Code'] });
+    expect(session.id).toBeTruthy();
+    expect(session.ended_at).toBeNull();
+
+    const fetched = getSession(session.id);
+    expect(fetched).not.toBeNull();
+    expect(JSON.parse(fetched!.apps)).toEqual(['Chrome', 'VS Code']);
+
+    updateSession(session.id, { topic: 'Coding session', capture_count: 10 });
+    const updated = getSession(session.id);
+    expect(updated!.topic).toBe('Coding session');
+    expect(updated!.capture_count).toBe(10);
+
+    endSession(session.id, 'Productive coding');
+    const ended = getSession(session.id);
+    expect(ended!.ended_at).not.toBeNull();
+    expect(ended!.summary).toBe('Productive coding');
+  });
+
+  test('incrementSessionCaptureCount', () => {
+    const session = createSession({ startedAt: Date.now() });
+    incrementSessionCaptureCount(session.id);
+    incrementSessionCaptureCount(session.id);
+    incrementSessionCaptureCount(session.id);
+
+    const updated = getSession(session.id);
+    expect(updated!.capture_count).toBe(3);
+  });
+
+  test('getRecentSessions', () => {
+    createSession({ startedAt: Date.now() - 3000 });
+    createSession({ startedAt: Date.now() - 2000 });
+    createSession({ startedAt: Date.now() - 1000 });
+
+    const sessions = getRecentSessions(2);
+    expect(sessions.length).toBe(2);
+  });
+});
+
+describe('Vault — Awareness Suggestions', () => {
+  beforeEach(() => initDatabase(':memory:'));
+
+  test('create + dismiss + act on suggestions', () => {
+    const s = createSuggestion({
+      type: 'error',
+      title: 'Error in VS Code',
+      body: 'TypeScript compilation error detected',
+      context: { appName: 'VS Code' },
+    });
+    expect(s.id).toBeTruthy();
+    expect(s.dismissed).toBe(0);
+    expect(s.acted_on).toBe(0);
+
+    markSuggestionDismissed(s.id);
+    const recent = getRecentSuggestions(1);
+    expect(recent[0]!.dismissed).toBe(1);
+
+    const s2 = createSuggestion({ type: 'stuck', title: 'Stuck', body: 'You seem stuck' });
+    markSuggestionActedOn(s2.id);
+    const all = getRecentSuggestions(10);
+    expect(all.find(x => x.id === s2.id)!.acted_on).toBe(1);
+  });
+
+  test('getSuggestionStats', () => {
+    const now = Date.now();
+    createSuggestion({ type: 'error', title: 'E1', body: 'b' });
+    const s2 = createSuggestion({ type: 'stuck', title: 'E2', body: 'b' });
+    markSuggestionActedOn(s2.id);
+
+    const stats = getSuggestionStats(now - 10000, now + 10000);
+    expect(stats.total).toBe(2);
+    expect(stats.actedOn).toBe(1);
+  });
+
+  test('getSuggestionCountSince', () => {
+    const before = Date.now() - 1000;
+    createSuggestion({ type: 'general', title: 'T', body: 'B' });
+    createSuggestion({ type: 'general', title: 'T2', body: 'B2' });
+
+    expect(getSuggestionCountSince(before)).toBe(2);
+    expect(getSuggestionCountSince(Date.now() + 10000)).toBe(0);
+  });
+});
+
+describe('ContextTracker', () => {
+  beforeEach(() => initDatabase(':memory:'));
+
+  test('detects context changes', () => {
+    const tracker = new ContextTracker(testConfig);
+
+    // First capture
+    const r1 = tracker.processCapture('cap-1', 'some text', 'index.ts - VS Code');
+    expect(r1.context.appName).toBe('VS Code');
+    expect(r1.events.some(e => e.type === 'session_started')).toBe(true);
+
+    // Same app — no context change
+    const r2 = tracker.processCapture('cap-2', 'more text', 'utils.ts - VS Code');
+    expect(r2.context.appName).toBe('VS Code');
+
+    // Different app — context change
+    const r3 = tracker.processCapture('cap-3', 'google.com', 'Google - Chrome');
+    expect(r3.context.appName).toBe('Chrome');
+    expect(r3.events.some(e => e.type === 'context_changed')).toBe(true);
+  });
+
+  test('detects errors in OCR text', () => {
+    const tracker = new ContextTracker(testConfig);
+
+    const r = tracker.processCapture('cap-1', 'Compilation error: module not found. Build failed at line 42', 'app.js - VS Code');
+    expect(r.events.some(e => e.type === 'error_detected')).toBe(true);
+  });
+
+  test('detects stuck state', () => {
+    const tracker = new ContextTracker(testConfig);
+
+    // First capture
+    tracker.processCapture('cap-1', 'same text here', 'Page - Browser');
+
+    // Simulate time passing (>5s with same text)
+    const originalNow = Date.now;
+    Date.now = () => originalNow() + 6000;
+
+    const r2 = tracker.processCapture('cap-2', 'same text here', 'Page - Browser');
+    expect(r2.events.some(e => e.type === 'stuck_detected')).toBe(true);
+
+    Date.now = originalNow;
+  });
+
+  test('extracts app name from window title', () => {
+    const tracker = new ContextTracker(testConfig);
+
+    const r1 = tracker.processCapture('1', '', 'index.ts - Visual Studio Code');
+    expect(r1.context.appName).toBe('Visual Studio Code');
+
+    const r2 = tracker.processCapture('2', '', 'Google - Mozilla Firefox');
+    expect(r2.context.appName).toBe('Mozilla Firefox');
+  });
+});
+
+describe('SuggestionEngine', () => {
+  beforeEach(() => initDatabase(':memory:'));
+
+  test('generates error suggestion', async () => {
+    const engine = new SuggestionEngine(0); // no rate limit for test
+
+    const context: ScreenContext = {
+      captureId: 'cap-1',
+      timestamp: Date.now(),
+      appName: 'VS Code',
+      windowTitle: 'test.ts - VS Code',
+      url: null,
+      filePath: null,
+      ocrText: 'TypeError: undefined',
+      sessionId: 'sess-1',
+      isSignificantChange: false,
+    };
+
+    const events: AwarenessEvent[] = [{
+      type: 'error_detected',
+      data: { errorText: 'TypeError', errorContext: 'undefined is not a function', appName: 'VS Code' },
+      timestamp: Date.now(),
+    }];
+
+    const suggestion = await engine.evaluate(context, events);
+    expect(suggestion).not.toBeNull();
+    expect(suggestion!.type).toBe('error');
+    expect(suggestion!.title).toContain('VS Code');
+  });
+
+  test('deduplicates suggestions', async () => {
+    const engine = new SuggestionEngine(0);
+
+    const context: ScreenContext = {
+      captureId: 'cap-1',
+      timestamp: Date.now(),
+      appName: 'VS Code',
+      windowTitle: 'test.ts - VS Code',
+      url: null,
+      filePath: null,
+      ocrText: 'error',
+      sessionId: 'sess-1',
+      isSignificantChange: false,
+    };
+
+    const events: AwarenessEvent[] = [{
+      type: 'error_detected',
+      data: { errorText: 'error', errorContext: 'some error', appName: 'VS Code' },
+      timestamp: Date.now(),
+    }];
+
+    const s1 = await engine.evaluate(context, events);
+    expect(s1).not.toBeNull();
+
+    // Same suggestion should be deduped
+    const s2 = await engine.evaluate(context, events);
+    expect(s2).toBeNull();
+  });
+});
+
+describe('ContextGraph', () => {
+  beforeEach(() => initDatabase(':memory:'));
+
+  test('creates app entity for new apps', () => {
+    const graph = new ContextGraph();
+    const { searchEntitiesByName } = require('../vault/entities.ts');
+
+    const context: ScreenContext = {
+      captureId: 'cap-1',
+      timestamp: Date.now(),
+      appName: 'Visual Studio Code',
+      windowTitle: 'test.ts - Visual Studio Code',
+      url: null,
+      filePath: null,
+      ocrText: 'some code here',
+      sessionId: 'sess-1',
+      isSignificantChange: false,
+    };
+
+    graph.linkCaptureToEntities(context);
+
+    const entities = searchEntitiesByName('Visual Studio Code');
+    expect(entities.length).toBeGreaterThan(0);
+    expect(entities[0].type).toBe('tool');
+  });
+});

package/src/awareness/capture-engine.ts

@@ -0,0 +1,305 @@
+/**
+ * Capture Engine — Continuous Desktop Screenshot Capture
+ *
+ * Implements the Observer interface. Captures the full desktop at configurable
+ * intervals, computes pixel-diff to skip unchanged frames, manages disk storage
+ * with tiered retention, and adapts capture frequency based on system load.
+ */
+
+import { mkdirSync, existsSync, unlinkSync, readdirSync, statSync, rmdirSync } from 'node:fs';
+import { writeFileSync } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import type { Observer, ObserverEventHandler } from '../observers/index.ts';
+import type { AwarenessConfig } from '../config/types.ts';
+import type { DesktopController } from '../actions/app-control/desktop-controller.ts';
+import { generateId } from '../vault/schema.ts';
+import { deleteCapturesBefore } from '../vault/awareness.ts';
+import type { CaptureFrame } from './types.ts';
+
+let sharp: any = null;
+try {
+  sharp = require('sharp');
+} catch { /* sharp not available — thumbnails disabled */ }
+
+export class CaptureEngine implements Observer {
+  name = 'awareness-capture';
+
+  private config: AwarenessConfig;
+  private desktop: DesktopController;
+  private handler: ObserverEventHandler | null = null;
+  private running = false;
+  private captureTimer: ReturnType<typeof setInterval> | null = null;
+  private cleanupTimer: ReturnType<typeof setInterval> | null = null;
+  private previousBuffer: Buffer | null = null;
+  private latestFrame: CaptureFrame | null = null;
+  private captureDir: string;
+  private currentIntervalMs: number;
+
+  constructor(config: AwarenessConfig, desktop: DesktopController) {
+    this.config = config;
+    this.desktop = desktop;
+    this.captureDir = config.capture_dir.replace(/^~/, os.homedir());
+    this.currentIntervalMs = config.capture_interval_ms;
+  }
+
+  async start(): Promise<void> {
+    if (this.running) return;
+    this.running = true;
+
+    // Ensure capture directory exists
+    mkdirSync(this.captureDir, { recursive: true });
+
+    console.log(`[CaptureEngine] Started — interval: ${this.currentIntervalMs}ms, dir: ${this.captureDir}`);
+
+    // Start capture loop
+    this.captureTimer = setInterval(() => this.captureLoop(), this.currentIntervalMs);
+
+    // Start retention cleanup every 10 minutes
+    this.cleanupTimer = setInterval(() => this.cleanupRetention(), 10 * 60 * 1000);
+  }
+
+  async stop(): Promise<void> {
+    if (!this.running) return;
+    this.running = false;
+
+    if (this.captureTimer) {
+      clearInterval(this.captureTimer);
+      this.captureTimer = null;
+    }
+    if (this.cleanupTimer) {
+      clearInterval(this.cleanupTimer);
+      this.cleanupTimer = null;
+    }
+
+    console.log('[CaptureEngine] Stopped');
+  }
+
+  isRunning(): boolean {
+    return this.running;
+  }
+
+  onEvent(handler: ObserverEventHandler): void {
+    this.handler = handler;
+  }
+
+  getLatestFrame(): CaptureFrame | null {
+    return this.latestFrame;
+  }
+
+  getPreviousBuffer(): Buffer | null {
+    return this.previousBuffer;
+  }
+
+  private capturing = false;
+  private connectRetries = 0;
+  private readonly MAX_CONNECT_RETRIES = 3;
+
+  private async captureLoop(): Promise<void> {
+    if (!this.running || this.capturing) return;
+    this.capturing = true;
+
+    try {
+      // Adaptive throttle: check system load (use CPU count as baseline)
+      const cpuCount = os.cpus().length;
+      const load = os.loadavg()[0];
+      const loadThreshold = Math.max(cpuCount * 1.5, 8);
+      if (load! > loadThreshold) return;
+
+      // Connect to sidecar with timeout
+      if (!this.desktop.connected) {
+        if (this.connectRetries >= this.MAX_CONNECT_RETRIES) {
+          // Stop retrying — sidecar isn't available
+          if (this.connectRetries === this.MAX_CONNECT_RETRIES) {
+            console.error('[CaptureEngine] Sidecar unavailable after 3 attempts — stopping captures');
+            this.connectRetries++;
+          }
+          return;
+        }
+
+        try {
+          const connectPromise = this.desktop.connect();
+          const timeoutPromise = new Promise<never>((_, reject) =>
+            setTimeout(() => reject(new Error('Sidecar connect timeout (10s)')), 10000)
+          );
+          await Promise.race([connectPromise, timeoutPromise]);
+          this.connectRetries = 0;
+          console.log('[CaptureEngine] Sidecar connected');
+        } catch (err) {
+          this.connectRetries++;
+          console.error(`[CaptureEngine] Sidecar connect failed (attempt ${this.connectRetries}/${this.MAX_CONNECT_RETRIES}):`, err instanceof Error ? err.message : err);
+          return;
+        }
+      }
+
+      const imageBuffer = await this.desktop.captureScreen();
+
+      // Pixel diff against previous frame
+      const changePct = this.computePixelDiff(imageBuffer);
+
+      if (changePct < this.config.min_change_threshold && this.previousBuffer) {
+        return;
+      }
+
+      const id = generateId();
+      const now = Date.now();
+
+      // Save to disk
+      const imagePath = this.saveCapture(imageBuffer, now);
+
+      // Generate thumbnail (non-blocking, best-effort)
+      const thumbnailPath = await this.generateThumbnail(imagePath);
+
+      const frame: CaptureFrame = {
+        id,
+        timestamp: now,
+        imageBuffer,
+        pixelChangePct: changePct,
+      };
+
+      this.latestFrame = frame;
+      this.previousBuffer = imageBuffer;
+
+      // Emit event for processing pipeline
+      if (this.handler) {
+        this.handler({
+          type: 'screen_capture',
+          data: {
+            captureId: id,
+            pixelChangePct: changePct,
+            imagePath,
+            thumbnailPath,
+            imageBuffer,
+          },
+          timestamp: now,
+        });
+      }
+    } catch (err) {
+      console.error('[CaptureEngine] Capture failed:', err instanceof Error ? err.message : err);
+      // If socket died, reset connection state
+      if (err instanceof Error && (err.message.includes('ECONNREFUSED') || err.message.includes('destroyed') || err.message.includes('closed'))) {
+        try { await this.desktop.disconnect(); } catch { /* ignore */ }
+      }
+    } finally {
+      this.capturing = false;
+    }
+  }
+
+  /**
+   * Fast pixel diff using sampled byte comparison.
+   * Compares every 100th byte of the raw buffer. Returns percentage of changed samples (0.0-1.0).
+   */
+  private computePixelDiff(current: Buffer): number {
+    if (!this.previousBuffer) return 1.0; // First frame — always significant
+    if (current.length !== this.previousBuffer.length) return 1.0; // Size changed — significant
+
+    const step = 100;
+    let changed = 0;
+    let total = 0;
+
+    for (let i = 0; i < current.length; i += step) {
+      total++;
+      if (current[i] !== this.previousBuffer[i]) {
+        changed++;
+      }
+    }
+
+    return total > 0 ? changed / total : 1.0;
+  }
+
+  /**
+   * Save PNG to disk in date-organized directory.
+   * Returns the file path.
+   */
+  private saveCapture(imageBuffer: Buffer, timestamp: number): string {
+    const date = new Date(timestamp);
+    const dateDir = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
+    const fileName = `${String(date.getHours()).padStart(2, '0')}-${String(date.getMinutes()).padStart(2, '0')}-${String(date.getSeconds()).padStart(2, '0')}.png`;
+
+    const dir = path.join(this.captureDir, dateDir);
+    mkdirSync(dir, { recursive: true });
+
+    const filePath = path.join(dir, fileName);
+    writeFileSync(filePath, imageBuffer);
+
+    return filePath;
+  }
+
+  /**
+   * Generate a 200px-wide JPEG thumbnail from a full PNG capture.
+   * Uses sharp if available. Returns thumbnail path or null.
+   */
+  private async generateThumbnail(fullImagePath: string): Promise<string | null> {
+    if (!sharp) return null;
+
+    const thumbPath = fullImagePath.replace(/\.png$/, '-thumb.jpg');
+    try {
+      await sharp(fullImagePath).resize(200).jpeg({ quality: 60 }).toFile(thumbPath);
+      return thumbPath;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Clean up old captures based on tiered retention config.
+   * - 'full' tier: delete images + DB rows older than full_hours
+   * - 'key_moment' tier: delete images + DB rows older than key_moment_hours
+   * Disk files older than the max retention window are removed as orphans.
+   */
+  private cleanupRetention(): void {
+    try {
+      const now = Date.now();
+      const fullCutoff = now - (this.config.retention.full_hours * 60 * 60 * 1000);
+      const keyMomentCutoff = now - (this.config.retention.key_moment_hours * 60 * 60 * 1000);
+
+      // Delete DB rows by tier
+      let fullDeleted = 0;
+      let keyDeleted = 0;
+      try {
+        fullDeleted = deleteCapturesBefore(fullCutoff, 'full');
+        keyDeleted = deleteCapturesBefore(keyMomentCutoff, 'key_moment');
+      } catch { /* DB may not be initialized in tests */ }
+
+      // Clean up orphan files on disk
+      if (!existsSync(this.captureDir)) return;
+
+      const dateDirs = readdirSync(this.captureDir);
+      for (const dateDir of dateDirs) {
+        const dirPath = path.join(this.captureDir, dateDir);
+        try {
+          const stat = statSync(dirPath);
+          if (!stat.isDirectory()) continue;
+
+          const files = readdirSync(dirPath);
+          let remaining = files.length;
+
+          for (const file of files) {
+            const filePath = path.join(dirPath, file);
+            try {
+              const fileStat = statSync(filePath);
+              // Delete files older than max retention (key_moment_hours)
+              if (fileStat.mtimeMs < keyMomentCutoff) {
+                unlinkSync(filePath);
+                remaining--;
+              }
+            } catch { /* file already gone */ }
+          }
+
+          // Remove empty date directories
+          if (remaining === 0) {
+            try {
+              if (readdirSync(dirPath).length === 0) rmdirSync(dirPath);
+            } catch { /* ignore */ }
+          }
+        } catch { /* skip */ }
+      }
+
+      if (fullDeleted > 0 || keyDeleted > 0) {
+        console.log(`[CaptureEngine] Retention cleanup: ${fullDeleted} full, ${keyDeleted} key_moment captures deleted`);
+      }
+    } catch (err) {
+      console.error('[CaptureEngine] Retention cleanup error:', err instanceof Error ? err.message : err);
+    }
+  }
+}