nova-terminal-assistant 0.1.0

package/src/core/utils/VectorMemoryStore.ts

@@ -0,0 +1,440 @@
+// ============================================================================
+// VectorMemoryStore - Vector-based semantic memory storage
+// ============================================================================
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { createLogger } from './Logger.js';
+import { generateId } from './helpers.js';
+
+const logger = createLogger('VectorMemoryStore');
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * Memory entry with vector embedding
+ */
+export interface VectorMemoryEntry {
+  id: string;
+  content: string;
+  embedding: number[];
+  metadata: {
+    session: string;
+    timestamp: Date;
+    type: 'decision' | 'code' | 'error' | 'insight' | 'context' | 'preference';
+    importance: number;  // 0-1 score
+    tags: string[];
+    source?: string;     // File or URL source
+  };
+}
+
+/**
+ * Search result with similarity score
+ */
+export interface MemoryMatch {
+  entry: VectorMemoryEntry;
+  score: number;  // Cosine similarity 0-1
+}
+
+/**
+ * Memory store configuration
+ */
+export interface VectorMemoryConfig {
+  /** Storage directory for persistence */
+  storageDir: string;
+  /** Maximum entries to keep */
+  maxEntries?: number;
+  /** Minimum similarity score for search results */
+  minSimilarity?: number;
+  /** Embedding dimension (default 384 for small models) */
+  embeddingDimension?: number;
+}
+
+// ============================================================================
+// Embedding Functions
+// ============================================================================
+
+/**
+ * Simple embedding function using character n-grams
+ * This is a lightweight alternative to neural embeddings
+ */
+function simpleEmbedding(text: string, dimension: number = 384): number[] {
+  const embedding = new Array(dimension).fill(0);
+  
+  // Normalize text
+  const normalized = text.toLowerCase().replace(/[^a-z0-9\u4e00-\u9fff\s]/g, ' ');
+  const words = normalized.split(/\s+/).filter(w => w.length > 0);
+  
+  // Hash-based embedding
+  for (const word of words) {
+    // Simple hash function
+    let hash = 0;
+    for (let i = 0; i < word.length; i++) {
+      const char = word.charCodeAt(i);
+      hash = ((hash << 5) - hash) + char;
+      hash = hash & hash; // Convert to 32-bit integer
+    }
+    
+    // Map hash to embedding dimensions
+    const pos = Math.abs(hash) % dimension;
+    const sign = hash > 0 ? 1 : -1;
+    embedding[pos] += sign * (1 / words.length);
+    
+    // Add n-gram features
+    for (let i = 0; i < word.length - 1; i++) {
+      const bigram = word.slice(i, i + 2);
+      let bigramHash = 0;
+      for (let j = 0; j < bigram.length; j++) {
+        bigramHash = ((bigramHash << 5) - bigramHash) + bigram.charCodeAt(j);
+        bigramHash = bigramHash & bigramHash;
+      }
+      const bigramPos = Math.abs(bigramHash) % dimension;
+      embedding[bigramPos] += 0.1 / words.length;
+    }
+  }
+  
+  // Normalize embedding
+  const norm = Math.sqrt(embedding.reduce((sum, v) => sum + v * v, 0));
+  if (norm > 0) {
+    for (let i = 0; i < embedding.length; i++) {
+      embedding[i] /= norm;
+    }
+  }
+  
+  return embedding;
+}
+
+/**
+ * Calculate cosine similarity between two vectors
+ */
+function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) return 0;
+  
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+  
+  for (let i = 0; i < a.length; i++) {
+    const aVal = a[i] ?? 0;
+    const bVal = b[i] ?? 0;
+    dotProduct += aVal * bVal;
+    normA += aVal * aVal;
+    normB += bVal * bVal;
+  }
+  
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom > 0 ? dotProduct / denom : 0;
+}
+
+// ============================================================================
+// VectorMemoryStore
+// ============================================================================
+
+/**
+ * Vector-based memory store for semantic search
+ */
+export class VectorMemoryStore {
+  private entries: VectorMemoryEntry[] = [];
+  private config: Required<VectorMemoryConfig>;
+  private storageFile: string;
+  private isDirty = false;
+  
+  constructor(config: VectorMemoryConfig) {
+    this.config = {
+      maxEntries: config.maxEntries ?? 1000,
+      minSimilarity: config.minSimilarity ?? 0.5,
+      embeddingDimension: config.embeddingDimension ?? 384,
+      ...config,
+    };
+    
+    this.storageFile = path.join(this.config.storageDir, 'vector-memory.json');
+    this.load();
+    
+    // Auto-save on process exit
+    process.on('beforeExit', () => this.save());
+  }
+  
+  // -----------------------------------------------------------------------
+  // Core Operations
+  // -----------------------------------------------------------------------
+  
+  /**
+   * Add a memory entry
+   */
+  async addMemory(
+    content: string,
+    metadata: VectorMemoryEntry['metadata']
+  ): Promise<VectorMemoryEntry> {
+    // Generate embedding
+    const embedding = simpleEmbedding(content, this.config.embeddingDimension);
+    
+    const entry: VectorMemoryEntry = {
+      id: generateId(),
+      content,
+      embedding,
+      metadata: {
+        ...metadata,
+        timestamp: metadata.timestamp ?? new Date(),
+      },
+    };
+    
+    this.entries.push(entry);
+    this.isDirty = true;
+    
+    // Prune if over limit
+    if (this.entries.length > this.config.maxEntries) {
+      this.prune();
+    }
+    
+    logger.debug(`Added memory entry: ${entry.id}`, { type: metadata.type });
+    return entry;
+  }
+  
+  /**
+   * Search for similar memories
+   */
+  async searchSimilar(
+    query: string,
+    options: {
+      topK?: number;
+      minSimilarity?: number;
+      type?: VectorMemoryEntry['metadata']['type'];
+      tags?: string[];
+    } = {}
+  ): Promise<MemoryMatch[]> {
+    const { topK = 5, minSimilarity, type, tags } = options;
+    const threshold = minSimilarity ?? this.config.minSimilarity;
+    
+    // Generate query embedding
+    const queryEmbedding = simpleEmbedding(query, this.config.embeddingDimension);
+    
+    // Calculate similarities
+    const matches: MemoryMatch[] = [];
+    
+    for (const entry of this.entries) {
+      // Filter by type
+      if (type && entry.metadata.type !== type) continue;
+      
+      // Filter by tags
+      if (tags && tags.length > 0) {
+        const hasTag = tags.some(t => entry.metadata.tags.includes(t));
+        if (!hasTag) continue;
+      }
+      
+      const score = cosineSimilarity(queryEmbedding, entry.embedding);
+      
+      if (score >= threshold) {
+        matches.push({ entry, score });
+      }
+    }
+    
+    // Sort by score and return top K
+    matches.sort((a, b) => b.score - a.score);
+    return matches.slice(0, topK);
+  }
+  
+  /**
+   * Get entry by ID
+   */
+  getEntry(id: string): VectorMemoryEntry | undefined {
+    return this.entries.find(e => e.id === id);
+  }
+  
+  /**
+   * Delete entry by ID
+   */
+  deleteEntry(id: string): boolean {
+    const index = this.entries.findIndex(e => e.id === id);
+    if (index >= 0) {
+      this.entries.splice(index, 1);
+      this.isDirty = true;
+      return true;
+    }
+    return false;
+  }
+  
+  /**
+   * Get all entries of a specific type
+   */
+  getEntriesByType(type: VectorMemoryEntry['metadata']['type']): VectorMemoryEntry[] {
+    return this.entries.filter(e => e.metadata.type === type);
+  }
+  
+  /**
+   * Get entries by tags
+   */
+  getEntriesByTags(tags: string[]): VectorMemoryEntry[] {
+    return this.entries.filter(e => 
+      tags.some(t => e.metadata.tags.includes(t))
+    );
+  }
+  
+  /**
+   * Get entries for a session
+   */
+  getEntriesBySession(sessionId: string): VectorMemoryEntry[] {
+    return this.entries.filter(e => e.metadata.session === sessionId);
+  }
+  
+  // -----------------------------------------------------------------------
+  // Persistence
+  // -----------------------------------------------------------------------
+  
+  /**
+   * Save to disk
+   */
+  save(): void {
+    if (!this.isDirty) return;
+    
+    try {
+      // Ensure directory exists
+      const dir = path.dirname(this.storageFile);
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+      }
+      
+      // Serialize entries
+      const data = {
+        version: 1,
+        entries: this.entries.map(e => ({
+          ...e,
+          metadata: {
+            ...e.metadata,
+            timestamp: e.metadata.timestamp.toISOString(),
+          },
+        })),
+      };
+      
+      fs.writeFileSync(this.storageFile, JSON.stringify(data, null, 2), 'utf-8');
+      this.isDirty = false;
+      
+      logger.debug(`Saved ${this.entries.length} memory entries`);
+    } catch (error) {
+      logger.error('Failed to save memory store', { error });
+    }
+  }
+  
+  /**
+   * Load from disk
+   */
+  load(): void {
+    try {
+      if (!fs.existsSync(this.storageFile)) {
+        logger.debug('No existing memory store found, starting fresh');
+        return;
+      }
+      
+      const data = JSON.parse(fs.readFileSync(this.storageFile, 'utf-8'));
+      
+      if (data.version === 1 && Array.isArray(data.entries)) {
+        this.entries = data.entries.map((e: any) => ({
+          ...e,
+          metadata: {
+            ...e.metadata,
+            timestamp: new Date(e.metadata.timestamp),
+          },
+        }));
+        
+        logger.debug(`Loaded ${this.entries.length} memory entries`);
+      }
+    } catch (error) {
+      logger.error('Failed to load memory store', { error });
+      this.entries = [];
+    }
+  }
+  
+  // -----------------------------------------------------------------------
+  // Maintenance
+  // -----------------------------------------------------------------------
+  
+  /**
+   * Prune entries based on importance and age
+   */
+  private prune(): void {
+    if (this.entries.length <= this.config.maxEntries) return;
+    
+    // Calculate age scores (newer is better)
+    const now = Date.now();
+    const maxAge = 30 * 24 * 60 * 60 * 1000; // 30 days
+    
+    const scoredEntries = this.entries.map(entry => {
+      const age = now - entry.metadata.timestamp.getTime();
+      const ageScore = Math.max(0, 1 - age / maxAge);
+      
+      // Combined score: importance * 0.7 + ageScore * 0.3
+      const score = entry.metadata.importance * 0.7 + ageScore * 0.3;
+      
+      return { entry, score };
+    });
+    
+    // Sort by score and keep top entries
+    scoredEntries.sort((a, b) => b.score - a.score);
+    this.entries = scoredEntries
+      .slice(0, this.config.maxEntries)
+      .map(s => s.entry);
+    
+    this.isDirty = true;
+    logger.debug(`Pruned memory to ${this.entries.length} entries`);
+  }
+  
+  /**
+   * Get statistics
+   */
+  getStats(): {
+    totalEntries: number;
+    byType: Record<string, number>;
+    oldestEntry: Date | null;
+    newestEntry: Date | null;
+    avgImportance: number;
+  } {
+    const byType: Record<string, number> = {};
+    let oldest: Date | null = null;
+    let newest: Date | null = null;
+    let totalImportance = 0;
+    
+    for (const entry of this.entries) {
+      byType[entry.metadata.type] = (byType[entry.metadata.type] ?? 0) + 1;
+      
+      if (!oldest || entry.metadata.timestamp < oldest) {
+        oldest = entry.metadata.timestamp;
+      }
+      if (!newest || entry.metadata.timestamp > newest) {
+        newest = entry.metadata.timestamp;
+      }
+      
+      totalImportance += entry.metadata.importance;
+    }
+    
+    return {
+      totalEntries: this.entries.length,
+      byType,
+      oldestEntry: oldest,
+      newestEntry: newest,
+      avgImportance: this.entries.length > 0 ? totalImportance / this.entries.length : 0,
+    };
+  }
+  
+  /**
+   * Clear all entries
+   */
+  clear(): void {
+    this.entries = [];
+    this.isDirty = true;
+  }
+}
+
+// ============================================================================
+// Factory function
+// ============================================================================
+
+/**
+ * Create a vector memory store with default configuration
+ */
+export function createVectorMemoryStore(
+  storageDir?: string
+): VectorMemoryStore {
+  const defaultDir = storageDir ?? path.join(process.env.HOME ?? '~', '.nova', 'memory');
+  return new VectorMemoryStore({ storageDir: defaultDir });
+}

package/src/core/utils/helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["helpers.ts"],"names":[],"mappings":"AAMA,2BAA2B;AAC3B,wBAAgB,UAAU,CAAC,MAAM,SAAK,GAAG,MAAM,CAG9C;AAED,qCAAqC;AACrC,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/C;AAED,gDAAgD;AAChD,wBAAsB,KAAK,CAAC,CAAC,EAC3B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,OAAO,GAAE;IACP,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC;CACpC,GACL,OAAO,CAAC,CAAC,CAAC,CAiBZ;AAED,wCAAwC;AACxC,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAQ,GAAG,MAAM,CAGhF;AAED,4CAA4C;AAC5C,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CASjD;AAED,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAKjD;AAED,2BAA2B;AAC3B,wBAAgB,SAAS,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAEtC;AAED,0BAA0B;AAC1B,wBAAgB,QAAQ,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,CAMvH;AAED,uCAAuC;AACvC,wBAAgB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAErE"}

package/src/core/utils/helpers.ts

@@ -0,0 +1,89 @@
+// ============================================================================
+// Utility Helpers
+// ============================================================================
+
+import { randomBytes } from 'node:crypto';
+
+/** Generate a random ID */
+export function generateId(prefix = ''): string {
+  const bytes = randomBytes(8).toString('hex');
+  return prefix ? `${prefix}_${bytes}` : bytes;
+}
+
+/** Sleep for a specified duration */
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Retry a function with exponential backoff */
+export async function retry<T>(
+  fn: () => Promise<T>,
+  options: {
+    maxAttempts?: number;
+    baseDelay?: number;
+    maxDelay?: number;
+    shouldRetry?: (error: Error) => boolean;
+  } = {}
+): Promise<T> {
+  const { maxAttempts = 3, baseDelay = 1000, maxDelay = 30000, shouldRetry } = options;
+  let lastError: Error | undefined;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastError = err as Error;
+      if (attempt === maxAttempts) break;
+      if (shouldRetry && !shouldRetry(lastError)) break;
+      const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
+      await sleep(delay);
+    }
+  }
+
+  throw lastError;
+}
+
+/** Truncate text to a maximum length */
+export function truncate(text: string, maxLength: number, suffix = '...'): string {
+  if (text.length <= maxLength) return text;
+  return text.slice(0, maxLength - suffix.length) + suffix;
+}
+
+/** Format bytes as human-readable string */
+export function formatBytes(bytes: number): string {
+  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+  let size = bytes;
+  let unitIndex = 0;
+  while (size >= 1024 && unitIndex < units.length - 1) {
+    size /= 1024;
+    unitIndex++;
+  }
+  return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
+}
+
+/** Format duration as human-readable string */
+export function formatDuration(ms: number): string {
+  if (ms < 1000) return `${ms}ms`;
+  if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
+  if (ms < 3600000) return `${Math.floor(ms / 60000)}m ${Math.floor((ms % 60000) / 1000)}s`;
+  return `${Math.floor(ms / 3600000)}h ${Math.floor((ms % 3600000) / 60000)}m`;
+}
+
+/** Deep clone an object */
+export function deepClone<T>(obj: T): T {
+  return JSON.parse(JSON.stringify(obj));
+}
+
+/** Debounce a function */
+export function debounce<T extends (...args: unknown[]) => void>(fn: T, delay: number): (...args: Parameters<T>) => void {
+  let timer: ReturnType<typeof setTimeout>;
+  return (...args: Parameters<T>) => {
+    clearTimeout(timer);
+    timer = setTimeout(() => fn(...args), delay);
+  };
+}
+
+/** Ensure a value is within a range */
+export function clamp(value: number, min: number, max: number): number {
+  return Math.max(min, Math.min(max, value));
+}

package/src/core/utils/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACnD,YAAY,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC"}

package/src/core/utils/index.ts

@@ -0,0 +1,19 @@
+export { Logger, createLogger } from './Logger.js';
+export type { LogEntry } from './Logger.js';
+export { CheckpointManager } from './CheckpointManager.js';
+export type { Checkpoint, CheckpointFile, CheckpointStats } from './CheckpointManager.js';
+export { generateId, sleep, retry, truncate, formatBytes, formatDuration, deepClone, debounce, clamp } from './helpers.js';
+export { TokenCounter, tokenCounter, countTokens, countMessagesTokens } from './TokenCounter.js';
+export { 
+  withRetry, 
+  createRetryWrapper, 
+  RateLimiter, 
+  withRateLimit, 
+  ConcurrencyLimiter,
+  createResilientFunction,
+  createDefaultApiRateLimiter,
+  createDefaultConcurrencyLimiter,
+} from './RetryManager.js';
+export type { RetryConfig, RateLimiterConfig } from './RetryManager.js';
+export { VectorMemoryStore, createVectorMemoryStore } from './VectorMemoryStore.js';
+export type { VectorMemoryEntry, MemoryMatch, VectorMemoryConfig } from './VectorMemoryStore.js';