@hashgraphonline/conversational-agent 0.1.204 → 0.1.206

package/src/memory/MemoryWindow.ts

@@ -0,0 +1,247 @@
+import type { BaseMessage } from '@langchain/core/messages';
+import { TokenCounter } from './TokenCounter';
+
+/**
+ * Result of adding a message to the memory window
+ */
+export interface AddMessageResult {
+  /** Whether the message was successfully added */
+  added: boolean;
+  /** Messages that were pruned to make room */
+  prunedMessages: BaseMessage[];
+  /** Current token count after operation */
+  currentTokenCount: number;
+  /** Remaining token capacity */
+  remainingCapacity: number;
+}
+
+/**
+ * Memory window that manages conversation history with token-based size limits
+ * Automatically prunes old messages to stay within token limits while preserving conversational context
+ */
+export class MemoryWindow {
+  private messages: BaseMessage[] = [];
+  private maxTokens: number;
+  private reserveTokens: number;
+  private tokenCounter: TokenCounter;
+  private systemPrompt: string = '';
+  private systemPromptTokens: number = 0;
+
+  // Default token limits for different model contexts
+  public static readonly DEFAULT_MAX_TOKENS = 8000; // Conservative limit for most models
+  public static readonly DEFAULT_RESERVE_TOKENS = 1000; // Reserve for response generation
+  public static readonly PRUNING_BATCH_SIZE = 2; // Remove messages in pairs to maintain conversation flow
+
+  constructor(
+    maxTokens: number = MemoryWindow.DEFAULT_MAX_TOKENS,
+    reserveTokens: number = MemoryWindow.DEFAULT_RESERVE_TOKENS,
+    tokenCounter?: TokenCounter
+  ) {
+    if (reserveTokens >= maxTokens) {
+      throw new Error('Reserve tokens must be less than max tokens');
+    }
+
+    this.maxTokens = maxTokens;
+    this.reserveTokens = reserveTokens;
+    this.tokenCounter = tokenCounter || new TokenCounter();
+  }
+
+  /**
+   * Add a message to the memory window, pruning old messages if necessary
+   * @param message - The message to add
+   * @returns Result of the add operation including any pruned messages
+   */
+  addMessage(message: BaseMessage): AddMessageResult {
+    // Calculate tokens for the new message
+    const messageTokens = this.tokenCounter.countMessageTokens(message);
+    
+    // Add the message first
+    this.messages.push(message);
+    
+    // Check if we need to prune
+    const currentTokens = this.getCurrentTokenCount();
+    const availableTokens = this.maxTokens - this.reserveTokens;
+    
+    let prunedMessages: BaseMessage[] = [];
+    
+    if (currentTokens > availableTokens) {
+      // Need to prune - remove the new message temporarily
+      this.messages.pop();
+      
+      // Prune old messages to make room
+      prunedMessages = this.pruneToFit();
+      
+      // Add the new message back
+      this.messages.push(message);
+    }
+
+    return {
+      added: true,
+      prunedMessages,
+      currentTokenCount: this.getCurrentTokenCount(),
+      remainingCapacity: this.getRemainingTokenCapacity()
+    };
+  }
+
+  /**
+   * Prune old messages to fit within token limits
+   * Removes messages in pairs to maintain conversational flow
+   * @returns Array of pruned messages
+   */
+  pruneToFit(): BaseMessage[] {
+    const prunedMessages: BaseMessage[] = [];
+    const targetTokens = this.maxTokens - this.reserveTokens;
+    
+    while (this.getCurrentTokenCount() > targetTokens && this.messages.length > 0) {
+      // Remove messages in batches to maintain conversation flow
+      const batchSize = Math.min(MemoryWindow.PRUNING_BATCH_SIZE, this.messages.length);
+      
+      for (let i = 0; i < batchSize; i++) {
+        const prunedMessage = this.messages.shift();
+        if (prunedMessage) {
+          prunedMessages.push(prunedMessage);
+        }
+      }
+      
+      // Safety check to prevent infinite loop
+      if (prunedMessages.length > 1000) {
+        console.warn('MemoryWindow: Excessive pruning detected, stopping to prevent infinite loop');
+        break;
+      }
+    }
+
+    return prunedMessages;
+  }
+
+  /**
+   * Get current token count including system prompt and messages
+   * @returns Current token count
+   */
+  getCurrentTokenCount(): number {
+    const messageTokens = this.tokenCounter.countMessagesTokens(this.messages);
+    return this.systemPromptTokens + messageTokens;
+  }
+
+  /**
+   * Get remaining token capacity before hitting the reserve limit
+   * @returns Remaining tokens that can be used
+   */
+  getRemainingTokenCapacity(): number {
+    return Math.max(0, this.maxTokens - this.getCurrentTokenCount());
+  }
+
+  /**
+   * Check if a message can be added without exceeding limits
+   * @param message - The message to check
+   * @returns True if message can be added within reserve limits
+   */
+  canAddMessage(message: BaseMessage): boolean {
+    const messageTokens = this.tokenCounter.countMessageTokens(message);
+    const currentTokens = this.getCurrentTokenCount();
+    const wouldExceedReserve = (currentTokens + messageTokens) > (this.maxTokens - this.reserveTokens);
+    
+    // Always allow adding if we can prune to make room
+    // Only return false if the message itself is larger than our total capacity
+    if (messageTokens > this.maxTokens) {
+      return false;
+    }
+    
+    return !wouldExceedReserve || this.messages.length > 0;
+  }
+
+  /**
+   * Get all messages in the memory window
+   * @returns Copy of current messages array
+   */
+  getMessages(): BaseMessage[] {
+    return [...this.messages];
+  }
+
+  /**
+   * Clear all messages from the memory window
+   */
+  clear(): void {
+    this.messages = [];
+  }
+
+  /**
+   * Set the system prompt and update token calculations
+   * @param systemPrompt - The system prompt text
+   */
+  setSystemPrompt(systemPrompt: string): void {
+    this.systemPrompt = systemPrompt;
+    this.systemPromptTokens = this.tokenCounter.estimateSystemPromptTokens(systemPrompt);
+  }
+
+  /**
+   * Get the current system prompt
+   * @returns Current system prompt
+   */
+  getSystemPrompt(): string {
+    return this.systemPrompt;
+  }
+
+  /**
+   * Get current configuration
+   * @returns Memory window configuration
+   */
+  getConfig() {
+    return {
+      maxTokens: this.maxTokens,
+      reserveTokens: this.reserveTokens,
+      currentTokens: this.getCurrentTokenCount(),
+      messageCount: this.messages.length,
+      systemPromptTokens: this.systemPromptTokens
+    };
+  }
+
+  /**
+   * Update token limits
+   * @param maxTokens - New maximum token limit
+   * @param reserveTokens - New reserve token amount
+   */
+  updateLimits(maxTokens: number, reserveTokens?: number): void {
+    if (reserveTokens !== undefined && reserveTokens >= maxTokens) {
+      throw new Error('Reserve tokens must be less than max tokens');
+    }
+
+    this.maxTokens = maxTokens;
+    if (reserveTokens !== undefined) {
+      this.reserveTokens = reserveTokens;
+    }
+
+    // Prune if necessary after updating limits
+    if (this.getCurrentTokenCount() > (this.maxTokens - this.reserveTokens)) {
+      this.pruneToFit();
+    }
+  }
+
+  /**
+   * Get statistics about the memory window
+   * @returns Memory usage statistics
+   */
+  getStats() {
+    const currentTokens = this.getCurrentTokenCount();
+    const capacity = this.maxTokens;
+    const usagePercentage = (currentTokens / capacity) * 100;
+    
+    return {
+      totalMessages: this.messages.length,
+      currentTokens,
+      maxTokens: capacity,
+      reserveTokens: this.reserveTokens,
+      systemPromptTokens: this.systemPromptTokens,
+      usagePercentage: Math.round(usagePercentage * 100) / 100,
+      remainingCapacity: this.getRemainingTokenCapacity(),
+      canAcceptMore: this.getRemainingTokenCapacity() > this.reserveTokens
+    };
+  }
+
+  /**
+   * Clean up resources
+   */
+  dispose(): void {
+    this.clear();
+    this.tokenCounter.dispose();
+  }
+}

package/src/memory/ReferenceIdGenerator.ts

@@ -0,0 +1,84 @@
+import { createHash } from 'crypto';
+import { ReferenceId } from '../types/content-reference';
+
+/**
+ * Content-based reference ID generator using SHA-256 (HCS-1 style)
+ * 
+ * Generates deterministic reference IDs based on content hashing.
+ * Same content always produces the same reference ID.
+ */
+export class ReferenceIdGenerator {
+  /**
+   * Generate a content-based reference ID using SHA-256 hashing
+   * 
+   * @param content The content to generate a reference ID for
+   * @returns Deterministic reference ID based on content hash
+   */
+  static generateId(content: Buffer): ReferenceId {
+    const hash = createHash('sha256');
+    hash.update(content);
+    return hash.digest('base64url');
+  }
+  
+  /**
+   * Validate that a string is a properly formatted reference ID
+   * 
+   * @param id The ID to validate
+   * @returns true if the ID is valid format
+   */
+  static isValidReferenceId(id: string): id is ReferenceId {
+    if (!id || typeof id !== 'string') {
+      return false;
+    }
+    
+    // Check length (base64url encoding of SHA-256 hash = 43 chars)
+    if (id.length !== 43) {
+      return false;
+    }
+    
+    // Check character set (base64url: A-Z, a-z, 0-9, -, _)
+    return /^[A-Za-z0-9_-]+$/.test(id);
+  }
+  
+  /**
+   * Extract reference ID from ref:// format
+   * 
+   * @param input Input string that may contain a reference ID
+   * @returns Extracted reference ID or null if not found
+   */
+  static extractReferenceId(input: string): ReferenceId | null {
+    if (!input || typeof input !== 'string') {
+      return null;
+    }
+    
+    // Check for ref:// format
+    const refFormatMatch = input.match(/^ref:\/\/([A-Za-z0-9_-]{43})$/);
+    if (refFormatMatch) {
+      return refFormatMatch[1] as ReferenceId;
+    }
+    
+    // Check if input is directly a valid reference ID
+    return this.isValidReferenceId(input) ? input : null;
+  }
+  
+  /**
+   * Format a reference ID in the standard ref:// format
+   * 
+   * @param referenceId The reference ID to format
+   * @returns Formatted reference string
+   */
+  static formatReference(referenceId: ReferenceId): string {
+    return `ref://${referenceId}`;
+  }
+  
+  /**
+   * Generate a test reference ID (for testing purposes only)
+   * 
+   * @param testSeed A test seed to generate a fake but valid ID format
+   * @returns A valid format reference ID for testing
+   */
+  static generateTestId(testSeed: string): ReferenceId {
+    const content = Buffer.from(`test-${testSeed}-${Date.now()}`);
+    return this.generateId(content);
+  }
+}

package/src/memory/SmartMemoryManager.ts

@@ -0,0 +1,323 @@
+import type { BaseMessage } from '@langchain/core/messages';
+import { MemoryWindow } from './MemoryWindow';
+import { ContentStorage } from './ContentStorage';
+import { TokenCounter } from './TokenCounter';
+
+/**
+ * Configuration for SmartMemoryManager
+ */
+export interface SmartMemoryConfig {
+  /** Maximum tokens for active memory window */
+  maxTokens?: number;
+  /** Reserve tokens for response generation */
+  reserveTokens?: number;
+  /** Model name for token counting */
+  modelName?: string;
+  /** Maximum messages to store in content storage */
+  storageLimit?: number;
+}
+
+/**
+ * Search options for history search
+ */
+export interface SearchOptions {
+  /** Whether to perform case-sensitive search */
+  caseSensitive?: boolean;
+  /** Maximum number of results to return */
+  limit?: number;
+  /** Whether to use regex pattern matching */
+  useRegex?: boolean;
+}
+
+/**
+ * Memory statistics for active memory window
+ */
+export interface MemoryStats {
+  /** Total active messages in memory window */
+  totalActiveMessages: number;
+  /** Current token count including system prompt */
+  currentTokenCount: number;
+  /** Maximum token capacity */
+  maxTokens: number;
+  /** Remaining token capacity */
+  remainingCapacity: number;
+  /** System prompt token count */
+  systemPromptTokens: number;
+  /** Memory usage percentage */
+  usagePercentage: number;
+}
+
+/**
+ * Smart memory manager that combines active memory window with long-term storage
+ * Provides context-aware memory management with automatic pruning and searchable history
+ */
+export class SmartMemoryManager {
+  private memoryWindow: MemoryWindow;
+  private contentStorage: ContentStorage;
+  private tokenCounter: TokenCounter;
+  private config: Required<SmartMemoryConfig>;
+
+  // Default configuration values
+  private static readonly DEFAULT_CONFIG: Required<SmartMemoryConfig> = {
+    maxTokens: 8000,
+    reserveTokens: 1000,
+    modelName: 'gpt-4o',
+    storageLimit: 1000
+  };
+
+  constructor(config: SmartMemoryConfig = {}) {
+    this.config = { ...SmartMemoryManager.DEFAULT_CONFIG, ...config };
+    
+    // Initialize components
+    this.tokenCounter = new TokenCounter(this.config.modelName as any);
+    this.contentStorage = new ContentStorage(this.config.storageLimit);
+    this.memoryWindow = new MemoryWindow(
+      this.config.maxTokens,
+      this.config.reserveTokens,
+      this.tokenCounter
+    );
+  }
+
+  /**
+   * Add a message to the active memory window
+   * Automatically handles pruning and storage of displaced messages
+   * @param message - Message to add
+   */
+  addMessage(message: BaseMessage): void {
+    const result = this.memoryWindow.addMessage(message);
+    
+    // Store any pruned messages in content storage
+    if (result.prunedMessages.length > 0) {
+      this.contentStorage.storeMessages(result.prunedMessages);
+    }
+  }
+
+  /**
+   * Get all active messages from the memory window
+   * @returns Array of active messages in chronological order
+   */
+  getMessages(): BaseMessage[] {
+    return this.memoryWindow.getMessages();
+  }
+
+  /**
+   * Clear active memory window
+   * @param clearStorage - Whether to also clear the content storage (default: false)
+   */
+  clear(clearStorage: boolean = false): void {
+    this.memoryWindow.clear();
+    
+    if (clearStorage) {
+      this.contentStorage.clear();
+    }
+  }
+
+  /**
+   * Set the system prompt for the memory window
+   * @param systemPrompt - System prompt text
+   */
+  setSystemPrompt(systemPrompt: string): void {
+    this.memoryWindow.setSystemPrompt(systemPrompt);
+  }
+
+  /**
+   * Get the current system prompt
+   * @returns Current system prompt text
+   */
+  getSystemPrompt(): string {
+    return this.memoryWindow.getSystemPrompt();
+  }
+
+  /**
+   * Search through stored message history
+   * @param query - Search term or pattern
+   * @param options - Search configuration
+   * @returns Array of matching messages from history
+   */
+  searchHistory(query: string, options: SearchOptions = {}): BaseMessage[] {
+    return this.contentStorage.searchMessages(query, options);
+  }
+
+  /**
+   * Get recent messages from storage history
+   * @param count - Number of recent messages to retrieve
+   * @returns Array of recent messages from storage
+   */
+  getRecentHistory(count: number): BaseMessage[] {
+    return this.contentStorage.getRecentMessages(count);
+  }
+
+  /**
+   * Check if a message can be added without exceeding limits
+   * @param message - Message to test
+   * @returns True if message can be added
+   */
+  canAddMessage(message: BaseMessage): boolean {
+    return this.memoryWindow.canAddMessage(message);
+  }
+
+  /**
+   * Get statistics about the active memory window
+   * @returns Memory usage statistics
+   */
+  getMemoryStats(): MemoryStats {
+    const windowStats = this.memoryWindow.getStats();
+    
+    return {
+      totalActiveMessages: windowStats.totalMessages,
+      currentTokenCount: windowStats.currentTokens,
+      maxTokens: windowStats.maxTokens,
+      remainingCapacity: windowStats.remainingCapacity,
+      systemPromptTokens: windowStats.systemPromptTokens,
+      usagePercentage: windowStats.usagePercentage
+    };
+  }
+
+  /**
+   * Get statistics about the content storage
+   * @returns Storage usage statistics
+   */
+  getStorageStats() {
+    return this.contentStorage.getStorageStats();
+  }
+
+  /**
+   * Get combined statistics for both active memory and storage
+   * @returns Combined memory and storage statistics
+   */
+  getOverallStats() {
+    const memoryStats = this.getMemoryStats();
+    const storageStats = this.getStorageStats();
+    
+    return {
+      activeMemory: memoryStats,
+      storage: storageStats,
+      totalMessagesManaged: memoryStats.totalActiveMessages + storageStats.totalMessages,
+      activeMemoryUtilization: memoryStats.usagePercentage,
+      storageUtilization: storageStats.usagePercentage
+    };
+  }
+
+  /**
+   * Update the configuration and apply changes
+   * @param newConfig - New configuration options
+   */
+  updateConfig(newConfig: Partial<SmartMemoryConfig>): void {
+    this.config = { ...this.config, ...newConfig };
+    
+    // Update components with new configuration
+    if (newConfig.maxTokens !== undefined || newConfig.reserveTokens !== undefined) {
+      this.memoryWindow.updateLimits(
+        this.config.maxTokens,
+        this.config.reserveTokens
+      );
+    }
+    
+    if (newConfig.storageLimit !== undefined) {
+      this.contentStorage.updateStorageLimit(this.config.storageLimit);
+    }
+    
+    // Note: Model name changes would require recreating the token counter
+    // This is not implemented to avoid disrupting ongoing operations
+  }
+
+  /**
+   * Get current configuration
+   * @returns Current configuration settings
+   */
+  getConfig(): Required<SmartMemoryConfig> {
+    return { ...this.config };
+  }
+
+  /**
+   * Get messages from storage within a time range
+   * @param startTime - Start of time range
+   * @param endTime - End of time range
+   * @returns Messages within the specified time range
+   */
+  getHistoryFromTimeRange(startTime: Date, endTime: Date): BaseMessage[] {
+    return this.contentStorage.getMessagesFromTimeRange(startTime, endTime);
+  }
+
+  /**
+   * Get messages from storage by message type
+   * @param messageType - Type of messages to retrieve ('human', 'ai', 'system', etc.)
+   * @param limit - Maximum number of messages to return
+   * @returns Messages of the specified type
+   */
+  getHistoryByType(messageType: string, limit?: number): BaseMessage[] {
+    return this.contentStorage.getMessagesByType(messageType, limit);
+  }
+
+  /**
+   * Get recent messages from storage within the last N minutes
+   * @param minutes - Number of minutes to look back
+   * @returns Messages from the last N minutes
+   */
+  getRecentHistoryByTime(minutes: number): BaseMessage[] {
+    return this.contentStorage.getRecentMessagesByTime(minutes);
+  }
+
+  /**
+   * Export the current state for persistence or analysis
+   * @returns Serializable representation of memory state
+   */
+  exportState() {
+    return {
+      config: this.config,
+      activeMessages: this.memoryWindow.getMessages().map(msg => ({
+        content: msg.content,
+        type: msg._getType()
+      })),
+      systemPrompt: this.memoryWindow.getSystemPrompt(),
+      memoryStats: this.getMemoryStats(),
+      storageStats: this.getStorageStats(),
+      storedMessages: this.contentStorage.exportMessages()
+    };
+  }
+
+  /**
+   * Get a summary of conversation context for external use
+   * Useful for providing context to other systems or for logging
+   * @param includeStoredContext - Whether to include recent stored messages
+   * @returns Context summary object
+   */
+  getContextSummary(includeStoredContext: boolean = false) {
+    const activeMessages = this.getMessages();
+    const summary = {
+      activeMessageCount: activeMessages.length,
+      systemPrompt: this.getSystemPrompt(),
+      recentMessages: activeMessages.slice(-5), // Last 5 active messages
+      memoryUtilization: this.getMemoryStats().usagePercentage,
+      hasStoredHistory: this.getStorageStats().totalMessages > 0
+    };
+
+    if (includeStoredContext) {
+      return {
+        ...summary,
+        recentStoredMessages: this.getRecentHistory(10), // Last 10 stored messages
+        storageStats: this.getStorageStats()
+      };
+    }
+
+    return summary;
+  }
+
+  /**
+   * Perform maintenance operations
+   * Optimizes storage and cleans up resources
+   */
+  performMaintenance(): void {
+    // No specific maintenance needed currently
+    // This method is reserved for future optimizations
+  }
+
+  /**
+   * Clean up resources and dispose of components
+   */
+  dispose(): void {
+    this.memoryWindow.dispose();
+    this.contentStorage.dispose();
+    this.tokenCounter.dispose();
+  }
+}