htm 0.0.2 → 0.0.11

data/lib/htm.rb

@@ -3,15 +3,21 @@
 require_relative "htm/version"
 require_relative "htm/errors"
 require_relative "htm/configuration"
+require_relative "htm/circuit_breaker"
 require_relative "htm/active_record_config"
 require_relative "htm/database"
 require_relative "htm/long_term_memory"
 require_relative "htm/working_memory"
 require_relative "htm/embedding_service"
 require_relative "htm/tag_service"
+require_relative "htm/timeframe_extractor"
+require_relative "htm/timeframe"
 require_relative "htm/job_adapter"
 require_relative "htm/jobs/generate_embedding_job"
 require_relative "htm/jobs/generate_tags_job"
+require_relative "htm/loaders/paragraph_chunker"
+require_relative "htm/loaders/markdown_loader"
+require_relative "htm/observability"
 
 require "pg"
 require "securerandom"
@@ -24,14 +30,14 @@ require_relative "htm/railtie" if defined?(Rails::Railtie)
 #
 # HTM implements a two-tier memory system:
 # - Working Memory: Token-limited, active context for immediate LLM use
-# - Long-term Memory: Durable PostgreSQL/TimescaleDB storage for permanent knowledge
+# - Long-term Memory: Durable PostgreSQL storage with pgvector for permanent knowledge
 #
 # Key Features:
-# - Never forgets unless explicitly told
-# - RAG-based retrieval (temporal + semantic search)
+# - Never forgets unless explicitly told (soft delete by default)
+# - RAG-based retrieval (temporal + semantic search via pgvector)
 # - Multi-robot "hive mind" - all robots share global memory
-# - Relationship graphs for knowledge connections
-# - Time-series optimized with TimescaleDB
+# - Hierarchical tagging system for knowledge organization
+# - Async background processing for embeddings and tags
 #
 # @example Basic usage
 #   htm = HTM.new(robot_name: "Code Helper")
@@ -98,13 +104,10 @@ class HTM
   # Stores content in long-term memory and adds it to working memory.
   # Embeddings and hierarchical tags are automatically extracted by LLM in the background.
   #
-  # If content is empty, returns the ID of the most recent node without creating a duplicate.
-  # Nil values for content or source are converted to empty strings.
-  #
-  # @param content [String, nil] The information to remember
-  # @param source [String, nil] Where this content came from (defaults to empty string if not provided)
+  # @param content [String] The information to remember (required, cannot be nil or empty)
   # @param tags [Array<String>] Manual tags to assign (optional, in addition to auto-extracted tags)
   # @return [Integer] Database ID of the memory node
+  # @raise [ValidationError] If content is nil, empty, or exceeds maximum size
   #
   # @example Remember with source
   #   node_id = htm.remember("PostgreSQL is great for HTM")
@@ -112,16 +115,31 @@ class HTM
   # @example Remember with manual tags
   #   node_id = htm.remember("Time-series data", tags: ["database:timescaledb"])
   #
-  def remember(content, tags: [])
-    # Convert nil to empty string
-    content = content.to_s
+  # @example Remember with metadata
+  #   node_id = htm.remember("User prefers dark mode", metadata: { source: "user", confidence: 0.95 })
+  #
+  def remember(content, tags: [], metadata: {})
+    # Validate inputs
+    raise ValidationError, "Content cannot be nil" if content.nil?
+
+    content_str = content.to_s.strip
+    raise ValidationError, "Content cannot be empty" if content_str.empty?
 
-    # If content is empty, return the last node ID without creating a new entry
-    if content.empty?
-      last_node = HTM::Models::Node.order(created_at: :desc).first
-      return last_node&.id || 0
+    if content_str.bytesize > MAX_VALUE_LENGTH
+      raise ValidationError, "Content exceeds maximum size (#{MAX_VALUE_LENGTH} bytes)"
+    end
+
+    validate_array!(tags, "tags")
+    tags.each do |tag|
+      unless tag.is_a?(String) && tag.match?(/\A[a-z0-9\-]+(:[a-z0-9\-]+)*\z/)
+        raise ValidationError, "Invalid tag format: #{tag.inspect}. Tags must be lowercase alphanumeric with hyphens, separated by colons."
+      end
     end
 
+    validate_metadata!(metadata)
+
+    content = content_str
+
     # Calculate token count using configured counter
     token_count = HTM.count_tokens(content)
 
@@ -131,7 +149,8 @@ class HTM
       content: content,
       token_count: token_count,
       robot_id: @robot_id,
-      embedding: nil  # Will be generated in background
+      embedding: nil,  # Will be generated in background
+      metadata: metadata
     )
 
     node_id = result[:node_id]
@@ -155,9 +174,17 @@ class HTM
       end
     end
 
-    # Add to working memory (access_count starts at 0)
+    # Add to working memory (evict if needed, access_count starts at 0)
+    unless @working_memory.has_space?(token_count)
+      evicted = @working_memory.evict_to_make_space(token_count)
+      evicted_keys = evicted.map { |n| n[:key] }
+      @long_term_memory.mark_evicted(robot_id: @robot_id, node_ids: evicted_keys) if evicted_keys.any?
+    end
     @working_memory.add(node_id, content, token_count: token_count, access_count: 0)
 
+    # Mark node as in working memory in the robot_nodes join table
+    result[:robot_node].update!(working_memory: true)
+
     update_robot_activity
     node_id
   end
@@ -165,53 +192,70 @@ class HTM
   # Recall memories from a timeframe and topic
   #
   # @param topic [String] Topic to search for (required)
-  # @param timeframe [String, Range, nil] Time range (default: last 7 days). Examples: "last week", 7.days.ago..Time.now
+  # @param timeframe [nil, Range, Array<Range>, Date, DateTime, Time, String, Symbol] Time filter
+  #   - nil: No time filter (search all memories)
+  #   - Range: Time range (e.g., 7.days.ago..Time.now)
+  #   - Array<Range>: Multiple time windows (OR'd together)
+  #   - Date: Entire day
+  #   - DateTime/Time: Entire day containing that moment
+  #   - String: Natural language (e.g., "last week", "few days ago")
+  #   - :auto: Extract timeframe from topic query automatically
   # @param limit [Integer] Maximum number of nodes to retrieve (default: 20)
   # @param strategy [Symbol] Search strategy (:vector, :fulltext, :hybrid) (default: :vector)
   # @param with_relevance [Boolean] Include dynamic relevance scores (default: false)
   # @param query_tags [Array<String>] Tags to boost relevance (default: [])
   # @param raw [Boolean] Return full node hashes (true) or just content strings (false) (default: false)
+  # @param metadata [Hash] Filter by metadata fields using JSONB containment (default: {})
   # @return [Array<String>, Array<Hash>] Content strings (raw: false) or full node hashes (raw: true)
   #
-  # @example Basic usage (returns content strings)
+  # @example Basic usage - no time filter (returns content strings)
   #   memories = htm.recall("PostgreSQL")
   #   # => ["PostgreSQL is great for time-series data", "PostgreSQL with TimescaleDB..."]
   #
-  # @example Get full node hashes
-  #   nodes = htm.recall("PostgreSQL", raw: true)
-  #   # => [{"id" => 1, "content" => "...", "created_at" => "...", ...}, ...]
-  #
-  # @example With timeframe
+  # @example With explicit timeframe
   #   memories = htm.recall("PostgreSQL", timeframe: "last week")
+  #   memories = htm.recall("PostgreSQL", timeframe: Date.today)
+  #   memories = htm.recall("PostgreSQL", timeframe: 7.days.ago..Time.now)
   #
-  # @example With all options
-  #   memories = htm.recall("PostgreSQL",
-  #     timeframe: "last month",
-  #     limit: 50,
-  #     strategy: :hybrid,
-  #     with_relevance: true,
-  #     query_tags: ["database", "timeseries"])
+  # @example Auto-extract timeframe from query
+  #   memories = htm.recall("what did we discuss last week about PostgreSQL", timeframe: :auto)
+  #   # Extracts "last week" as timeframe, searches for "what did we discuss about PostgreSQL"
   #
-  def recall(topic, timeframe: nil, limit: 20, strategy: :vector, with_relevance: false, query_tags: [], raw: false)
-    # Use default timeframe if not provided (last 7 days)
-    timeframe ||= "last 7 days"
-
+  # @example Multiple time windows
+  #   memories = htm.recall("meetings", timeframe: [last_monday, last_friday])
+  #
+  # @example Filter by metadata
+  #   memories = htm.recall("preferences", metadata: { source: "user" })
+  #   memories = htm.recall("decisions", metadata: { confidence: 0.9, type: "architectural" })
+  #
+  def recall(topic, timeframe: nil, limit: 20, strategy: :vector, with_relevance: false, query_tags: [], raw: false, metadata: {})
     # Validate inputs
     validate_timeframe!(timeframe)
     validate_positive_integer!(limit, "limit")
     validate_recall_strategy!(strategy)
     validate_array!(query_tags, "query_tags")
-
-    parsed_timeframe = parse_timeframe(timeframe)
+    validate_metadata!(metadata)
+
+    # Normalize timeframe and potentially extract from query
+    search_query = topic
+    normalized_timeframe = if timeframe == :auto
+      result = HTM::Timeframe.normalize(:auto, query: topic)
+      search_query = result.query  # Use cleaned query for search
+      HTM.logger.debug "Auto-extracted timeframe: #{result.extracted.inspect}" if result.extracted
+      result.timeframe
+    else
+      HTM::Timeframe.normalize(timeframe)
+    end
 
     # Use relevance-based search if requested
     if with_relevance
       nodes = @long_term_memory.search_with_relevance(
-        timeframe: parsed_timeframe,
-        query: topic,
+        timeframe: normalized_timeframe,
+        query: search_query,
         query_tags: query_tags,
         limit: limit,
-        embedding_service: (strategy == :vector || strategy == :hybrid) ? HTM : nil
+        embedding_service: (strategy == :vector || strategy == :hybrid) ? HTM : nil,
+        metadata: metadata
       )
     else
       # Perform standard RAG-based retrieval
@@ -219,24 +263,27 @@ class HTM
       when :vector
         # Vector search using query embedding
         @long_term_memory.search(
-          timeframe: parsed_timeframe,
-          query: topic,
+          timeframe: normalized_timeframe,
+          query: search_query,
           limit: limit,
-          embedding_service: HTM
+          embedding_service: HTM,
+          metadata: metadata
         )
       when :fulltext
         @long_term_memory.search_fulltext(
-          timeframe: parsed_timeframe,
-          query: topic,
-          limit: limit
+          timeframe: normalized_timeframe,
+          query: search_query,
+          limit: limit,
+          metadata: metadata
         )
       when :hybrid
         # Hybrid search combining vector + fulltext
         @long_term_memory.search_hybrid(
-          timeframe: parsed_timeframe,
-          query: topic,
+          timeframe: normalized_timeframe,
+          query: search_query,
           limit: limit,
-          embedding_service: HTM
+          embedding_service: HTM,
+          metadata: metadata
         )
       end
     end
@@ -252,32 +299,222 @@ class HTM
     raw ? nodes : nodes.map { |node| node['content'] }
   end
 
-  # Forget a memory node (explicit deletion)
+  # Forget a memory node (soft delete by default, permanent delete requires confirmation)
   #
-  # @param key [String] Key of the node to delete
-  # @param confirm [Symbol] Must be :confirmed to proceed
+  # By default, performs a soft delete (sets deleted_at timestamp). The node
+  # remains in the database but is excluded from queries. Use soft: false
+  # with confirm: :confirmed for permanent deletion.
+  #
+  # @param node_id [Integer] ID of the node to delete
+  # @param soft [Boolean] If true (default), soft delete; if false, permanent delete
+  # @param confirm [Symbol] Must be :confirmed to proceed with permanent deletion
   # @return [Boolean] true if deleted
-  # @raise [ArgumentError] if confirmation not provided
+  # @raise [ArgumentError] if permanent deletion requested without confirmation
   # @raise [HTM::NotFoundError] if node doesn't exist
   #
-  def forget(node_id, confirm: false)
+  # @example Soft delete (recoverable)
+  #   htm.forget(node_id)
+  #   htm.forget(node_id, soft: true)
+  #
+  # @example Permanent delete (requires confirmation)
+  #   htm.forget(node_id, soft: false, confirm: :confirmed)
+  #
+  def forget(node_id, soft: true, confirm: false)
     # Validate inputs
-    raise ArgumentError, "node_id cannot be nil" if node_id.nil?
-    raise ArgumentError, "Must pass confirm: :confirmed to delete" unless confirm == :confirmed
+    raise ArgumentError, "Node ID cannot be nil" if node_id.nil?
 
-    # Verify node exists
-    unless @long_term_memory.exists?(node_id)
-      raise HTM::NotFoundError, "Node not found: #{node_id}"
+    # Permanent delete requires confirmation
+    if !soft && confirm != :confirmed
+      raise ArgumentError, "Permanent deletion requires confirm: :confirmed"
     end
 
-    # Delete the node and remove from working memory
-    @long_term_memory.delete(node_id)
+    # Verify node exists (including soft-deleted for restore scenarios)
+    node = HTM::Models::Node.with_deleted.find_by(id: node_id)
+    raise HTM::NotFoundError, "Node not found: #{node_id}" unless node
+
+    if soft
+      # Soft delete - mark as deleted but keep in database
+      node.soft_delete!
+      @long_term_memory.clear_cache!  # Invalidate cache since node is no longer visible
+      HTM.logger.info "Node #{node_id} soft deleted"
+    else
+      # Permanent delete (also invalidates cache internally)
+      @long_term_memory.delete(node_id)
+      HTM.logger.info "Node #{node_id} permanently deleted"
+    end
+
+    # Remove from working memory either way
     @working_memory.remove(node_id)
 
     update_robot_activity
     true
   end
 
+  # Restore a soft-deleted memory node
+  #
+  # @param node_id [Integer] ID of the soft-deleted node to restore
+  # @return [Boolean] true if restored
+  # @raise [HTM::NotFoundError] if node doesn't exist or isn't deleted
+  #
+  # @example
+  #   htm.forget(node_id)        # Soft delete
+  #   htm.restore(node_id)       # Bring it back
+  #
+  def restore(node_id)
+    raise ArgumentError, "Node ID cannot be nil" if node_id.nil?
+
+    # Find including soft-deleted nodes
+    node = HTM::Models::Node.with_deleted.find_by(id: node_id)
+    raise HTM::NotFoundError, "Node not found: #{node_id}" unless node
+
+    unless node.deleted?
+      raise ArgumentError, "Node #{node_id} is not soft-deleted"
+    end
+
+    node.restore!
+    HTM.logger.info "Node #{node_id} restored"
+
+    update_robot_activity
+    true
+  end
+
+  # Permanently delete all soft-deleted nodes older than specified time
+  #
+  # @param older_than [Time, ActiveSupport::Duration] Purge nodes soft-deleted before this time
+  # @param confirm [Symbol] Must be :confirmed to proceed
+  # @return [Integer] Number of nodes permanently deleted
+  # @raise [ArgumentError] if confirmation not provided
+  #
+  # @example Purge nodes deleted more than 30 days ago
+  #   htm.purge_deleted(older_than: 30.days.ago, confirm: :confirmed)
+  #
+  # @example Purge nodes deleted before a specific date
+  #   htm.purge_deleted(older_than: Time.new(2024, 1, 1), confirm: :confirmed)
+  #
+  def purge_deleted(older_than:, confirm: false)
+    raise ArgumentError, "Purge requires confirm: :confirmed" unless confirm == :confirmed
+
+    count = HTM::Models::Node.purge_deleted(older_than: older_than)
+    HTM.logger.info "Purged #{count} soft-deleted nodes older than #{older_than}"
+
+    count
+  end
+
+  # Clear all nodes from working memory
+  #
+  # Marks all nodes as evicted from working memory (in database) and clears
+  # the in-memory cache. Nodes remain in long-term memory.
+  #
+  # @return [Integer] Number of nodes cleared from working memory
+  #
+  # @example
+  #   htm.clear_working_memory  # => 5
+  #
+  def clear_working_memory
+    # Clear in-memory cache
+    @working_memory.clear
+
+    # Update database: mark all as evicted from working memory
+    count = HTM::Models::RobotNode
+      .where(robot_id: @robot_id, working_memory: true)
+      .update_all(working_memory: false)
+
+    HTM.logger.info "Cleared #{count} nodes from working memory"
+    count
+  end
+
+  # Load a single file into long-term memory
+  #
+  # Reads a text-based file (starting with markdown), chunks it by paragraph,
+  # and stores each chunk as a node. YAML frontmatter is preserved as metadata
+  # on the first chunk.
+  #
+  # @param path [String] Path to file
+  # @param force [Boolean] Force re-sync even if mtime unchanged (default: false)
+  # @return [Hash] Result with keys:
+  #   - :file_path [String] Absolute path to file
+  #   - :chunks_created [Integer] Number of new chunks created
+  #   - :chunks_updated [Integer] Number of existing chunks updated
+  #   - :chunks_deleted [Integer] Number of chunks soft-deleted
+  #   - :skipped [Boolean] True if file was unchanged and skipped
+  #
+  # @example Load a markdown file
+  #   result = htm.load_file('/path/to/doc.md')
+  #   # => { file_path: '/path/to/doc.md', chunks_created: 5, ... }
+  #
+  # @example Force re-sync even if unchanged
+  #   result = htm.load_file('/path/to/doc.md', force: true)
+  #
+  def load_file(path, force: false)
+    loader = HTM::Loaders::MarkdownLoader.new(self)
+    result = loader.load_file(path, force: force)
+
+    update_robot_activity unless result[:skipped]
+    result
+  end
+
+  # Load all matching files from a directory into long-term memory
+  #
+  # @param path [String] Directory path
+  # @param pattern [String] Glob pattern (default: '**/*.md')
+  # @param force [Boolean] Force re-sync even if unchanged (default: false)
+  # @return [Array<Hash>] Results for each file
+  #
+  # @example Load all markdown files recursively
+  #   results = htm.load_directory('/path/to/docs')
+  #
+  # @example Load only top-level markdown files
+  #   results = htm.load_directory('/path/to/docs', pattern: '*.md')
+  #
+  def load_directory(path, pattern: '**/*.md', force: false)
+    loader = HTM::Loaders::MarkdownLoader.new(self)
+    results = loader.load_directory(path, pattern: pattern, force: force)
+
+    # Update activity if any files were processed
+    if results.any? { |r| !r[:skipped] && !r[:error] }
+      update_robot_activity
+    end
+
+    results
+  end
+
+  # Get all nodes loaded from a specific file
+  #
+  # @param file_path [String] Path to the source file
+  # @return [ActiveRecord::Relation] Nodes from this file, ordered by chunk_position
+  #
+  # @example
+  #   nodes = htm.nodes_from_file('/path/to/doc.md')
+  #   nodes.each { |node| puts node.content }
+  #
+  def nodes_from_file(file_path)
+    source = HTM::Models::FileSource.find_by(file_path: File.expand_path(file_path))
+    return HTM::Models::Node.none unless source
+
+    HTM::Models::Node.from_source(source.id)
+  end
+
+  # Unload a file (soft-delete all its chunks and remove source record)
+  #
+  # @param file_path [String] Path to the source file
+  # @return [Integer] Number of nodes soft-deleted
+  #
+  # @example
+  #   count = htm.unload_file('/path/to/doc.md')
+  #   puts "Unloaded #{count} chunks"
+  #
+  def unload_file(file_path)
+    source = HTM::Models::FileSource.find_by(file_path: File.expand_path(file_path))
+    return 0 unless source
+
+    count = source.soft_delete_chunks!
+    @long_term_memory.clear_cache!
+    source.destroy
+
+    update_robot_activity
+    count
+  end
+
   private
 
   def register_robot
@@ -318,35 +555,30 @@ class HTM
     token_count = node['token_count'].to_i
     access_count = (node['access_count'] || 0).to_i
     last_accessed = node['last_accessed'] ? Time.parse(node['last_accessed'].to_s) : nil
+    node_id = node['id']
 
-    if @working_memory.has_space?(token_count)
-      @working_memory.add(
-        node['id'],
-        node['content'],
-        token_count: token_count,
-        access_count: access_count,
-        last_accessed: last_accessed,
-        from_recall: true
-      )
-    else
+    unless @working_memory.has_space?(token_count)
       # Evict to make space
       evicted = @working_memory.evict_to_make_space(token_count)
       evicted_keys = evicted.map { |n| n[:key] }
-      @long_term_memory.mark_evicted(evicted_keys) if evicted_keys.any?
-
-      # Now add the recalled node
-      @working_memory.add(
-        node['id'],
-        node['content'],
-        token_count: token_count,
-        access_count: access_count,
-        last_accessed: last_accessed,
-        from_recall: true
-      )
+      @long_term_memory.mark_evicted(robot_id: @robot_id, node_ids: evicted_keys) if evicted_keys.any?
     end
-  end
 
-  private
+    # Add to in-memory working memory
+    @working_memory.add(
+      node_id,
+      node['content'],
+      token_count: token_count,
+      access_count: access_count,
+      last_accessed: last_accessed,
+      from_recall: true
+    )
+
+    # Mark node as in working memory in the robot_nodes join table
+    HTM::Models::RobotNode
+      .find_by(robot_id: @robot_id, node_id: node_id)
+      &.update!(working_memory: true)
+  end
 
   # Validation helper methods
 
@@ -364,58 +596,24 @@ class HTM
 
 
   def validate_timeframe!(timeframe)
-    return if timeframe.is_a?(Range) || timeframe.is_a?(String)
-    raise ValidationError, "Timeframe must be a Range or String, got #{timeframe.class}"
+    return if HTM::Timeframe.valid?(timeframe)
+    raise ValidationError, "Invalid timeframe type: #{timeframe.class}. " \
+      "Expected nil, Range, Array<Range>, Date, DateTime, Time, String, or :auto"
   end
 
   def validate_positive_integer!(value, name)
     raise ValidationError, "#{name} must be a positive Integer" unless value.is_a?(Integer) && value > 0
   end
 
-  # Timeframe parsing methods
+  def validate_metadata!(metadata)
+    raise ValidationError, "Metadata must be a Hash" unless metadata.is_a?(Hash)
 
-  def parse_timeframe(timeframe)
-    case timeframe
-    when Range
-      timeframe
-    when String
-      parse_natural_timeframe(timeframe)
-    else
-      raise ArgumentError, "Invalid timeframe: #{timeframe}"
+    # Ensure all keys are strings or symbols (will be converted to strings in JSON)
+    metadata.each_key do |key|
+      unless key.is_a?(String) || key.is_a?(Symbol)
+        raise ValidationError, "Metadata keys must be strings or symbols"
+      end
     end
   end
 
-  def parse_natural_timeframe(text)
-    now = Time.now
-
-    case text.downcase
-    when /last week/
-      (now - 7 * 24 * 3600)..now
-    when /yesterday/
-      start_of_yesterday = Time.new(now.year, now.month, now.day - 1)
-      start_of_yesterday..(start_of_yesterday + 24 * 3600)
-    when /last (\d+) days?/
-      days = $1.to_i
-      (now - days * 24 * 3600)..now
-    when /last (\d+) seconds?/
-      seconds = $1.to_i
-      (now - seconds)..now
-    when /last (\d+) minutes?/
-      minutes = $1.to_i
-      (now - minutes * 60)..now
-    when /last (\d+) hours?/
-      hours = $1.to_i
-      (now - hours * 3600)..now
-    when /this month/
-      start_of_month = Time.new(now.year, now.month, 1)
-      start_of_month..now
-    when /last month/
-      start_of_last_month = Time.new(now.year, now.month - 1, 1)
-      end_of_last_month = Time.new(now.year, now.month, 1) - 1
-      start_of_last_month..end_of_last_month
-    else
-      # Default to last 24 hours
-      (now - 24 * 3600)..now
-    end
-  end
 end