htm 0.0.11 → 0.0.14

data/lib/htm/models/node.rb

@@ -55,6 +55,10 @@ class HTM
       scope :with_embeddings, -> { where.not(embedding: nil) }
       scope :from_source, ->(source_id) { where(source_id: source_id).order(:chunk_position) }
 
+      # Proposition scopes
+      scope :propositions, -> { where("metadata->>'is_proposition' = 'true'") }
+      scope :non_propositions, -> { where("metadata IS NULL OR metadata->>'is_proposition' IS NULL OR metadata->>'is_proposition' != 'true'") }
+
       # Soft delete scopes
       scope :deleted, -> { unscoped.where.not(deleted_at: nil) }
       scope :with_deleted, -> { unscoped }
@@ -170,19 +174,40 @@ class HTM
       end
 
       # Soft delete - mark node as deleted without removing from database
+      # Also cascades soft delete to associated robot_nodes and node_tags
       #
       # @return [Boolean] true if soft deleted successfully
       #
       def soft_delete!
-        update!(deleted_at: Time.current)
+        transaction do
+          now = Time.current
+          update!(deleted_at: now)
+
+          # Cascade soft delete to associated robot_nodes
+          robot_nodes.update_all(deleted_at: now)
+
+          # Cascade soft delete to associated node_tags
+          node_tags.update_all(deleted_at: now)
+        end
+        true
       end
 
       # Restore a soft-deleted node
+      # Also cascades restoration to associated robot_nodes and node_tags
       #
       # @return [Boolean] true if restored successfully
       #
       def restore!
-        update!(deleted_at: nil)
+        transaction do
+          update!(deleted_at: nil)
+
+          # Cascade restoration to associated robot_nodes
+          HTM::Models::RobotNode.unscoped.where(node_id: id).update_all(deleted_at: nil)
+
+          # Cascade restoration to associated node_tags
+          HTM::Models::NodeTag.unscoped.where(node_id: id).update_all(deleted_at: nil)
+        end
+        true
       end
 
       # Check if node is soft-deleted
@@ -193,6 +218,14 @@ class HTM
         deleted_at.present?
       end
 
+      # Check if node is a proposition (extracted atomic fact)
+      #
+      # @return [Boolean] true if metadata['is_proposition'] is true
+      #
+      def proposition?
+        metadata&.dig('is_proposition') == true
+      end
+
       private
 
       def set_content_hash

data/lib/htm/models/node_tag.rb

@@ -19,10 +19,41 @@ class HTM
       before_create :set_created_at
 
       # Scopes
+      # Soft delete - by default, only show non-deleted entries
+      default_scope { where(deleted_at: nil) }
+
       scope :for_node, ->(node_id) { where(node_id: node_id) }
       scope :for_tag, ->(tag_id) { where(tag_id: tag_id) }
       scope :recent, -> { order(created_at: :desc) }
 
+      # Soft delete scopes
+      scope :deleted, -> { unscoped.where.not(deleted_at: nil) }
+      scope :with_deleted, -> { unscoped }
+
+      # Soft delete - mark as deleted without removing from database
+      #
+      # @return [Boolean] true if soft deleted successfully
+      #
+      def soft_delete!
+        update!(deleted_at: Time.current)
+      end
+
+      # Restore a soft-deleted entry
+      #
+      # @return [Boolean] true if restored successfully
+      #
+      def restore!
+        update!(deleted_at: nil)
+      end
+
+      # Check if entry is soft-deleted
+      #
+      # @return [Boolean] true if deleted_at is set
+      #
+      def deleted?
+        deleted_at.present?
+      end
+
       private
 
       def set_created_at

data/lib/htm/models/robot_node.rb

@@ -30,12 +30,43 @@ class HTM
       validates :robot_id, uniqueness: { scope: :node_id, message: 'already linked to this node' }
 
       # Scopes
+      # Soft delete - by default, only show non-deleted entries
+      default_scope { where(deleted_at: nil) }
+
       scope :recent, -> { order(last_remembered_at: :desc) }
       scope :by_robot, ->(robot_id) { where(robot_id: robot_id) }
       scope :by_node, ->(node_id) { where(node_id: node_id) }
       scope :frequently_remembered, -> { where('remember_count > 1').order(remember_count: :desc) }
       scope :in_working_memory, -> { where(working_memory: true) }
 
+      # Soft delete scopes
+      scope :deleted, -> { unscoped.where.not(deleted_at: nil) }
+      scope :with_deleted, -> { unscoped }
+
+      # Soft delete - mark as deleted without removing from database
+      #
+      # @return [Boolean] true if soft deleted successfully
+      #
+      def soft_delete!
+        update!(deleted_at: Time.current)
+      end
+
+      # Restore a soft-deleted entry
+      #
+      # @return [Boolean] true if restored successfully
+      #
+      def restore!
+        update!(deleted_at: nil)
+      end
+
+      # Check if entry is soft-deleted
+      #
+      # @return [Boolean] true if deleted_at is set
+      #
+      def deleted?
+        deleted_at.present?
+      end
+
       # Record that a robot remembered this content again
       #
       # @return [RobotNode] Updated record

data/lib/htm/models/tag.rb

@@ -23,11 +23,31 @@ class HTM
       before_create :set_created_at
 
       # Scopes
+      # Soft delete - by default, only show non-deleted tags
+      default_scope { where(deleted_at: nil) }
+
       scope :by_name, ->(name) { where(name: name) }
       scope :with_prefix, ->(prefix) { where("name LIKE ?", "#{prefix}%") }
       scope :hierarchical, -> { where("name LIKE '%:%'") }
       scope :root_level, -> { where("name NOT LIKE '%:%'") }
 
+      # Soft delete scopes
+      scope :deleted, -> { unscoped.where.not(deleted_at: nil) }
+      scope :with_deleted, -> { unscoped }
+
+      # Orphaned tags - tags with no active (non-deleted) node associations
+      scope :orphaned, -> {
+        where(
+          "NOT EXISTS (
+            SELECT 1 FROM node_tags
+            JOIN nodes ON nodes.id = node_tags.node_id
+            WHERE node_tags.tag_id = tags.id
+            AND node_tags.deleted_at IS NULL
+            AND nodes.deleted_at IS NULL
+          )"
+        )
+      }
+
       # Class methods
 
       # Find tags with a given prefix (hierarchical query)
@@ -368,6 +388,30 @@ class HTM
         node_tags.count
       end
 
+      # Soft delete - mark tag as deleted without removing from database
+      #
+      # @return [Boolean] true if soft deleted successfully
+      #
+      def soft_delete!
+        update!(deleted_at: Time.current)
+      end
+
+      # Restore a soft-deleted tag
+      #
+      # @return [Boolean] true if restored successfully
+      #
+      def restore!
+        update!(deleted_at: nil)
+      end
+
+      # Check if tag is soft-deleted
+      #
+      # @return [Boolean] true if deleted_at is set
+      #
+      def deleted?
+        deleted_at.present?
+      end
+
       private
 
       def set_created_at

data/lib/htm/proposition_service.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require_relative 'errors'
+
+class HTM
+  # Proposition Service - Extracts atomic factual propositions from text
+  #
+  # This service breaks complex text into simple, self-contained factual
+  # statements that can be stored as independent memory nodes. Each proposition:
+  # - Expresses a single fact
+  # - Is understandable without context
+  # - Uses full names, not pronouns
+  # - Includes relevant dates/qualifiers
+  # - Contains one subject-predicate relationship
+  #
+  # The actual LLM call is delegated to HTM.configuration.proposition_extractor
+  #
+  # @example
+  #   propositions = HTM::PropositionService.extract(
+  #     "In 1969, Neil Armstrong became the first person to walk on the Moon during Apollo 11."
+  #   )
+  #   # => ["Neil Armstrong was an astronaut.",
+  #   #     "Neil Armstrong walked on the Moon in 1969.",
+  #   #     "Neil Armstrong was the first person to walk on the Moon.",
+  #   #     "Neil Armstrong walked on the Moon during the Apollo 11 mission.",
+  #   #     "The Apollo 11 mission occurred in 1969."]
+  #
+  class PropositionService
+    MIN_PROPOSITION_LENGTH = 10   # Minimum characters for a valid proposition
+    MAX_PROPOSITION_LENGTH = 1000 # Maximum characters for a valid proposition
+
+    # Circuit breaker for proposition extraction API calls
+    @circuit_breaker = nil
+    @circuit_breaker_mutex = Mutex.new
+
+    class << self
+      # Get or create the circuit breaker for proposition service
+      #
+      # @return [HTM::CircuitBreaker] The circuit breaker instance
+      #
+      def circuit_breaker
+        @circuit_breaker_mutex.synchronize do
+          @circuit_breaker ||= HTM::CircuitBreaker.new(
+            name: 'proposition_service',
+            failure_threshold: 5,
+            reset_timeout: 60
+          )
+        end
+      end
+
+      # Reset the circuit breaker (useful for testing)
+      #
+      # @return [void]
+      #
+      def reset_circuit_breaker!
+        @circuit_breaker_mutex.synchronize do
+          @circuit_breaker&.reset!
+        end
+      end
+    end
+
+    # Extract propositions from text content
+    #
+    # @param content [String] Text to analyze
+    # @return [Array<String>] Array of atomic propositions
+    # @raise [CircuitBreakerOpenError] If circuit breaker is open
+    # @raise [PropositionError] If extraction fails
+    #
+    def self.extract(content)
+      HTM.logger.debug "PropositionService: Extracting propositions from #{content.length} chars"
+
+      # Use circuit breaker to protect against cascading failures
+      raw_propositions = circuit_breaker.call do
+        HTM.configuration.proposition_extractor.call(content)
+      end
+
+      # Parse response (may be string or array)
+      parsed_propositions = parse_propositions(raw_propositions)
+
+      # Validate and filter propositions
+      valid_propositions = validate_and_filter_propositions(parsed_propositions)
+
+      HTM.logger.debug "PropositionService: Extracted #{valid_propositions.length} valid propositions"
+
+      valid_propositions
+
+    rescue HTM::CircuitBreakerOpenError
+      # Re-raise circuit breaker errors without wrapping
+      raise
+    rescue HTM::PropositionError
+      raise
+    rescue StandardError => e
+      HTM.logger.error "PropositionService: Failed to extract propositions: #{e.message}"
+      raise HTM::PropositionError, "Proposition extraction failed: #{e.message}"
+    end
+
+    # Parse proposition response (handles string or array input)
+    #
+    # @param raw_propositions [String, Array] Raw response from extractor
+    # @return [Array<String>] Parsed proposition strings
+    #
+    def self.parse_propositions(raw_propositions)
+      case raw_propositions
+      when Array
+        # Already an array, return as-is
+        raw_propositions.map(&:to_s).map(&:strip).reject(&:empty?)
+      when String
+        # String response - split by newlines, remove list markers
+        raw_propositions
+          .split("\n")
+          .map(&:strip)
+          .map { |line| line.sub(/^[-*•]\s*/, '') } # Remove bullet points
+          .map { |line| line.sub(/^\d+\.\s*/, '') } # Remove numbered lists
+          .map(&:strip)
+          .reject(&:empty?)
+      else
+        raise HTM::PropositionError, "Proposition response must be Array or String, got #{raw_propositions.class}"
+      end
+    end
+
+    # Validate and filter propositions
+    #
+    # @param propositions [Array<String>] Parsed propositions
+    # @return [Array<String>] Valid propositions only
+    #
+    def self.validate_and_filter_propositions(propositions)
+      valid_propositions = []
+
+      propositions.each do |proposition|
+        # Check minimum length
+        if proposition.length < MIN_PROPOSITION_LENGTH
+          HTM.logger.debug "PropositionService: Proposition too short, skipping: #{proposition}"
+          next
+        end
+
+        # Check maximum length
+        if proposition.length > MAX_PROPOSITION_LENGTH
+          HTM.logger.warn "PropositionService: Proposition too long, skipping: #{proposition[0..50]}..."
+          next
+        end
+
+        # Check for actual content (not just punctuation/whitespace)
+        unless proposition.match?(/[a-zA-Z]{3,}/)
+          HTM.logger.debug "PropositionService: Proposition lacks content, skipping: #{proposition}"
+          next
+        end
+
+        # Proposition is valid
+        valid_propositions << proposition
+      end
+
+      valid_propositions.uniq
+    end
+
+    # Validate single proposition
+    #
+    # @param proposition [String] Proposition to validate
+    # @return [Boolean] True if valid
+    #
+    def self.valid_proposition?(proposition)
+      return false unless proposition.is_a?(String)
+      return false if proposition.length < MIN_PROPOSITION_LENGTH
+      return false if proposition.length > MAX_PROPOSITION_LENGTH
+      return false unless proposition.match?(/[a-zA-Z]{3,}/)
+
+      true
+    end
+  end
+end

data/lib/htm/query_cache.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require 'lru_redux'
+require 'set'
+
+class HTM
+  # Thread-safe query result cache with TTL and statistics
+  #
+  # Provides LRU caching for expensive query results with:
+  # - Configurable size and TTL
+  # - Thread-safe statistics tracking
+  # - Fast cache key generation (using Ruby's built-in hash)
+  # - Selective cache invalidation by method type
+  #
+  # @example Create a cache
+  #   cache = HTM::QueryCache.new(size: 1000, ttl: 300)
+  #
+  # @example Use the cache
+  #   result = cache.fetch(:search, timeframe, query, limit) do
+  #     expensive_search_operation
+  #   end
+  #
+  # @example Check statistics
+  #   cache.stats
+  #   # => { hits: 42, misses: 10, hit_rate: 80.77, size: 52 }
+  #
+  # @example Selective invalidation
+  #   cache.invalidate_methods!(:search, :hybrid)  # Only invalidate search-related entries
+  #
+  class QueryCache
+    attr_reader :enabled
+
+    # Cache key prefix for method-based invalidation
+    METHOD_PREFIX = "m:".freeze
+
+    # Initialize a new query cache
+    #
+    # @param size [Integer] Maximum number of entries (default: 1000, use 0 to disable)
+    # @param ttl [Integer] Time-to-live in seconds (default: 300)
+    #
+    def initialize(size: 1000, ttl: 300)
+      @enabled = size > 0
+
+      if @enabled
+        @cache = LruRedux::TTL::ThreadSafeCache.new(size, ttl)
+        @hits = 0
+        @misses = 0
+        @mutex = Mutex.new
+        # Track keys by method for selective invalidation
+        @keys_by_method = Hash.new { |h, k| h[k] = Set.new }
+      end
+    end
+
+    # Fetch a value from cache or execute block
+    #
+    # @param method [Symbol] Method name for cache key
+    # @param args [Array] Arguments for cache key
+    # @yield Block that computes the value if not cached
+    # @return [Object] Cached or computed value
+    #
+    def fetch(method, *args, &block)
+      return yield unless @enabled
+
+      key = cache_key(method, *args)
+
+      if (cached = @cache[key])
+        @mutex.synchronize { @hits += 1 }
+        HTM::Telemetry.cache_operations.add(1, attributes: { 'operation' => 'hit' })
+        return cached
+      end
+
+      @mutex.synchronize { @misses += 1 }
+      HTM::Telemetry.cache_operations.add(1, attributes: { 'operation' => 'miss' })
+      result = yield
+      @cache[key] = result
+
+      # Track key for selective invalidation
+      @mutex.synchronize { @keys_by_method[method] << key }
+
+      result
+    end
+
+    # Clear all cached entries
+    #
+    # @return [void]
+    #
+    def clear!
+      return unless @enabled
+
+      @cache.clear
+      @mutex.synchronize { @keys_by_method.clear }
+    end
+
+    # Invalidate cache (alias for clear!)
+    #
+    # @return [void]
+    #
+    def invalidate!
+      clear!
+    end
+
+    # Invalidate cache entries for specific methods only
+    #
+    # More efficient than full invalidation when only certain
+    # types of cached data need to be refreshed.
+    #
+    # @param methods [Array<Symbol>] Method names to invalidate
+    # @return [Integer] Number of entries invalidated
+    #
+    def invalidate_methods!(*methods)
+      return 0 unless @enabled
+
+      count = 0
+      @mutex.synchronize do
+        methods.each do |method|
+          keys = @keys_by_method.delete(method) || Set.new
+          keys.each do |key|
+            @cache.delete(key)
+            count += 1
+          end
+        end
+      end
+      count
+    end
+
+    # Get cache statistics
+    #
+    # @return [Hash, nil] Statistics hash or nil if disabled
+    #
+    def stats
+      return nil unless @enabled
+
+      total = @hits + @misses
+      hit_rate = total > 0 ? (@hits.to_f / total * 100).round(2) : 0.0
+
+      {
+        hits: @hits,
+        misses: @misses,
+        hit_rate: hit_rate,
+        size: @cache.count
+      }
+    end
+
+    # Check if cache is enabled
+    #
+    # @return [Boolean]
+    #
+    def enabled?
+      @enabled
+    end
+
+    private
+
+    # Generate a cache key from method and arguments
+    #
+    # Uses Ruby's built-in hash method which is much faster than SHA-256.
+    # The combination of method name and argument hash provides sufficient
+    # uniqueness for cache keys while being ~10x faster than cryptographic hashing.
+    #
+    # @param method [Symbol] Method name
+    # @param args [Array] Arguments
+    # @return [String] Hash-based key
+    #
+    def cache_key(method, *args)
+      # Build composite hash from all arguments
+      args_hash = args.map { |arg| normalize_arg(arg) }.hash
+      # Combine method and args into a single key
+      "#{method}:#{args_hash}"
+    end
+
+    # Normalize an argument for cache key generation
+    #
+    # Uses type-safe serialization to prevent cache poisoning via malicious to_s.
+    # Only known safe types are serialized; unknown types include class name.
+    #
+    # @param arg [Object] Argument to normalize
+    # @return [String] Normalized string representation
+    #
+    def normalize_arg(arg)
+      case arg
+      when nil
+        "nil"
+      when Integer, Float
+        # Safe numeric types
+        "#{arg.class}:#{arg}"
+      when String
+        # Include class to differentiate from symbols
+        "String:#{arg}"
+      when Symbol
+        "Symbol:#{arg}"
+      when TrueClass, FalseClass
+        "Bool:#{arg}"
+      when Time, DateTime
+        # Use ISO8601 for consistent time representation
+        "Time:#{arg.to_i}"
+      when Date
+        "Date:#{arg.iso8601}"
+      when Range
+        # Use normalized form for range endpoints
+        "Range:#{normalize_arg(arg.begin)}-#{normalize_arg(arg.end)}"
+      when Array
+        # Recursively normalize array elements
+        "Array:[#{arg.map { |a| normalize_arg(a) }.join(',')}]"
+      when Hash
+        # Sort keys for deterministic ordering, recursively normalize values
+        "Hash:{#{arg.sort_by { |k, _| k.to_s }.map { |k, v| "#{normalize_arg(k)}=>#{normalize_arg(v)}" }.join(',')}}"
+      else
+        # Unknown types: use class name and object_id to prevent collision
+        # Don't rely on to_s which could be maliciously overridden
+        "#{arg.class}##{arg.object_id}"
+      end
+    end
+  end
+end