htm 0.0.11 → 0.0.14

data/lib/htm/sql_builder.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+class HTM
+  # SQL building utilities for constructing safe, parameterized queries
+  #
+  # Provides class methods for building SQL conditions for:
+  # - Timeframe filtering (single range or multiple ranges)
+  # - Metadata filtering (JSONB containment)
+  # - Embedding sanitization and padding (SQL injection prevention)
+  # - LIKE pattern sanitization (wildcard injection prevention)
+  #
+  # All methods use proper escaping and parameterization to prevent SQL injection.
+  #
+  # @example Build a timeframe condition
+  #   HTM::SqlBuilder.timeframe_condition(1.week.ago..Time.now)
+  #   # => "(created_at BETWEEN '2024-01-01' AND '2024-01-08')"
+  #
+  # @example Build a metadata condition
+  #   HTM::SqlBuilder.metadata_condition({ priority: "high" })
+  #   # => "(metadata @> '{\"priority\":\"high\"}'::jsonb)"
+  #
+  # @example Sanitize an embedding
+  #   HTM::SqlBuilder.sanitize_embedding([0.1, 0.2, 0.3])
+  #   # => "[0.1,0.2,0.3]"
+  #
+  # @example Sanitize a LIKE pattern
+  #   HTM::SqlBuilder.sanitize_like_pattern("test%pattern")
+  #   # => "test\\%pattern"
+  #
+  class SqlBuilder
+    # Maximum embedding dimension supported by pgvector with HNSW index
+    MAX_EMBEDDING_DIMENSION = 2000
+
+    class << self
+      # Sanitize embedding for SQL use
+      #
+      # Validates that all values are numeric and converts to safe PostgreSQL vector format.
+      # This prevents SQL injection by ensuring only valid numeric values are included.
+      #
+      # @param embedding [Array<Numeric>] Embedding vector
+      # @return [String] Sanitized vector string for PostgreSQL (e.g., "[0.1,0.2,0.3]")
+      # @raise [ArgumentError] If embedding contains non-numeric values
+      #
+      def sanitize_embedding(embedding)
+        unless embedding.is_a?(Array)
+          raise ArgumentError, "Embedding must be an Array, got #{embedding.class}"
+        end
+
+        if embedding.empty?
+          raise ArgumentError, "Embedding cannot be empty"
+        end
+
+        # Find invalid values for detailed error message
+        invalid_indices = []
+        embedding.each_with_index do |v, i|
+          unless v.is_a?(Numeric) && v.respond_to?(:finite?) && v.finite?
+            invalid_indices << i
+          end
+        end
+
+        unless invalid_indices.empty?
+          sample = invalid_indices.first(5).map { |i| "index #{i}: #{embedding[i].inspect}" }.join(", ")
+          raise ArgumentError, "Embedding contains invalid values at #{sample}"
+        end
+
+        "[#{embedding.map { |v| v.to_f }.join(',')}]"
+      end
+
+      # Pad embedding to target dimension
+      #
+      # Pads embedding with zeros to reach the target dimension for pgvector compatibility.
+      #
+      # @param embedding [Array<Numeric>] Embedding vector
+      # @param target_dimension [Integer] Target dimension (default: MAX_EMBEDDING_DIMENSION)
+      # @return [Array<Numeric>] Padded embedding
+      #
+      def pad_embedding(embedding, target_dimension: MAX_EMBEDDING_DIMENSION)
+        return embedding if embedding.length >= target_dimension
+
+        embedding + Array.new(target_dimension - embedding.length, 0.0)
+      end
+
+      # Sanitize a string for use in SQL LIKE patterns
+      #
+      # Escapes SQL LIKE wildcards (% and _) to prevent pattern injection.
+      #
+      # @param pattern [String] Pattern to sanitize
+      # @return [String] Sanitized pattern safe for LIKE queries
+      #
+      def sanitize_like_pattern(pattern)
+        return "" if pattern.nil?
+
+        pattern.to_s.gsub(/[%_\\]/) { |match| "\\#{match}" }
+      end
+
+      # Build SQL condition for timeframe filtering
+      #
+      # @param timeframe [nil, Range, Array<Range>] Time range(s)
+      # @param table_alias [String, nil] Table alias (default: none)
+      # @param column [String] Column name (default: "created_at")
+      # @return [String, nil] SQL condition or nil for no filter
+      #
+      def timeframe_condition(timeframe, table_alias: nil, column: "created_at")
+        return nil if timeframe.nil?
+
+        prefix = table_alias ? "#{table_alias}." : ""
+        full_column = "#{prefix}#{column}"
+        conn = ActiveRecord::Base.connection
+
+        case timeframe
+        when Range
+          begin_quoted = conn.quote(timeframe.begin.iso8601)
+          end_quoted = conn.quote(timeframe.end.iso8601)
+          "(#{full_column} BETWEEN #{begin_quoted} AND #{end_quoted})"
+        when Array
+          conditions = timeframe.map do |range|
+            begin_quoted = conn.quote(range.begin.iso8601)
+            end_quoted = conn.quote(range.end.iso8601)
+            "(#{full_column} BETWEEN #{begin_quoted} AND #{end_quoted})"
+          end
+          "(#{conditions.join(' OR ')})"
+        end
+      end
+
+      # Apply timeframe filter to ActiveRecord scope
+      #
+      # @param scope [ActiveRecord::Relation] Base scope
+      # @param timeframe [nil, Range, Array<Range>] Time range(s)
+      # @param column [Symbol] Column name (default: :created_at)
+      # @return [ActiveRecord::Relation] Scoped query
+      #
+      def apply_timeframe(scope, timeframe, column: :created_at)
+        return scope if timeframe.nil?
+
+        case timeframe
+        when Range
+          scope.where(column => timeframe)
+        when Array
+          conditions = timeframe.map { |range| scope.where(column => range) }
+          conditions.reduce { |result, condition| result.or(condition) }
+        else
+          scope
+        end
+      end
+
+      # Build SQL condition for metadata filtering (JSONB containment)
+      #
+      # @param metadata [Hash] Metadata to filter by
+      # @param table_alias [String, nil] Table alias (default: none)
+      # @param column [String] Column name (default: "metadata")
+      # @return [String, nil] SQL condition or nil for no filter
+      #
+      def metadata_condition(metadata, table_alias: nil, column: "metadata")
+        return nil if metadata.nil? || metadata.empty?
+
+        prefix = table_alias ? "#{table_alias}." : ""
+        full_column = "#{prefix}#{column}"
+        conn = ActiveRecord::Base.connection
+
+        quoted_metadata = conn.quote(metadata.to_json)
+        "(#{full_column} @> #{quoted_metadata}::jsonb)"
+      end
+
+      # Apply metadata filter to ActiveRecord scope
+      #
+      # @param scope [ActiveRecord::Relation] Base scope
+      # @param metadata [Hash] Metadata to filter by
+      # @param column [String] Column name (default: "metadata")
+      # @return [ActiveRecord::Relation] Scoped query
+      #
+      def apply_metadata(scope, metadata, column: "metadata")
+        return scope if metadata.nil? || metadata.empty?
+
+        scope.where("#{column} @> ?::jsonb", metadata.to_json)
+      end
+    end
+  end
+end

data/lib/htm/tag_service.rb

@@ -15,7 +15,6 @@ class HTM
   # The actual LLM call is delegated to HTM.configuration.tag_extractor
   #
   class TagService
-    MAX_DEPTH = 4  # Maximum hierarchy depth (3 colons)
     TAG_FORMAT = /^[a-z0-9\-]+(:[a-z0-9\-]+)*$/  # Validation regex
 
     # Circuit breaker for tag extraction API calls
@@ -23,16 +22,26 @@ class HTM
     @circuit_breaker_mutex = Mutex.new
 
     class << self
+      # Maximum tag hierarchy depth (configurable, default 4)
+      #
+      # @return [Integer] Max depth (3 colons max by default)
+      #
+      def max_depth
+        HTM.configuration.max_tag_depth
+      end
+
       # Get or create the circuit breaker for tag service
       #
       # @return [HTM::CircuitBreaker] The circuit breaker instance
       #
       def circuit_breaker
+        config = HTM.configuration
         @circuit_breaker_mutex.synchronize do
           @circuit_breaker ||= HTM::CircuitBreaker.new(
             name: 'tag_service',
-            failure_threshold: 5,
-            reset_timeout: 60
+            failure_threshold: config.circuit_breaker_failure_threshold,
+            reset_timeout: config.circuit_breaker_reset_timeout,
+            half_open_max_calls: config.circuit_breaker_half_open_max_calls
           )
         end
       end
@@ -119,8 +128,9 @@ class HTM
 
         # Check depth
         depth = tag.count(':')
-        if depth >= MAX_DEPTH
-          HTM.logger.warn "TagService: Tag depth #{depth + 1} exceeds max #{MAX_DEPTH}, skipping: #{tag}"
+        max_tag_depth = max_depth
+        if depth >= max_tag_depth
+          HTM.logger.warn "TagService: Tag depth #{depth + 1} exceeds max #{max_tag_depth}, skipping: #{tag}"
           next
         end
 
@@ -155,7 +165,7 @@ class HTM
       return false unless tag.is_a?(String)
       return false if tag.empty?
       return false unless tag.match?(TAG_FORMAT)
-      return false if tag.count(':') >= MAX_DEPTH
+      return false if tag.count(':') >= max_depth
 
       # Ontological validation
       levels = tag.split(':')

data/lib/htm/tasks.rb

@@ -8,17 +8,23 @@
 #
 # This will make the following tasks available:
 #
-# Database tasks:
+# Database tasks (all respect RAILS_ENV, default: development):
+#   rake htm:db:create     # Create database if it doesn't exist
 #   rake htm:db:setup      # Set up HTM database schema and run migrations
 #   rake htm:db:migrate    # Run pending database migrations
 #   rake htm:db:status     # Show migration status
 #   rake htm:db:info       # Show database info
-#   rake htm:db:test       # Test database connection
+#   rake htm:db:verify     # Verify database connection
 #   rake htm:db:console    # Open PostgreSQL console
 #   rake htm:db:seed       # Seed database with sample data
 #   rake htm:db:drop       # Drop all HTM tables (destructive!)
 #   rake htm:db:reset      # Drop and recreate database (destructive!)
 #
+# Examples:
+#   RAILS_ENV=test rake htm:db:create   # Create htm_test database
+#   RAILS_ENV=test rake htm:db:setup    # Setup test database with migrations
+#   RAILS_ENV=test rake htm:db:drop     # Drop test database
+#
 # Async job tasks:
 #   rake htm:jobs:stats              # Show async job statistics
 #   rake htm:jobs:process_embeddings # Process pending embedding jobs

data/lib/htm/telemetry.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+class HTM
+  # OpenTelemetry-based observability for HTM
+  #
+  # Provides opt-in metrics collection with zero overhead when disabled.
+  # Uses the null object pattern - when telemetry is disabled or the SDK
+  # is not available, all metric operations are no-ops.
+  #
+  # @example Enable telemetry
+  #   HTM.configure do |config|
+  #     config.telemetry_enabled = true
+  #   end
+  #
+  # @example Set destination via environment
+  #   # Export to OTLP endpoint
+  #   ENV['OTEL_METRICS_EXPORTER'] = 'otlp'
+  #   ENV['OTEL_EXPORTER_OTLP_ENDPOINT'] = 'http://localhost:4318'
+  #
+  # @see notes/ot.md for full implementation details
+  #
+  module Telemetry
+    # Null meter that creates null instruments
+    # Used when telemetry is disabled or SDK unavailable
+    class NullMeter
+      include Singleton
+
+      def create_counter(*)
+        NullInstrument.instance
+      end
+
+      def create_histogram(*)
+        NullInstrument.instance
+      end
+
+      def create_up_down_counter(*)
+        NullInstrument.instance
+      end
+    end
+
+    # Null instrument that accepts but ignores all metric operations
+    class NullInstrument
+      include Singleton
+
+      def add(*) = nil
+      def record(*) = nil
+    end
+
+    class << self
+      # Check if telemetry is enabled and SDK is available
+      #
+      # @return [Boolean] true if telemetry should be active
+      #
+      def enabled?
+        HTM.configuration.telemetry_enabled && sdk_available?
+      end
+
+      # Check if OpenTelemetry SDK is installed
+      #
+      # @return [Boolean] true if SDK can be loaded
+      #
+      def sdk_available?
+        return @sdk_available if defined?(@sdk_available)
+
+        @sdk_available = begin
+          require 'opentelemetry-metrics-sdk'
+          true
+        rescue LoadError
+          false
+        end
+      end
+
+      # Initialize OpenTelemetry SDK
+      #
+      # Called automatically when telemetry is enabled.
+      # Safe to call multiple times.
+      #
+      # @return [void]
+      #
+      def setup
+        return unless enabled?
+        return if @setup_complete
+
+        OpenTelemetry::SDK.configure do |c|
+          c.service_name = 'htm'
+        end
+
+        @setup_complete = true
+        HTM.logger.info "Telemetry: OpenTelemetry SDK initialized"
+      end
+
+      # Get the meter for creating instruments
+      #
+      # @return [OpenTelemetry::Metrics::Meter, NullMeter] Real or null meter
+      #
+      def meter
+        return NullMeter.instance unless enabled?
+
+        setup
+        @meter ||= OpenTelemetry.meter_provider.meter('htm')
+      end
+
+      # Reset telemetry state (for testing)
+      #
+      # @return [void]
+      #
+      def reset!
+        @meter = nil
+        @job_counter = nil
+        @embedding_latency = nil
+        @tag_latency = nil
+        @search_latency = nil
+        @cache_operations = nil
+        @setup_complete = false
+        # Don't reset @sdk_available - that's a system property
+      end
+
+      # =========================================
+      # Instrument Accessors
+      # =========================================
+
+      # Counter for job execution (enqueued, completed, failed)
+      #
+      # @return [OpenTelemetry::Metrics::Counter, NullInstrument]
+      #
+      # @example Record a completed job
+      #   Telemetry.job_counter.add(1, attributes: { 'job' => 'embedding', 'status' => 'success' })
+      #
+      def job_counter
+        @job_counter ||= meter.create_counter(
+          'htm.jobs',
+          unit: 'count',
+          description: 'Job execution counts by type and status'
+        )
+      end
+
+      # Histogram for embedding generation latency
+      #
+      # @return [OpenTelemetry::Metrics::Histogram, NullInstrument]
+      #
+      # @example Record latency
+      #   Telemetry.embedding_latency.record(145, attributes: { 'provider' => 'ollama', 'status' => 'success' })
+      #
+      def embedding_latency
+        @embedding_latency ||= meter.create_histogram(
+          'htm.embedding.latency',
+          unit: 'ms',
+          description: 'Embedding generation latency in milliseconds'
+        )
+      end
+
+      # Histogram for tag extraction latency
+      #
+      # @return [OpenTelemetry::Metrics::Histogram, NullInstrument]
+      #
+      # @example Record latency
+      #   Telemetry.tag_latency.record(250, attributes: { 'provider' => 'ollama', 'status' => 'success' })
+      #
+      def tag_latency
+        @tag_latency ||= meter.create_histogram(
+          'htm.tag.latency',
+          unit: 'ms',
+          description: 'Tag extraction latency in milliseconds'
+        )
+      end
+
+      # Histogram for search operation latency
+      #
+      # @return [OpenTelemetry::Metrics::Histogram, NullInstrument]
+      #
+      # @example Record latency
+      #   Telemetry.search_latency.record(50, attributes: { 'strategy' => 'vector' })
+      #
+      def search_latency
+        @search_latency ||= meter.create_histogram(
+          'htm.search.latency',
+          unit: 'ms',
+          description: 'Search operation latency in milliseconds'
+        )
+      end
+
+      # Counter for cache operations (hits, misses)
+      #
+      # @return [OpenTelemetry::Metrics::Counter, NullInstrument]
+      #
+      # @example Record a cache hit
+      #   Telemetry.cache_operations.add(1, attributes: { 'operation' => 'hit' })
+      #
+      def cache_operations
+        @cache_operations ||= meter.create_counter(
+          'htm.cache.operations',
+          unit: 'count',
+          description: 'Cache hit/miss counts'
+        )
+      end
+
+      # =========================================
+      # Convenience Methods for Timing
+      # =========================================
+
+      # Measure execution time of a block and record to a histogram
+      #
+      # @param histogram [OpenTelemetry::Metrics::Histogram, NullInstrument] The histogram to record to
+      # @param attributes [Hash] Attributes to attach to the measurement
+      # @yield The block to measure
+      # @return [Object] The result of the block
+      #
+      # @example Measure embedding generation
+      #   result = Telemetry.measure(Telemetry.embedding_latency, 'provider' => 'ollama') do
+      #     generate_embedding(text)
+      #   end
+      #
+      def measure(histogram, attributes = {})
+        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        result = yield
+        elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round
+        histogram.record(elapsed_ms, attributes: attributes)
+        result
+      end
+    end
+  end
+end

data/lib/htm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class HTM
-  VERSION = "0.0.11"
+  VERSION = '0.0.14'
 end

data/lib/htm.rb

@@ -10,14 +10,17 @@ require_relative "htm/long_term_memory"
 require_relative "htm/working_memory"
 require_relative "htm/embedding_service"
 require_relative "htm/tag_service"
+require_relative "htm/proposition_service"
 require_relative "htm/timeframe_extractor"
 require_relative "htm/timeframe"
 require_relative "htm/job_adapter"
 require_relative "htm/jobs/generate_embedding_job"
 require_relative "htm/jobs/generate_tags_job"
-require_relative "htm/loaders/paragraph_chunker"
+require_relative "htm/jobs/generate_propositions_job"
+require_relative "htm/loaders/markdown_chunker"
 require_relative "htm/loaders/markdown_loader"
 require_relative "htm/observability"
+require_relative "htm/telemetry"
 
 require "pg"
 require "securerandom"
@@ -26,7 +29,7 @@ require "uri"
 # Load Rails integration if Rails is defined
 require_relative "htm/railtie" if defined?(Rails::Railtie)
 
-# HTM (Hierarchical Temporary Memory) - Intelligent memory management for LLM robots
+# HTM (Hierarchical Temporal Memory) - Intelligent memory management for LLM robots
 #
 # HTM implements a two-tier memory system:
 # - Working Memory: Token-limited, active context for immediate LLM use
@@ -163,6 +166,11 @@ class HTM
       # Only for NEW nodes - existing nodes already have embeddings/tags
       enqueue_embedding_job(node_id)
       enqueue_tags_job(node_id, manual_tags: tags)
+
+      # Enqueue proposition extraction if enabled and not already a proposition
+      if HTM.configuration.extract_propositions && !metadata[:is_proposition]
+        enqueue_propositions_job(node_id)
+      end
     else
       HTM.logger.info "Node #{node_id} already exists, linked to robot #{@robot_name} (remember_count: #{result[:robot_node].remember_count})"
 
@@ -228,7 +236,7 @@ class HTM
   #   memories = htm.recall("preferences", metadata: { source: "user" })
   #   memories = htm.recall("decisions", metadata: { confidence: 0.9, type: "architectural" })
   #
-  def recall(topic, timeframe: nil, limit: 20, strategy: :vector, with_relevance: false, query_tags: [], raw: false, metadata: {})
+  def recall(topic, timeframe: nil, limit: 20, strategy: :fulltext, with_relevance: false, query_tags: [], raw: false, metadata: {})
     # Validate inputs
     validate_timeframe!(timeframe)
     validate_positive_integer!(limit, "limit")
@@ -350,6 +358,51 @@ class HTM
     true
   end
 
+  # Forget all nodes whose content includes the given string
+  #
+  # Performs a soft delete on all matching nodes. The nodes remain in the
+  # database but are excluded from queries. Use case-insensitive LIKE matching.
+  #
+  # @param content_substring [String] Substring to search for in node content
+  # @param soft [Boolean] If true (default), soft delete; if false, permanent delete
+  # @param confirm [Symbol] Must be :confirmed to proceed with permanent deletion
+  # @return [Array<Integer>] Array of node IDs that were deleted
+  # @raise [ArgumentError] if content_substring is blank
+  # @raise [ArgumentError] if permanent deletion requested without confirmation
+  #
+  # @example Soft delete all nodes containing "deprecated"
+  #   htm.forget_content("deprecated")
+  #   # => [42, 56, 78]  # IDs of deleted nodes
+  #
+  # @example Permanent delete all nodes containing "test data"
+  #   htm.forget_content("test data", soft: false, confirm: :confirmed)
+  #
+  def forget_content(content_substring, soft: true, confirm: false)
+    raise ArgumentError, "Content substring cannot be blank" if content_substring.to_s.strip.empty?
+
+    # Permanent delete requires confirmation
+    if !soft && confirm != :confirmed
+      raise ArgumentError, "Permanent deletion requires confirm: :confirmed"
+    end
+
+    # Find all nodes containing the substring (case-insensitive)
+    matching_nodes = HTM::Models::Node.where("content ILIKE ?", "%#{content_substring}%")
+    node_ids = matching_nodes.pluck(:id)
+
+    if node_ids.empty?
+      HTM.logger.info "No nodes found containing: #{content_substring}"
+      return []
+    end
+
+    # Delete each matching node
+    node_ids.each do |node_id|
+      forget(node_id, soft: soft, confirm: confirm)
+    end
+
+    HTM.logger.info "Forgot #{node_ids.length} nodes containing: #{content_substring}"
+    node_ids
+  end
+
   # Restore a soft-deleted memory node
   #
   # @param node_id [Integer] ID of the soft-deleted node to restore
@@ -550,6 +603,14 @@ class HTM
     HTM.logger.error "Failed to enqueue tags job for node #{node_id}: #{e.message}"
   end
 
+  def enqueue_propositions_job(node_id)
+    # Enqueue proposition extraction using configured job backend
+    # Job will use HTM.extract_propositions which delegates to configured proposition_extractor
+    HTM::JobAdapter.enqueue(HTM::Jobs::GeneratePropositionsJob, node_id: node_id, robot_id: @robot_id)
+  rescue StandardError => e
+    HTM.logger.error "Failed to enqueue propositions job for node #{node_id}: #{e.message}"
+  end
+
   def add_to_working_memory(node)
     # Convert token_count to integer (may be String from database/cache)
     token_count = node['token_count'].to_i

data/lib/tasks/doc.rake

@@ -22,7 +22,7 @@ namespace :htm do
         "--plugin markdown",
         "--output-dir #{output_dir}",
         "--format markdown",
-        "--title 'HTM - Hierarchical Temporary Memory API'",
+        "--title 'HTM - Hierarchical Temporal Memory API'",
         "--markup markdown",
         "--charset utf-8",
         "--protected",