htm 0.0.31 → 0.0.32

data/lib/htm/long_term_memory/relevance_scorer.rb

@@ -58,21 +58,21 @@ class HTM
       def calculate_relevance(node:, query_tags: [], vector_similarity: nil, node_tags: nil)
         # 1. Vector similarity (semantic match)
         semantic_score = if vector_similarity
-          vector_similarity
-        elsif node['similarity']
-          node['similarity'].to_f
-        else
-          DEFAULT_NEUTRAL_SCORE  # Neutral if no embedding
-        end
+                           vector_similarity
+                         elsif node['similarity']
+                           node['similarity'].to_f
+                         else
+                           DEFAULT_NEUTRAL_SCORE  # Neutral if no embedding
+                         end
 
         # 2. Tag overlap (categorical relevance)
         # Use pre-loaded tags if provided, otherwise fetch (for backward compatibility)
         node_tags ||= get_node_tags(node['id'])
         tag_score = if query_tags.any? && node_tags.any?
-          weighted_hierarchical_jaccard(query_tags, node_tags)
-        else
-          DEFAULT_NEUTRAL_SCORE  # Neutral if no tags
-        end
+                      weighted_hierarchical_jaccard(query_tags, node_tags)
+                    else
+                      DEFAULT_NEUTRAL_SCORE  # Neutral if no tags
+                    end
 
         # 3. Recency (temporal relevance) - exponential decay with half-life
         age_hours = (Time.now - Time.parse(node['created_at'].to_s)) / 3600.0
@@ -108,16 +108,22 @@ class HTM
       def search_with_relevance(timeframe:, query: nil, query_tags: [], limit: 20, embedding_service: nil, metadata: {})
         # Get candidates from appropriate search method
         candidates = if query && embedding_service
-          # Vector search (returns hashes directly)
-          search_uncached(timeframe: timeframe, query: query, limit: limit * 2, embedding_service: embedding_service, metadata: metadata)
-        elsif query
-          # Full-text search (returns hashes directly)
-          search_fulltext_uncached(timeframe: timeframe, query: query, limit: limit * 2, metadata: metadata)
-        else
-          # Time-range only - use raw SQL to avoid ORM object instantiation
-          # This is more efficient than .map(&:attributes) which creates intermediate objects
-          fetch_candidates_by_timeframe(timeframe: timeframe, metadata: metadata, limit: limit * 2)
-        end
+                       # Vector search (returns hashes directly)
+                       search_uncached(timeframe: timeframe, query: query, limit: limit * 2, embedding_service: embedding_service,
+                                       metadata: metadata)
+                     elsif query
+                       # Full-text search (returns hashes directly)
+                       search_fulltext_uncached(timeframe: timeframe, query: query, limit: limit * 2, metadata: metadata)
+                     else
+                       # Time-range only - use raw SQL to avoid ORM object instantiation
+                       # This is more efficient than .map(&:attributes) which creates intermediate objects
+                       fetch_candidates_by_timeframe(timeframe: timeframe, metadata: metadata, limit: limit * 2)
+                     end
+
+        # Normalize similarity and text_rank to [0,1] across all candidates
+        # before scoring so weighted sum is unbiased (ts_rank is unbounded,
+        # similarity is already [0,1] but may be narrow)
+        normalize_scores_batch(candidates)
 
         # Batch load all tags for candidates (fixes N+1 query)
         node_ids = candidates.map { |n| n['id'] }
@@ -183,62 +189,49 @@ class HTM
       def search_by_tags(tags:, match_all: false, timeframe: nil, limit: 20)
         return [] if tags.empty?
 
-        # Build base query with specific columns to avoid loading unnecessary data
-        query = HTM::Models::Node
-          .select(
-            Sequel[:nodes][:id],
-            Sequel[:nodes][:content],
-            Sequel[:nodes][:access_count],
-            Sequel[:nodes][:created_at],
-            Sequel[:nodes][:token_count]
-          )
-          .join(:node_tags, node_id: :id)
-          .join(:tags, id: Sequel[:node_tags][:tag_id])
-          .where(Sequel[:tags][:name] => tags)
-          .distinct
-
-        # Apply timeframe filter if provided
-        query = query.where(Sequel[:nodes][:created_at] => timeframe) if timeframe
-
-        if match_all
-          # Match ALL tags (intersection)
-          query = query
-            .group(Sequel[:nodes][:id])
-            .having { Sequel.function(:count, Sequel[:tags][:name].distinct) =~ tags.size }
-        end
+        nodes = fetch_nodes_by_tags(tags, match_all: match_all, timeframe: timeframe, limit: limit)
+        enrich_nodes_with_relevance(nodes, query_tags: tags)
+      end
 
-        # Fetch and convert to hashes with string keys
-        nodes = query.limit(limit).all.map do |row|
-          {
-            'id' => row[:id],
-            'content' => row[:content],
-            'access_count' => row[:access_count],
-            'created_at' => row[:created_at],
-            'token_count' => row[:token_count]
-          }
+      private
+
+      def fetch_nodes_by_tags(tags, match_all:, timeframe:, limit:)
+        query = build_tag_base_query(tags, timeframe)
+        query = apply_match_all_constraint(query, tags) if match_all
+        query.limit(limit).all.map do |row|
+          { 'id' => row[:id], 'content' => row[:content],
+            'access_count' => row[:access_count], 'created_at' => row[:created_at], 'token_count' => row[:token_count] }
         end
+      end
 
-        # Batch load all tags for nodes (fixes N+1 query)
-        node_ids = nodes.map { |n| n['id'] }
-        tags_by_node = batch_load_node_tags(node_ids)
+      def build_tag_base_query(tags, timeframe)
+        cols  = [Sequel[:nodes][:id], Sequel[:nodes][:content], Sequel[:nodes][:access_count],
+                 Sequel[:nodes][:created_at], Sequel[:nodes][:token_count]]
+        query = HTM::Models::Node
+                .select(*cols)
+                .join(:node_tags, node_id: :id)
+                .join(:tags, id: Sequel[:node_tags][:tag_id])
+                .where(Sequel[:tags][:name] => tags)
+                .distinct
+        timeframe ? query.where(Sequel[:nodes][:created_at] => timeframe) : query
+      end
 
-        # Calculate relevance and enrich with tags (modify in-place)
-        nodes.map do |node|
-          node_tags = tags_by_node[node['id']] || []
-          relevance = calculate_relevance(
-            node: node,
-            query_tags: tags,
-            node_tags: node_tags
-          )
+      def apply_match_all_constraint(query, tags)
+        query.group(Sequel[:nodes][:id])
+             .having { Sequel.function(:count, Sequel[:tags][:name].distinct) =~ tags.size }
+      end
 
-          node['relevance'] = relevance
+      def enrich_nodes_with_relevance(nodes, query_tags:)
+        tags_by_node = batch_load_node_tags(nodes.map { |n| n['id'] })
+        enriched = nodes.map do |node|
+          node_tags = tags_by_node[node['id']] || []
+          node['relevance'] = calculate_relevance(node: node, query_tags: query_tags, node_tags: node_tags)
           node['tags'] = node_tags
           node
-        end.sort_by { |n| -n['relevance'] }
+        end
+        enriched.sort_by { |n| -n['relevance'] }
       end
 
-      private
-
       # Calculate Jaccard similarity between two sets
       #
       # @param set_a [Array] First set
@@ -309,7 +302,7 @@ class HTM
           end
         end
 
-        total_weights > 0 ? total_weighted_similarity / total_weights : 0.0
+        total_weights.positive? ? total_weighted_similarity / total_weights : 0.0
       end
 
       # Calculate similarity between two pre-split hierarchical tags
@@ -341,6 +334,45 @@ class HTM
         [similarity, depth_weight]
       end
 
+      # Min-max normalize signal columns across all candidates to [0, 1]
+      #
+      # Normalizes 'similarity' and 'text_rank' in-place so the weighted
+      # composite in calculate_relevance is not biased by different scales
+      # (ts_rank is unbounded, similarity is [0,1]).
+      #
+      # Handles edge cases:
+      # - Single element: no-op (already effectively normalized)
+      # - All-same values: maps to 1.0 (avoids division by zero)
+      # - Missing keys: skips normalization for that signal
+      #
+      # @param candidates [Array<Hash>] Candidate nodes (modified in-place)
+      # @return [Array<Hash>] Same array, normalized
+      #
+      def normalize_scores_batch(candidates)
+        return candidates if candidates.size <= 1
+
+        %w[similarity text_rank].each do |key|
+          values = candidates.filter_map { |c| c[key]&.to_f }
+          next if values.empty?
+
+          min_val = values.min
+          max_val = values.max
+          range = max_val - min_val
+
+          candidates.each do |c|
+            next unless c.key?(key) && c[key]
+
+            c[key] = if range.zero?
+                       1.0
+                     else
+                       (c[key].to_f - min_val) / range
+                     end
+          end
+        end
+
+        candidates
+      end
+
       # Calculate similarity between two hierarchical tags (string version)
       #
       # Compares tags level by level, returning both similarity and a weight

data/lib/htm/long_term_memory/tag_operations.rb

@@ -78,45 +78,10 @@ class HTM
       # - default - LIKE prefix match (e.g., "database" matches "database:postgresql")
       #
       def nodes_by_topic(topic_path, exact: false, fuzzy: false, min_similarity: DEFAULT_TAG_SIMILARITY_THRESHOLD, limit: 50)
-        # Enforce limit to prevent DoS
-        safe_limit = [[limit.to_i, 1].max, MAX_TAG_QUERY_LIMIT].min
-
-        # Build base query with joins
-        # Use subquery with DISTINCT ON to get unique nodes by id
-        if exact
-          node_ids = HTM::Models::Node
-            .select(Sequel[:nodes][:id])
-            .join(:node_tags, node_id: :id)
-            .join(:tags, id: Sequel[:node_tags][:tag_id])
-            .where(Sequel[:tags][:name] => topic_path)
-            .distinct
-            .select_map(Sequel[:nodes][:id])
-        elsif fuzzy
-          # Trigram similarity search - tolerates typos and partial matches
-          safe_similarity = [[min_similarity.to_f, 0.0].max, 1.0].min
-          node_ids = HTM::Models::Node
-            .select(Sequel[:nodes][:id])
-            .join(:node_tags, node_id: :id)
-            .join(:tags, id: Sequel[:node_tags][:tag_id])
-            .where(Sequel.lit("similarity(tags.name, ?) >= ?", topic_path, safe_similarity))
-            .distinct
-            .select_map(Sequel[:nodes][:id])
-        else
-          # Sanitize LIKE pattern to prevent wildcard injection
-          safe_pattern = HTM::SqlBuilder.sanitize_like_pattern(topic_path)
-          node_ids = HTM::Models::Node
-            .select(Sequel[:nodes][:id])
-            .join(:node_tags, node_id: :id)
-            .join(:tags, id: Sequel[:node_tags][:tag_id])
-            .where(Sequel.like(Sequel[:tags][:name], "#{safe_pattern}%"))
-            .distinct
-            .select_map(Sequel[:nodes][:id])
-        end
-
-        # Return empty array if no node_ids found
+        safe_limit = limit.to_i.clamp(1, MAX_TAG_QUERY_LIMIT)
+        node_ids   = node_ids_for_topic(topic_path, exact: exact, fuzzy: fuzzy, min_similarity: min_similarity)
         return [] if node_ids.empty?
 
-        # Fetch full node records for the matching ids
         HTM::Models::Node
           .where(id: node_ids)
           .order(Sequel.desc(:created_at))
@@ -143,7 +108,7 @@ class HTM
       #
       def topic_relationships(min_shared_nodes: 2, limit: 50)
         # Enforce limit to prevent DoS
-        safe_limit = [[limit.to_i, 1].max, MAX_TAG_QUERY_LIMIT].min
+        safe_limit = limit.to_i.clamp(1, MAX_TAG_QUERY_LIMIT)
         safe_min = [min_shared_nodes.to_i, 1].max
 
         sql = <<~SQL
@@ -200,9 +165,9 @@ class HTM
 
         # Single query to get all tags for all nodes
         results = HTM::Models::NodeTag
-          .join(:tags, id: :tag_id)
-          .where(node_id: node_ids)
-          .select_map([:node_id, Sequel[:tags][:name]])
+                  .join(:tags, id: :tag_id)
+                  .where(node_id: node_ids)
+                  .select_map([:node_id, Sequel[:tags][:name]])
 
         # Group by node_id
         results.group_by(&:first).transform_values { |pairs| pairs.map(&:last) }
@@ -218,25 +183,11 @@ class HTM
       # @return [Array<Hash>] Tags with usage counts
       #
       def popular_tags(limit: 20, timeframe: nil)
-        # Enforce limit to prevent DoS
-        safe_limit = [[limit.to_i, 1].max, MAX_TAG_QUERY_LIMIT].min
-
-        query = HTM::Models::Tag
-          .join(:node_tags, tag_id: :id)
-          .join(:nodes, id: Sequel[:node_tags][:node_id])
-          .group(Sequel[:tags][:id], Sequel[:tags][:name])
-          .select(Sequel[:tags][:name], Sequel.function(:count, Sequel[:node_tags][:id]).as(:usage_count))
-
-        if timeframe
-          query = query.where(Sequel[:nodes][:created_at] >= timeframe.begin)
-            .where(Sequel[:nodes][:created_at] <= timeframe.end)
-        end
-
-        query
-          .order(Sequel.desc(:usage_count))
-          .limit(safe_limit)
-          .all
-          .map { |tag| { name: tag[:name], usage_count: tag[:usage_count].to_i } }
+        safe_limit = limit.to_i.clamp(1, MAX_TAG_QUERY_LIMIT)
+        query = base_popular_tags_query
+        query = filter_by_timeframe(query, timeframe) if timeframe
+        query.order(Sequel.desc(:usage_count)).limit(safe_limit).all
+             .map { |tag| { name: tag[:name], usage_count: tag[:usage_count].to_i } }
       end
 
       # Fuzzy search for tags using trigram similarity
@@ -254,8 +205,8 @@ class HTM
         return [] if query.nil? || query.strip.empty?
 
         # Enforce limits
-        safe_limit = [[limit.to_i, 1].max, MAX_TAG_QUERY_LIMIT].min
-        safe_similarity = [[min_similarity.to_f, 0.0].max, 1.0].min
+        safe_limit = limit.to_i.clamp(1, MAX_TAG_QUERY_LIMIT)
+        safe_similarity = min_similarity.to_f.clamp(0.0, 1.0)
 
         sql = <<~SQL
           SELECT name, similarity(name, ?) as similarity
@@ -266,8 +217,8 @@ class HTM
         SQL
 
         HTM.db.fetch(sql, query, query, safe_similarity, safe_limit)
-          .all
-          .map { |r| { name: r[:name], similarity: r[:similarity].to_f } }
+           .all
+           .map { |r| { name: r[:name], similarity: r[:similarity].to_f } }
       rescue Sequel::Error => e
         HTM.logger.error("Failed to search tags: #{e.message}")
         []
@@ -366,76 +317,92 @@ class HTM
       # @param min_similarity [Float] Minimum similarity for trigram matching
       # @return [Array<String>] Matched tag names
       #
-      def find_matching_tags_unified(exact_candidates:, prefix_candidates:, component_candidates:, fuzzy_fallback: true, min_similarity: DEFAULT_TAG_SIMILARITY_THRESHOLD)
+      def find_matching_tags_unified(exact_candidates:, prefix_candidates:, component_candidates:, fuzzy_fallback: true,
+                                     min_similarity: DEFAULT_TAG_SIMILARITY_THRESHOLD)
         return [] if exact_candidates.empty? && prefix_candidates.empty? && component_candidates.empty?
 
         conditions = []
         params = []
+        append_exact_conditions(conditions, params, exact_candidates)
+        append_prefix_conditions(conditions, params, prefix_candidates)
+        append_component_conditions(conditions, params, component_candidates)
+        append_trigram_conditions(conditions, params, component_candidates, min_similarity) if fuzzy_fallback && component_candidates.any?
+        return [] if conditions.empty?
 
-        # Exact matches (highest priority)
-        # Use Sequel.lit with ? placeholders for proper parameter binding
-        if exact_candidates.any?
-          placeholders = exact_candidates.map { '?' }.join(', ')
-          conditions << "(SELECT name, 1 as priority FROM tags WHERE name IN (#{placeholders}))"
-          params.concat(exact_candidates)
-        end
-
-        # Prefix matches
-        if prefix_candidates.any?
-          placeholders = prefix_candidates.map { '?' }.join(', ')
-          conditions << "(SELECT name, 2 as priority FROM tags WHERE name IN (#{placeholders}))"
-          params.concat(prefix_candidates)
-        end
-
-        # Component matches
-        if component_candidates.any?
-          component_conditions = component_candidates.map do |_|
-            "(name = ? OR name LIKE ? OR name LIKE ? OR name LIKE ?)"
-          end
+        params << MAX_TAG_QUERY_LIMIT
+        sql = "SELECT DISTINCT name FROM (#{conditions.join(' UNION ')}) AS matches ORDER BY name LIMIT ?"
+        HTM.db.fetch(sql, *params).all.map { |r| r[:name] }
+      rescue Sequel::Error => e
+        HTM.logger.error("Failed to find matching tags: #{e.message}")
+        []
+      end
 
-          component_params = component_candidates.flat_map do |component|
-            safe_component = HTM::SqlBuilder.sanitize_like_pattern(component)
-            [
-              component,                 # exact match
-              "#{safe_component}:%",     # starts with
-              "%:#{safe_component}",     # ends with
-              "%:#{safe_component}:%"    # in middle
-            ]
-          end
+      def base_popular_tags_query
+        HTM::Models::Tag
+          .join(:node_tags, tag_id: :id)
+          .join(:nodes, id: Sequel[:node_tags][:node_id])
+          .group(Sequel[:tags][:id], Sequel[:tags][:name])
+          .select(Sequel[:tags][:name], Sequel.function(:count, Sequel[:node_tags][:id]).as(:usage_count))
+      end
 
-          conditions << "(SELECT name, 3 as priority FROM tags WHERE #{component_conditions.join(' OR ')})"
-          params.concat(component_params)
-        end
+      def filter_by_timeframe(query, timeframe)
+        query
+          .where(Sequel[:nodes][:created_at] >= timeframe.begin)
+          .where(Sequel[:nodes][:created_at] <= timeframe.end)
+      end
 
-        # Trigram fuzzy matches (lowest priority - fallback for typos)
-        if fuzzy_fallback && component_candidates.any?
-          safe_similarity = [[min_similarity.to_f, 0.0].max, 1.0].min
-          trigram_conditions = component_candidates.map do |_|
-            "similarity(name, ?) >= ?"
+      def node_ids_for_topic(topic_path, exact:, fuzzy:, min_similarity:)
+        base = HTM::Models::Node
+               .select(Sequel[:nodes][:id])
+               .join(:node_tags, node_id: :id)
+               .join(:tags, id: Sequel[:node_tags][:tag_id])
+               .distinct
+
+        node_ids_dataset =
+          if exact
+            base.where(Sequel[:tags][:name] => topic_path)
+          elsif fuzzy
+            safe_sim = min_similarity.to_f.clamp(0.0, 1.0)
+            base.where(Sequel.lit("similarity(tags.name, ?) >= ?", topic_path, safe_sim))
+          else
+            safe_pattern = HTM::SqlBuilder.sanitize_like_pattern(topic_path)
+            base.where(Sequel.like(Sequel[:tags][:name], "#{safe_pattern}%"))
           end
-          trigram_params = component_candidates.flat_map { |c| [c, safe_similarity] }
 
-          conditions << "(SELECT name, 4 as priority FROM tags WHERE #{trigram_conditions.join(' OR ')})"
-          params.concat(trigram_params)
-        end
+        node_ids_dataset.select_map(Sequel[:nodes][:id])
+      end
 
-        return [] if conditions.empty?
+      def append_exact_conditions(conditions, params, exact_candidates)
+        return unless exact_candidates.any?
+        placeholders = exact_candidates.map { '?' }.join(', ')
+        conditions << "(SELECT name, 1 as priority FROM tags WHERE name IN (#{placeholders}))"
+        params.concat(exact_candidates)
+      end
 
-        # Combine with UNION and order by priority
-        params << MAX_TAG_QUERY_LIMIT
+      def append_prefix_conditions(conditions, params, prefix_candidates)
+        return unless prefix_candidates.any?
+        placeholders = prefix_candidates.map { '?' }.join(', ')
+        conditions << "(SELECT name, 2 as priority FROM tags WHERE name IN (#{placeholders}))"
+        params.concat(prefix_candidates)
+      end
 
-        sql = <<~SQL
-          SELECT DISTINCT name FROM (
-            #{conditions.join(' UNION ')}
-          ) AS matches
-          ORDER BY name
-          LIMIT ?
-        SQL
+      def append_component_conditions(conditions, params, component_candidates)
+        return unless component_candidates.any?
+        component_conditions = component_candidates.map { "(name = ? OR name LIKE ? OR name LIKE ? OR name LIKE ?)" }
+        component_params = component_candidates.flat_map do |component|
+          safe = HTM::SqlBuilder.sanitize_like_pattern(component)
+          [component, "#{safe}:%", "%:#{safe}", "%:#{safe}:%"]
+        end
+        conditions << "(SELECT name, 3 as priority FROM tags WHERE #{component_conditions.join(' OR ')})"
+        params.concat(component_params)
+      end
 
-        HTM.db.fetch(sql, *params).all.map { |r| r[:name] }
-      rescue Sequel::Error => e
-        HTM.logger.error("Failed to find matching tags: #{e.message}")
-        []
+      def append_trigram_conditions(conditions, params, component_candidates, min_similarity)
+        safe_similarity = min_similarity.to_f.clamp(0.0, 1.0)
+        trigram_conditions = component_candidates.map { "similarity(name, ?) >= ?" }
+        trigram_params = component_candidates.flat_map { |c| [c, safe_similarity] }
+        conditions << "(SELECT name, 4 as priority FROM tags WHERE #{trigram_conditions.join(' OR ')})"
+        params.concat(trigram_params)
       end
     end
   end

data/lib/htm/long_term_memory/vector_search.rb

@@ -28,7 +28,7 @@ class HTM
       #
       def search(timeframe:, query:, limit:, embedding_service:, metadata: {})
         # Enforce limit to prevent DoS
-        safe_limit = [[limit.to_i, 1].max, MAX_VECTOR_LIMIT].min
+        safe_limit = limit.to_i.clamp(1, MAX_VECTOR_LIMIT)
 
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         result = @cache.fetch(:search, timeframe, query, safe_limit, metadata) do

data/lib/htm/long_term_memory.rb

@@ -86,7 +86,8 @@ class HTM
     # @example Disable caching
     #   ltm = LongTermMemory.new(config, cache_size: 0)
     #
-    def initialize(config, pool_size: nil, query_timeout: DEFAULT_QUERY_TIMEOUT, cache_size: DEFAULT_CACHE_SIZE, cache_ttl: DEFAULT_CACHE_TTL)
+    def initialize(config, pool_size: nil, query_timeout: DEFAULT_QUERY_TIMEOUT, cache_size: DEFAULT_CACHE_SIZE,
+                   cache_ttl: DEFAULT_CACHE_TTL)
       @config = config
       @query_timeout = query_timeout  # in milliseconds