noiseless 0.7.2 → 0.8.0

data/lib/noiseless/adapters/execution_modules/postgresql_execution.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "pgvector_support"
 
 module Noiseless
   module Adapters
@@ -10,61 +9,84 @@ module Noiseless
       # and optionally pgvector for semantic search
       module PostgresqlExecution
         include PgvectorSupport
-
-        SIMILARITY_THRESHOLD = 0.3
-        DEFAULT_LIMIT = 20
+        include PostgresqlQuery
 
         private
 
         def execute_search(query_hash, model_class: nil, **)
           model = resolve_model(query_hash[:indexes], model_class)
-          return empty_response unless model
+          raise Noiseless::SearchError, "no ActiveRecord model for index #{query_hash[:indexes]&.first.inspect}" unless model
+
+          with_search_errors do
+            next execute_vector_search(model, query_hash) if query_hash[:vector] || query_hash[:hybrid]
 
-          # Check if this is a vector search
-          return execute_vector_search(model, query_hash) if query_hash[:vector]
+            scope = build_search_scope(model, query_hash)
+            total = scope.except(:order, :limit, :offset).count
+            records = apply_pagination(scope, query_hash[:paginate]).to_a
 
-          scope = build_search_scope(model, query_hash)
-          total = scope.except(:order, :limit, :offset).count
-          records = apply_pagination(scope, query_hash[:paginate]).to_a
+            format_as_search_response(records, model, total: total)
+          end
+        end
 
-          format_as_search_response(records, model, total: total)
+        def with_search_errors
+          yield
+        rescue ActiveRecord::ConnectionNotEstablished => e
+          raise Noiseless::ConnectionError, "postgresql unreachable: #{e.message}"
+        rescue Noiseless::Error
+          raise
         rescue StandardError => e
-          error_response(e)
+          raise Noiseless::SearchError, "postgresql search failed: #{e.message}"
         end
 
         def execute_vector_search(model, query_hash)
-          vector_node = query_hash[:vector]
-          return empty_response unless vector_node && pgvector_available?
-
-          # Start with base scope
-          scope = model.all
-
-          # Apply any filters first
-          scope = apply_filter_clauses(scope, query_hash[:bool]&.filter || [], model)
-
-          # Apply vector search
-          scope = vector_search(
-            scope,
-            vector_node.embedding,
-            column: vector_node.field,
-            limit: vector_node.k,
-            distance_metric: vector_node.distance_metric
-          )
-
-          records = scope.to_a
-          format_vector_response(records, model, vector_node)
-        rescue StandardError => e
-          error_response(e)
+          raise Noiseless::SearchError, "vector search requires the pgvector extension" unless pgvector_available?
+
+          scope = apply_filter_clauses(model.all, query_hash[:bool]&.filter || [], model)
+          neighbors, k = nearest(scope, query_hash)
+
+          paginate_node = query_hash[:paginate]
+          return format_vector_response(neighbors.to_a, model) unless paginate_node
+
+          records = paginate_neighbors(neighbors, paginate_node, k).to_a
+          format_vector_response(records, model, total: [scope.count, k].min)
         end
 
-        def format_vector_response(records, model, _vector_node)
+        def nearest(scope, query_hash)
+          if (hybrid = query_hash[:hybrid])
+            vector = hybrid.vector
+            neighbors = hybrid_search(scope, text_query: hybrid.text_query, embedding: vector.embedding, text_fields: hybrid.fields,
+                                             vector_column: vector.field, text_weight: hybrid.text_weight,
+                                             vector_weight: hybrid.vector_weight, limit: vector.k)
+            return [neighbors, vector.k]
+          end
+
+          vector = query_hash[:vector]
+          [vector_search(scope, vector.embedding, column: vector.field, limit: vector.k, distance_metric: vector.distance_metric),
+           vector.k]
+        end
+
+        def paginate_neighbors(scope, paginate_node, k)
+          page = paginate_node.page || 1
+          per_page = paginate_node.per_page || DEFAULT_LIMIT
+          offset = (page - 1) * per_page
+          limit = (k - offset).clamp(0, per_page)
+          return scope.none if limit.zero?
+
+          scope.limit(limit).offset(offset)
+        end
+
+        def format_vector_response(records, model, total: records.size)
           hits = records.map do |record|
-            distance = record.respond_to?(:vector_distance) ? record.vector_distance : 0
+            score = if record.has_attribute?(:combined_score)
+                      record.combined_score
+                    else
+                      1.0 - (record.respond_to?(:vector_distance) ? record.vector_distance : 0)
+                    end
             {
               "_index" => model.table_name,
               "_id" => record.id.to_s,
-              "_score" => 1.0 - distance, # Convert distance to similarity score
-              "_source" => record.as_json(except: [:vector_distance])
+              "_score" => score,
+              "_source" => record.as_json(except: %w[vector_distance combined_score])
             }
           end
 
@@ -73,7 +95,7 @@ module Noiseless
             "timed_out" => false,
             "_shards" => { "total" => 1, "successful" => 1, "skipped" => 0, "failed" => 0 },
             "hits" => {
-              "total" => { "value" => hits.size, "relation" => "eq" },
+              "total" => { "value" => total, "relation" => "eq" },
               "max_score" => hits.first&.dig("_score"),
               "hits" => hits
             }
@@ -101,8 +123,6 @@ module Noiseless
         def execute_index_exists?(index_name)
           model = resolve_model([index_name])
           model.present? && model.table_exists?
-        rescue StandardError
-          false
         end
 
         # Document writes are no-ops: the table IS the index, so queries always
@@ -124,8 +144,6 @@ module Noiseless
         def execute_document_exists?(index, id)
           model = resolve_model([index])
           model&.exists?(id: id) || false
-        rescue StandardError
-          false
         end
 
         def execute_cluster_health(**)
@@ -144,203 +162,6 @@ module Noiseless
           }
         end
 
-        # Query building methods
-
-        def build_search_scope(model, query_hash)
-          scope = model.all
-
-          # Apply must clauses (full-text search)
-          scope = apply_must_clauses(scope, query_hash[:bool]&.must || [], model)
-
-          # Apply filter clauses (exact matches)
-          scope = apply_filter_clauses(scope, query_hash[:bool]&.filter || [], model)
-
-          # Apply sorting (pagination is applied by the caller, after counting)
-          apply_sorting(scope, query_hash[:sort] || [], model)
-        end
-
-        def apply_must_clauses(scope, must_nodes, model)
-          return scope if must_nodes.empty?
-
-          must_nodes.each do |node|
-            scope = case node
-                    when AST::Match
-                      apply_match(scope, node, model)
-                    when AST::MultiMatch
-                      apply_multi_match(scope, node, model)
-                    when AST::Wildcard
-                      apply_wildcard(scope, node, model)
-                    when AST::Range
-                      apply_range(scope, node, model)
-                    when AST::Prefix
-                      apply_prefix(scope, node, model)
-                    else
-                      scope
-                    end
-          end
-
-          scope
-        end
-
-        def apply_match(scope, node, model)
-          field = node.field.to_s
-          value = node.value.to_s
-
-          # Mapping-only fields (denormalized into a search index, not columns)
-          # cannot be satisfied here; fail closed rather than matching everything.
-          return scope.none unless column?(model, field)
-
-          # Use pg_trgm similarity for fuzzy matching, accent-insensitive when
-          # the unaccent extension is present.
-          if trgm_available? && text_column?(model, field)
-            scope.where(
-              "#{fuzzy_column(field)} % #{fuzzy_param} OR " \
-              "#{fuzzy_column(field)} ILIKE #{fuzzy_param}",
-              value,
-              "%#{sanitize_like(value)}%"
-            )
-          else
-            # Fallback to ILIKE
-            scope.where("#{quoted_column(field)} ILIKE ?", "%#{sanitize_like(value)}%")
-          end
-        end
-
-        def apply_multi_match(scope, node, model)
-          query = node.query.to_s
-          # Drop mapping-only fields; if none of the requested fields are real
-          # columns the query cannot be satisfied — fail closed.
-          fields = node.fields.map(&:to_s).select { |field| column?(model, field) }
-          return scope.none if fields.empty?
-
-          conditions = fields.map do |field|
-            if trgm_available? && text_column?(model, field)
-              "(#{fuzzy_column(field)} % #{fuzzy_param} OR " \
-                "#{fuzzy_column(field)} ILIKE #{fuzzy_param})"
-            else
-              "#{quoted_column(field)} ILIKE ?"
-            end
-          end
-
-          params = fields.flat_map do |field|
-            if trgm_available? && text_column?(model, field)
-              [query, "%#{sanitize_like(query)}%"]
-            else
-              ["%#{sanitize_like(query)}%"]
-            end
-          end
-
-          scope.where(conditions.join(" OR "), *params)
-        end
-
-        def apply_wildcard(scope, node, model = nil)
-          field = node.field.to_s
-          return scope.none if model && !column?(model, field)
-
-          # Convert OpenSearch wildcards to SQL: * -> %, ? -> _
-          pattern = node.value.to_s.tr("*", "%").tr("?", "_")
-
-          scope.where("#{quoted_column(field)} ILIKE ?", pattern)
-        end
-
-        def apply_range(scope, node, model = nil)
-          return scope.none if model && !column?(model, node.field.to_s)
-
-          field = quoted_column(node.field.to_s)
-
-          scope = scope.where("#{field} >= ?", node.gte) if node.gte
-          scope = scope.where("#{field} <= ?", node.lte) if node.lte
-          scope = scope.where("#{field} > ?", node.gt) if node.gt
-          scope = scope.where("#{field} < ?", node.lt) if node.lt
-
-          scope
-        end
-
-        def apply_prefix(scope, node, model = nil)
-          return scope.none if model && !column?(model, node.field.to_s)
-
-          scope.where("#{quoted_column(node.field.to_s)} ILIKE ?", "#{sanitize_like(node.value)}%")
-        end
-
-        def apply_filter_clauses(scope, filter_nodes, model = nil)
-          return scope if filter_nodes.empty?
-
-          filter_nodes.each do |node|
-            value = node.value
-
-            scope = if value.is_a?(Hash) && value[:geo_distance]
-                      apply_geo_filter(scope, node)
-                    elsif model && !column?(model, node.field.to_s)
-                      # A filter on a mapping-only field cannot be enforced;
-                      # silently dropping it would broaden results, so fail closed.
-                      scope.none
-                    elsif model && array_column?(model, node.field.to_s)
-                      apply_array_filter(scope, node.field.to_s, value, model)
-                    else
-                      scope.where(node.field => value)
-                    end
-          end
-
-          scope
-        end
-
-        def apply_array_filter(scope, field, value, model)
-          cast = "#{model.columns_hash[field].sql_type.sub(/\[\]\z/, '')}[]"
-          operator = value.is_a?(Array) ? "&&" : "@>"
-
-          scope.where("#{quoted_column(field)} #{operator} ARRAY[?]::#{cast}", value)
-        end
-
-        def apply_geo_filter(scope, node)
-          # Requires PostGIS
-          geo_config = node.value[:geo_distance]
-          distance = geo_config[:distance]
-          field = node.field.to_s
-
-          # Find the geo point in config
-          geo_point = geo_config.find { |_k, v| v.is_a?(Hash) && v[:lat] && v[:lon] }&.last
-          return scope unless geo_point
-
-          # Use PostGIS ST_DWithin for efficient geo filtering
-          scope.where(
-            "ST_DWithin(#{field}::geography, ST_SetSRID(ST_MakePoint(?, ?), 4326)::geography, ?)",
-            geo_point[:lon],
-            geo_point[:lat],
-            parse_distance(distance)
-          )
-        rescue StandardError
-          # If PostGIS not available, skip geo filter
-          scope
-        end
-
-        def apply_sorting(scope, sort_nodes, model = nil)
-          sorted_fields = []
-          order_clauses = sort_nodes.filter_map do |node|
-            field = node.field.to_s
-            # Sorting on a mapping-only field is cosmetic — drop it instead of
-            # erroring the whole query.
-            next if model && !column?(model, field)
-
-            sorted_fields << field
-            direction = node.direction.to_s.upcase == "DESC" ? "DESC" : "ASC"
-            "#{quoted_column(field)} #{direction}"
-          end
-
-          primary_key = model.respond_to?(:primary_key) ? model.primary_key : nil
-          order_clauses << "#{quoted_column(primary_key)} ASC" if primary_key && !sorted_fields.include?(primary_key.to_s)
-          return scope if order_clauses.empty?
-
-          scope.order(Arel.sql(order_clauses.join(", ")))
-        end
-
-        def apply_pagination(scope, paginate_node)
-          page = paginate_node&.page || 1
-          per_page = paginate_node&.per_page || DEFAULT_LIMIT
-
-          offset = (page - 1) * per_page
-
-          scope.limit(per_page).offset(offset)
-        end
-
         # Response formatting
 
         def format_as_search_response(records, model, total: records.size)
@@ -365,33 +186,6 @@ module Noiseless
           }
         end
 
-        def empty_response
-          {
-            "took" => 0,
-            "timed_out" => false,
-            "_shards" => { "total" => 1, "successful" => 1, "skipped" => 0, "failed" => 0 },
-            "hits" => {
-              "total" => { "value" => 0, "relation" => "eq" },
-              "max_score" => nil,
-              "hits" => []
-            }
-          }
-        end
-
-        def error_response(error)
-          {
-            "took" => 0,
-            "timed_out" => false,
-            "_shards" => { "total" => 1, "successful" => 0, "skipped" => 0, "failed" => 1 },
-            "hits" => {
-              "total" => { "value" => 0, "relation" => "eq" },
-              "max_score" => nil,
-              "hits" => []
-            },
-            "error" => { "type" => error.class.name, "reason" => error.message }
-          }
-        end
-
         # Helper methods
 
         def resolve_model(indexes, model_class = nil)
@@ -406,69 +200,14 @@ module Noiseless
           # Try cached model first (populated via register_model)
           return @model_class_cache[index_name] if @model_class_cache&.key?(index_name)
 
-          # Try to infer model from index name
-          model_name = index_name.to_s.classify
-          model_name.constantize
-        rescue NameError
-          nil
+          candidate = index_name.to_s.classify.safe_constantize
+          candidate if active_record_model?(candidate)
         end
 
         def active_record_model?(klass)
           klass.is_a?(Class) && klass < ActiveRecord::Base
         end
 
-        def trgm_available?
-          @trgm_available ||= available_extensions.include?("pg_trgm")
-        end
-
-        def unaccent_available?
-          @unaccent_available ||= available_extensions.include?("unaccent")
-        end
-
-        def column?(model, field)
-          model.columns_hash.key?(field.to_s)
-        end
-
-        def text_column?(model, field)
-          column = model.columns_hash[field.to_s]
-          column && %i[string text citext].include?(column.type) && !column.array
-        end
-
-        def array_column?(model, field)
-          model.columns_hash[field.to_s]&.array
-        end
-
-        def fuzzy_column(field)
-          unaccent_available? ? "unaccent(#{quoted_column(field)})" : quoted_column(field)
-        end
-
-        def fuzzy_param
-          unaccent_available? ? "unaccent(?)" : "?"
-        end
-
-        def quoted_column(field)
-          ActiveRecord::Base.connection.quote_column_name(field)
-        end
-
-        def sanitize_like(value)
-          # Escape special LIKE characters
-          value.to_s.gsub(/[%_\\]/) { |x| "\\#{x}" }
-        end
-
-        def parse_distance(distance)
-          # Parse OpenSearch distance format (e.g., "10km", "5mi")
-          case distance.to_s
-          when /(\d+(?:\.\d+)?)\s*km/i
-            ::Regexp.last_match(1).to_f * 1000
-          when /(\d+(?:\.\d+)?)\s*mi/i
-            ::Regexp.last_match(1).to_f * 1609.34
-          when /(\d+(?:\.\d+)?)\s*m/i
-            ::Regexp.last_match(1).to_f
-          else
-            distance.to_f
-          end
-        end
-
         def process_bulk_action(action)
           if action[:index]
             { "index" => execute_index_document(action[:index][:_index], action[:index][:_id], nil) }