noiseless 0.0.0 → 0.1.0

data/lib/noiseless/adapters/execution_modules/pgvector_support.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module Adapters
+    module ExecutionModules
+      # pgvector support for semantic/vector search in PostgreSQL
+      # Provides similarity search using embeddings
+      #
+      # Required:
+      #   CREATE EXTENSION IF NOT EXISTS vector;
+      #
+      # Table setup:
+      #   ALTER TABLE your_table ADD COLUMN embedding vector(1536);
+      #   CREATE INDEX ON your_table USING ivfflat (embedding vector_cosine_ops);
+      #
+      module PgvectorSupport
+        # Perform semantic search using vector similarity
+        #
+        # @param scope [ActiveRecord::Relation] The base scope to search
+        # @param embedding [Array<Float>] The query embedding vector
+        # @param column [Symbol] The column containing embeddings (default: :embedding)
+        # @param limit [Integer] Maximum results to return
+        # @param distance_threshold [Float] Maximum distance threshold (optional)
+        # @param distance_metric [Symbol] :cosine, :l2, or :inner_product
+        # @return [ActiveRecord::Relation] Scope with vector similarity ordering
+        #
+        def vector_search(scope, embedding, column: :embedding, limit: 20, distance_threshold: nil,
+                          distance_metric: :cosine)
+          return scope unless pgvector_available?
+
+          vector_string = "[#{embedding.join(',')}]"
+          distance_op = distance_operator(distance_metric)
+
+          # Build the query with distance calculation
+          scope = scope.select(
+            "#{scope.table_name}.*",
+            "#{quoted_column(column)} #{distance_op} '#{vector_string}' AS vector_distance"
+          )
+
+          # Apply distance threshold if specified
+          if distance_threshold
+            scope = scope.where(
+              "#{quoted_column(column)} #{distance_op} '#{vector_string}' < ?",
+              distance_threshold
+            )
+          end
+
+          # Order by similarity (ascending distance = more similar)
+          scope.order(Arel.sql("#{quoted_column(column)} #{distance_op} '#{vector_string}'"))
+               .limit(limit)
+        end
+
+        # Hybrid search combining text and vector search
+        #
+        # @param scope [ActiveRecord::Relation] Base scope
+        # @param text_query [String] Text query for pg_trgm search
+        # @param embedding [Array<Float>] Query embedding for vector search
+        # @param text_fields [Array<Symbol>] Fields to search with text
+        # @param vector_column [Symbol] Column containing embeddings
+        # @param text_weight [Float] Weight for text similarity (0.0-1.0)
+        # @param vector_weight [Float] Weight for vector similarity (0.0-1.0)
+        # @return [ActiveRecord::Relation]
+        #
+        def hybrid_search(scope, text_query:, embedding:, text_fields:, vector_column: :embedding,
+                          text_weight: 0.5, vector_weight: 0.5, limit: 20)
+          return scope unless pgvector_available?
+
+          vector_string = "[#{embedding.join(',')}]"
+          text_conditions = text_fields.map { |f| "similarity(#{quoted_column(f)}, ?)" }.join(" + ")
+          text_similarity_count = text_fields.size
+
+          # Normalized combined score
+          scope.select(
+            "#{scope.table_name}.*",
+            # Text similarity (0-1 per field, averaged)
+            Arel.sql(
+              "(#{text_conditions}) / #{text_similarity_count} * #{text_weight} AS text_score"
+            ),
+            # Vector similarity (convert distance to similarity: 1 - distance for cosine)
+            "(1 - (#{quoted_column(vector_column)} <=> '#{vector_string}')) * #{vector_weight} AS vector_score",
+            # Combined score
+            "(((#{text_conditions}) / #{text_similarity_count}) * #{text_weight} + " \
+            "(1 - (#{quoted_column(vector_column)} <=> '#{vector_string}')) * #{vector_weight}) AS combined_score"
+          ).where(
+            "#{text_conditions} > 0 OR #{quoted_column(vector_column)} IS NOT NULL",
+            *Array.new(text_similarity_count, text_query)
+          ).order(Arel.sql("combined_score DESC"))
+               .limit(limit)
+               .tap { |s| s.bind_values.concat(Array.new(text_similarity_count, text_query)) }
+        end
+
+        # Execute a KNN (K-Nearest Neighbors) search
+        #
+        # @param model [Class] The ActiveRecord model
+        # @param embedding [Array<Float>] Query embedding
+        # @param k [Integer] Number of nearest neighbors
+        # @param column [Symbol] Embedding column
+        # @param filters [Hash] Additional WHERE conditions
+        # @return [Array<Hash>] Results with distance scores
+        #
+        def knn_search(model, embedding, k: 10, column: :embedding, filters: {})
+          return [] unless pgvector_available?
+
+          vector_string = "[#{embedding.join(',')}]"
+
+          scope = model.all
+          scope = scope.where(filters) if filters.any?
+
+          results = scope.select(
+            "#{model.table_name}.*",
+            "#{quoted_column(column)} <=> '#{vector_string}' AS distance"
+          ).order(Arel.sql("#{quoted_column(column)} <=> '#{vector_string}'"))
+                         .limit(k)
+
+          format_knn_response(results, model)
+        end
+
+        # Store an embedding for a record
+        #
+        # @param record [ActiveRecord::Base] The record to update
+        # @param embedding [Array<Float>] The embedding vector
+        # @param column [Symbol] The column to store the embedding
+        #
+        def store_embedding(record, embedding, column: :embedding)
+          return false unless pgvector_available?
+
+          vector_string = "[#{embedding.join(',')}]"
+          record.update_column(column, vector_string)
+        end
+
+        # Batch store embeddings
+        #
+        # @param model [Class] The ActiveRecord model
+        # @param embeddings [Hash<String, Array<Float>>] Map of ID -> embedding
+        # @param column [Symbol] The column to store embeddings
+        #
+        def batch_store_embeddings(model, embeddings, column: :embedding)
+          return 0 unless pgvector_available?
+
+          # Use UPDATE FROM VALUES for efficient batch update
+          values = embeddings.map do |id, emb|
+            "(#{ActiveRecord::Base.connection.quote(id)}, '[#{emb.join(',')}]'::vector)"
+          end.join(",")
+
+          sql = <<~SQL.squish
+            UPDATE #{model.table_name}
+            SET #{column} = v.embedding
+            FROM (VALUES #{values}) AS v(id, embedding)
+            WHERE #{model.table_name}.id = v.id::uuid
+          SQL
+
+          ActiveRecord::Base.connection.execute(sql)
+          embeddings.size
+        rescue StandardError => e
+          Rails.logger.error("Failed to batch store embeddings: #{e.message}")
+          0
+        end
+
+        # Find similar records to a given record
+        #
+        # @param record [ActiveRecord::Base] The reference record
+        # @param limit [Integer] Number of similar records
+        # @param column [Symbol] Embedding column
+        # @param exclude_self [Boolean] Exclude the reference record
+        # @return [ActiveRecord::Relation]
+        #
+        def find_similar(record, limit: 10, column: :embedding, exclude_self: true)
+          embedding = record.send(column)
+          return record.class.none unless embedding && pgvector_available?
+
+          scope = record.class.where.not(column => nil)
+          scope = scope.where.not(id: record.id) if exclude_self
+
+          vector_search(scope, embedding, column: column, limit: limit)
+        end
+
+        # Check if pgvector is available
+        def pgvector_available?
+          @pgvector_available ||= available_extensions.include?("vector")
+        end
+
+        private
+
+        def distance_operator(metric)
+          case metric
+          when :l2, :euclidean
+            "<->"  # L2/Euclidean distance
+          when :inner_product
+            "<#>"  # Negative inner product
+          else
+            "<=>"  # Cosine distance (default)
+          end
+        end
+
+        def format_knn_response(records, model)
+          hits = records.map do |record|
+            {
+              "_index" => model.table_name,
+              "_id" => record.id.to_s,
+              "_score" => 1.0 - (record.respond_to?(:distance) ? record.distance : 0),
+              "_source" => record.as_json(except: [:distance])
+            }
+          end
+
+          {
+            "took" => 0,
+            "timed_out" => false,
+            "_shards" => { "total" => 1, "successful" => 1, "skipped" => 0, "failed" => 0 },
+            "hits" => {
+              "total" => { "value" => hits.size, "relation" => "eq" },
+              "max_score" => hits.first&.dig("_score"),
+              "hits" => hits
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/noiseless/adapters/execution_modules/postgresql_execution.rb

@@ -0,0 +1,461 @@
+# frozen_string_literal: true
+
+require_relative "pgvector_support"
+
+module Noiseless
+  module Adapters
+    module ExecutionModules
+      # PostgreSQL execution module - translates noiseless AST to PostgreSQL queries
+      # Uses pg_trgm for fuzzy matching, unaccent for accent-insensitive search,
+      # and optionally pgvector for semantic search
+      module PostgresqlExecution
+        include PgvectorSupport
+
+        SIMILARITY_THRESHOLD = 0.3
+        DEFAULT_LIMIT = 20
+
+        private
+
+        def execute_search(query_hash, model_class: nil, **)
+          model = resolve_model(query_hash[:indexes], model_class)
+          return empty_response unless model
+
+          # Check if this is a vector search
+          return execute_vector_search(model, query_hash) if query_hash[:vector]
+
+          scope = build_search_scope(model, query_hash)
+          records = scope.to_a
+
+          format_as_search_response(records, model)
+        rescue StandardError => e
+          error_response(e)
+        end
+
+        def execute_vector_search(model, query_hash)
+          vector_node = query_hash[:vector]
+          return empty_response unless vector_node && pgvector_available?
+
+          # Start with base scope
+          scope = model.all
+
+          # Apply any filters first
+          scope = apply_filter_clauses(scope, query_hash[:bool]&.filter || [])
+
+          # Apply vector search
+          scope = vector_search(
+            scope,
+            vector_node.embedding,
+            column: vector_node.field,
+            limit: vector_node.k,
+            distance_metric: vector_node.distance_metric
+          )
+
+          records = scope.to_a
+          format_vector_response(records, model, vector_node)
+        rescue StandardError => e
+          error_response(e)
+        end
+
+        def format_vector_response(records, model, _vector_node)
+          hits = records.map do |record|
+            distance = record.respond_to?(:vector_distance) ? record.vector_distance : 0
+            {
+              "_index" => model.table_name,
+              "_id" => record.id.to_s,
+              "_score" => 1.0 - distance, # Convert distance to similarity score
+              "_source" => record.as_json(except: [:vector_distance])
+            }
+          end
+
+          {
+            "took" => 0,
+            "timed_out" => false,
+            "_shards" => { "total" => 1, "successful" => 1, "skipped" => 0, "failed" => 0 },
+            "hits" => {
+              "total" => { "value" => hits.size, "relation" => "eq" },
+              "max_score" => hits.first&.dig("_score"),
+              "hits" => hits
+            }
+          }
+        end
+
+        def execute_bulk(actions, **)
+          results = actions.map do |action|
+            process_bulk_action(action)
+          end
+
+          { "items" => results, "errors" => results.any? { |r| r["error"] } }
+        end
+
+        def execute_create_index(_index_name, **)
+          # No-op for PostgreSQL - tables already exist
+          { "acknowledged" => true }
+        end
+
+        def execute_delete_index(_index_name, **)
+          # No-op - we don't delete tables via search adapter
+          { "acknowledged" => true }
+        end
+
+        def execute_index_exists?(index_name)
+          model = resolve_model([index_name])
+          model.present? && model.table_exists?
+        rescue StandardError
+          false
+        end
+
+        def execute_index_document(index, id, document, **)
+          model = resolve_model([index])
+          return { "_id" => id, "result" => "error", "error" => "Model not found" } unless model
+
+          record = model.find_or_initialize_by(id: id)
+          record.assign_attributes(document.slice(*model.column_names))
+          record.save!
+
+          { "_index" => index, "_id" => id, "result" => record.previously_new_record? ? "created" : "updated" }
+        rescue StandardError => e
+          { "_index" => index, "_id" => id, "result" => "error", "error" => e.message }
+        end
+
+        def execute_update_document(index, id, changes, **)
+          model = resolve_model([index])
+          return { "_id" => id, "result" => "error", "error" => "Model not found" } unless model
+
+          record = model.find(id)
+          record.update!(changes.slice(*model.column_names))
+
+          { "_index" => index, "_id" => id, "result" => "updated" }
+        rescue ActiveRecord::RecordNotFound
+          { "_index" => index, "_id" => id, "result" => "not_found" }
+        rescue StandardError => e
+          { "_index" => index, "_id" => id, "result" => "error", "error" => e.message }
+        end
+
+        def execute_delete_document(index, id, **)
+          model = resolve_model([index])
+          return { "_id" => id, "result" => "error", "error" => "Model not found" } unless model
+
+          model.destroy(id)
+          { "_index" => index, "_id" => id, "result" => "deleted" }
+        rescue ActiveRecord::RecordNotFound
+          { "_index" => index, "_id" => id, "result" => "not_found" }
+        rescue StandardError => e
+          { "_index" => index, "_id" => id, "result" => "error", "error" => e.message }
+        end
+
+        def execute_document_exists?(index, id)
+          model = resolve_model([index])
+          model&.exists?(id: id) || false
+        rescue StandardError
+          false
+        end
+
+        def execute_cluster_health(**)
+          # Verify PostgreSQL connection
+          ActiveRecord::Base.connection.execute("SELECT 1")
+          {
+            "cluster_name" => "postgresql",
+            "status" => "green",
+            "number_of_nodes" => 1
+          }
+        rescue StandardError => e
+          {
+            "cluster_name" => "postgresql",
+            "status" => "red",
+            "error" => e.message
+          }
+        end
+
+        # Query building methods
+
+        def build_search_scope(model, query_hash)
+          scope = model.all
+
+          # Apply must clauses (full-text search)
+          scope = apply_must_clauses(scope, query_hash[:bool]&.must || [], model)
+
+          # Apply filter clauses (exact matches)
+          scope = apply_filter_clauses(scope, query_hash[:bool]&.filter || [])
+
+          # Apply sorting
+          scope = apply_sorting(scope, query_hash[:sort] || [])
+
+          # Apply pagination
+          apply_pagination(scope, query_hash[:paginate])
+        end
+
+        def apply_must_clauses(scope, must_nodes, model)
+          return scope if must_nodes.empty?
+
+          must_nodes.each do |node|
+            scope = case node
+                    when AST::Match
+                      apply_match(scope, node, model)
+                    when AST::MultiMatch
+                      apply_multi_match(scope, node, model)
+                    when AST::Wildcard
+                      apply_wildcard(scope, node)
+                    when AST::Range
+                      apply_range(scope, node)
+                    when AST::Prefix
+                      apply_prefix(scope, node)
+                    else
+                      scope
+                    end
+          end
+
+          scope
+        end
+
+        def apply_match(scope, node, model)
+          field = node.field.to_s
+          value = node.value.to_s
+
+          # Use pg_trgm similarity for fuzzy matching with unaccent
+          if trgm_available? && text_column?(model, field)
+            scope.where(
+              "unaccent(#{quoted_column(field)}) % unaccent(?) OR " \
+              "unaccent(#{quoted_column(field)}) ILIKE unaccent(?)",
+              value,
+              "%#{sanitize_like(value)}%"
+            )
+          else
+            # Fallback to ILIKE
+            scope.where("#{quoted_column(field)} ILIKE ?", "%#{sanitize_like(value)}%")
+          end
+        end
+
+        def apply_multi_match(scope, node, model)
+          query = node.query.to_s
+          fields = node.fields.map(&:to_s)
+
+          conditions = fields.map do |field|
+            if trgm_available? && text_column?(model, field)
+              "(unaccent(#{quoted_column(field)}) % unaccent(?) OR " \
+                "unaccent(#{quoted_column(field)}) ILIKE unaccent(?))"
+            else
+              "#{quoted_column(field)} ILIKE ?"
+            end
+          end
+
+          params = fields.flat_map do |field|
+            if trgm_available? && text_column?(model, field)
+              [query, "%#{sanitize_like(query)}%"]
+            else
+              ["%#{sanitize_like(query)}%"]
+            end
+          end
+
+          scope.where(conditions.join(" OR "), *params)
+        end
+
+        def apply_wildcard(scope, node)
+          field = node.field.to_s
+          # Convert OpenSearch wildcards to SQL: * -> %, ? -> _
+          pattern = node.value.to_s.tr("*", "%").tr("?", "_")
+
+          scope.where("#{quoted_column(field)} ILIKE ?", pattern)
+        end
+
+        def apply_range(scope, node)
+          field = quoted_column(node.field.to_s)
+
+          scope = scope.where("#{field} >= ?", node.gte) if node.gte
+          scope = scope.where("#{field} <= ?", node.lte) if node.lte
+          scope = scope.where("#{field} > ?", node.gt) if node.gt
+          scope = scope.where("#{field} < ?", node.lt) if node.lt
+
+          scope
+        end
+
+        def apply_prefix(scope, node)
+          scope.where("#{quoted_column(node.field.to_s)} ILIKE ?", "#{sanitize_like(node.value)}%")
+        end
+
+        def apply_filter_clauses(scope, filter_nodes)
+          return scope if filter_nodes.empty?
+
+          filter_nodes.each do |node|
+            value = node.value
+
+            scope = if value.is_a?(Hash) && value[:geo_distance]
+                      apply_geo_filter(scope, node)
+                    else
+                      scope.where(node.field => value)
+                    end
+          end
+
+          scope
+        end
+
+        def apply_geo_filter(scope, node)
+          # Requires PostGIS
+          geo_config = node.value[:geo_distance]
+          distance = geo_config[:distance]
+          field = node.field.to_s
+
+          # Find the geo point in config
+          geo_point = geo_config.find { |_k, v| v.is_a?(Hash) && v[:lat] && v[:lon] }&.last
+          return scope unless geo_point
+
+          # Use PostGIS ST_DWithin for efficient geo filtering
+          scope.where(
+            "ST_DWithin(#{field}::geography, ST_SetSRID(ST_MakePoint(?, ?), 4326)::geography, ?)",
+            geo_point[:lon],
+            geo_point[:lat],
+            parse_distance(distance)
+          )
+        rescue StandardError
+          # If PostGIS not available, skip geo filter
+          scope
+        end
+
+        def apply_sorting(scope, sort_nodes)
+          return scope if sort_nodes.empty?
+
+          order_clauses = sort_nodes.map do |node|
+            direction = node.direction.to_s.upcase == "DESC" ? "DESC" : "ASC"
+            "#{quoted_column(node.field.to_s)} #{direction}"
+          end
+
+          scope.order(Arel.sql(order_clauses.join(", ")))
+        end
+
+        def apply_pagination(scope, paginate_node)
+          page = paginate_node&.page || 1
+          per_page = paginate_node&.per_page || DEFAULT_LIMIT
+
+          offset = (page - 1) * per_page
+
+          scope.limit(per_page).offset(offset)
+        end
+
+        # Response formatting
+
+        def format_as_search_response(records, model)
+          total = records.size
+
+          hits = records.map do |record|
+            {
+              "_index" => model.table_name,
+              "_id" => record.id.to_s,
+              "_score" => 1.0,
+              "_source" => record.as_json
+            }
+          end
+
+          {
+            "took" => 0,
+            "timed_out" => false,
+            "_shards" => { "total" => 1, "successful" => 1, "skipped" => 0, "failed" => 0 },
+            "hits" => {
+              "total" => { "value" => total, "relation" => "eq" },
+              "max_score" => hits.any? ? 1.0 : nil,
+              "hits" => hits
+            }
+          }
+        end
+
+        def empty_response
+          {
+            "took" => 0,
+            "timed_out" => false,
+            "_shards" => { "total" => 1, "successful" => 1, "skipped" => 0, "failed" => 0 },
+            "hits" => {
+              "total" => { "value" => 0, "relation" => "eq" },
+              "max_score" => nil,
+              "hits" => []
+            }
+          }
+        end
+
+        def error_response(error)
+          {
+            "took" => 0,
+            "timed_out" => false,
+            "_shards" => { "total" => 1, "successful" => 0, "skipped" => 0, "failed" => 1 },
+            "hits" => {
+              "total" => { "value" => 0, "relation" => "eq" },
+              "max_score" => nil,
+              "hits" => []
+            },
+            "error" => { "type" => error.class.name, "reason" => error.message }
+          }
+        end
+
+        # Helper methods
+
+        def resolve_model(indexes, model_class = nil)
+          return model_class if model_class
+
+          index_name = indexes&.first
+          return nil unless index_name
+
+          # Try cached model first
+          return @model_class_cache[index_name] if @model_class_cache&.key?(index_name)
+
+          # Try to infer model from index name
+          model_name = index_name.to_s.classify
+          model_name.constantize
+        rescue NameError
+          nil
+        end
+
+        def trgm_available?
+          @trgm_available ||= available_extensions.include?("pg_trgm")
+        end
+
+        def unaccent_available?
+          @unaccent_available ||= available_extensions.include?("unaccent")
+        end
+
+        def text_column?(model, field)
+          column = model.columns_hash[field.to_s]
+          column && %i[string text].include?(column.type)
+        end
+
+        def quoted_column(field)
+          ActiveRecord::Base.connection.quote_column_name(field)
+        end
+
+        def sanitize_like(value)
+          # Escape special LIKE characters
+          value.to_s.gsub(/[%_\\]/) { |x| "\\#{x}" }
+        end
+
+        def parse_distance(distance)
+          # Parse OpenSearch distance format (e.g., "10km", "5mi")
+          case distance.to_s
+          when /(\d+(?:\.\d+)?)\s*km/i
+            ::Regexp.last_match(1).to_f * 1000
+          when /(\d+(?:\.\d+)?)\s*mi/i
+            ::Regexp.last_match(1).to_f * 1609.34
+          when /(\d+(?:\.\d+)?)\s*m/i
+            ::Regexp.last_match(1).to_f
+          else
+            distance.to_f
+          end
+        end
+
+        def process_bulk_action(action)
+          if action[:index]
+            index = action[:index][:_index]
+            id = action[:index][:_id]
+            data = action[:index][:data]
+
+            result = execute_index_document(index, id, data)
+            { "index" => result }
+          elsif action[:delete]
+            index = action[:delete][:_index]
+            id = action[:delete][:_id]
+
+            result = execute_delete_document(index, id)
+            { "delete" => result }
+          else
+            { "error" => "Unknown action type" }
+          end
+        end
+      end
+    end
+  end
+end