noiseless 0.0.0 → 0.2.0

data/lib/noiseless/adapters/execution_modules/elasticsearch_execution.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "es_compatible_execution"
+
+module Noiseless
+  module Adapters
+    module ExecutionModules
+      module ElasticsearchExecution
+        include EsCompatibleExecution
+
+        private
+
+        def execute_search(query_hash, indexes: [], **_opts)
+          path = indexes.any? ? "/#{indexes.join(',')}/_search" : "/_search"
+          body = JSON.generate(query_hash)
+
+          response = post_request(path, body)
+          parse_json_response!(response, error_class: Noiseless::SearchError, context: "search")
+        ensure
+          response&.close
+        end
+
+        def execute_create_index(index_name, mappings: nil, settings: nil, **_opts)
+          body = {}
+          body[:mappings] = mappings if mappings
+          body[:settings] = settings if settings
+
+          response = put_request("/#{index_name}", body.any? ? JSON.generate(body) : nil)
+          parse_json_response!(response, context: "create index #{index_name}")
+        ensure
+          response&.close
+        end
+
+        def execute_index_document(index, id, document, **_opts)
+          path = id ? "/#{index}/_doc/#{id}" : "/#{index}/_doc"
+          body = JSON.generate(document)
+
+          response = id ? put_request(path, body) : post_request(path, body)
+          parse_json_response!(response, context: "index document #{index}/#{id}")
+        ensure
+          response&.close
+        end
+
+        def execute_cluster_health(**_opts)
+          response = get_request("/_cluster/health")
+          JSON.parse(response.read)
+        rescue StandardError => e
+          {
+            "cluster_name" => "unknown",
+            "status" => "red",
+            "timed_out" => false,
+            "number_of_nodes" => 0,
+            "number_of_data_nodes" => 0,
+            "active_primary_shards" => 0,
+            "active_shards" => 0,
+            "error" => {
+              "type" => e.class.name,
+              "reason" => e.message
+            }
+          }
+        ensure
+          response&.close
+        end
+      end
+    end
+  end
+end

data/lib/noiseless/adapters/execution_modules/es_compatible_execution.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "http_transport"
+
+module Noiseless
+  module Adapters
+    module ExecutionModules
+      # Document and index operations shared by the wire-compatible
+      # Elasticsearch and OpenSearch HTTP APIs.
+      module EsCompatibleExecution
+        include HttpTransport
+
+        private
+
+        def execute_bulk(actions, **_opts)
+          body = actions.map do |action|
+            if action[:index]
+              action_line = { index: { _index: action[:index][:_index], _id: action[:index][:_id] } }
+              data_line = action[:index][:data]
+              "#{JSON.generate(action_line)}\n#{JSON.generate(data_line)}\n"
+            else
+              "#{JSON.generate(action)}\n"
+            end
+          end.join
+
+          response = post_request("/_bulk", body, content_type: "application/x-ndjson")
+          parse_json_response!(response, context: "bulk")
+        ensure
+          response&.close
+        end
+
+        def execute_delete_index(index_name, **_opts)
+          response = delete_request("/#{index_name}")
+          parse_json_response!(response, context: "delete index #{index_name}")
+        ensure
+          response&.close
+        end
+
+        def execute_refresh_index(index_name)
+          response = post_request("/#{index_name}/_refresh", nil)
+          parse_json_response!(response, context: "refresh index #{index_name}")
+        ensure
+          response&.close
+        end
+
+        def execute_index_exists?(index_name)
+          response = head_request("/#{index_name}")
+          response.success?
+        rescue StandardError
+          false
+        ensure
+          response&.close
+        end
+
+        def execute_update_document(index, id, changes, **_opts)
+          body = JSON.generate(doc: changes)
+
+          response = post_request("/#{index}/_update/#{id}", body)
+          parse_json_response!(response, context: "update document #{index}/#{id}")
+        ensure
+          response&.close
+        end
+
+        def execute_delete_document(index, id, **_opts)
+          response = delete_request("/#{index}/_doc/#{id}")
+          parse_json_response!(response, context: "delete document #{index}/#{id}")
+        ensure
+          response&.close
+        end
+
+        def execute_document_exists?(index, id)
+          response = head_request("/#{index}/_doc/#{id}")
+          response.success?
+        rescue StandardError
+          false
+        ensure
+          response&.close
+        end
+      end
+    end
+  end
+end

data/lib/noiseless/adapters/execution_modules/http_transport.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module Adapters
+    module ExecutionModules
+      # Shared Async::HTTP connection handling for HTTP-based adapters.
+      # Host classes must provide a private +default_port+ method.
+      module HttpTransport
+        def initialize(hosts: [], **connection_params)
+          # Ensure we always have at least one host
+          hosts_array = Array(hosts)
+          @hosts = hosts_array.empty? ? ["http://localhost:#{default_port}"] : hosts_array
+          @connection_params = connection_params
+
+          # Initialize HTTP clients for each host
+          @clients = {}
+          @hosts.each do |host|
+            endpoint = Async::HTTP::Endpoint.parse(host)
+            @clients[host] = Async::HTTP::Client.new(endpoint)
+          end
+
+          super(hosts: @hosts, **connection_params)
+        end
+
+        def close
+          @clients&.each_value(&:close)
+        end
+
+        private
+
+        # HTTP helpers using Async::HTTP with connection pooling
+        def get_request(path)
+          with_client do |client|
+            client.get(path, default_headers)
+          end
+        end
+
+        def post_request(path, body, content_type: "application/json")
+          headers = body ? default_headers + [["content-type", content_type]] : default_headers
+
+          with_client do |client|
+            client.post(path, headers, body)
+          end
+        end
+
+        def put_request(path, body, content_type: "application/json")
+          headers = body ? default_headers + [["content-type", content_type]] : default_headers
+
+          with_client do |client|
+            client.put(path, headers, body)
+          end
+        end
+
+        def delete_request(path)
+          with_client do |client|
+            client.delete(path, default_headers)
+          end
+        end
+
+        def head_request(path)
+          with_client do |client|
+            client.head(path, default_headers)
+          end
+        end
+
+        def with_client
+          # Select a random host for load balancing
+          host = @hosts.sample
+          client = @clients[host]
+
+          yield(client)
+        end
+
+        def default_headers
+          [
+            ["accept", "application/json"],
+            ["user-agent", "Noiseless/#{Noiseless::VERSION} (Ruby/#{RUBY_VERSION})"]
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/noiseless/adapters/execution_modules/opensearch_execution.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "es_compatible_execution"
+
+module Noiseless
+  module Adapters
+    module ExecutionModules
+      module OpensearchExecution
+        include EsCompatibleExecution
+
+        private
+
+        def execute_search(query_hash, indexes: [], **_opts)
+          index_path = indexes.any? ? indexes.join(",") : "_all"
+          path = "/#{index_path}/_search"
+          body = JSON.generate(query_hash)
+
+          response = post_request(path, body)
+          parse_json_response!(response, error_class: Noiseless::SearchError, context: "search #{index_path}")
+        ensure
+          response&.close
+        end
+
+        def execute_create_index(index_name, mappings: nil, settings: nil, **opts)
+          body = opts.dup
+          body[:mappings] = mappings if mappings
+          body[:settings] = settings if settings
+
+          response = put_request("/#{index_name}", body.any? ? JSON.generate(body) : nil)
+          parse_json_response!(response, context: "create index #{index_name}")
+        ensure
+          response&.close
+        end
+
+        def execute_index_document(index, id, document, **_opts)
+          path = "/#{index}/_doc/#{id}"
+          body = JSON.generate(document)
+
+          response = put_request(path, body)
+          parse_json_response!(response, context: "index document #{index}/#{id}")
+        ensure
+          response&.close
+        end
+
+        def execute_cluster_health(**_opts)
+          response = get_request("/_cluster/health")
+          JSON.parse(response.read)
+        rescue StandardError => e
+          {
+            cluster_name: "unknown",
+            status: "red",
+            timed_out: false,
+            number_of_nodes: 0,
+            number_of_data_nodes: 0,
+            active_primary_shards: 0,
+            active_shards: 0,
+            relocating_shards: 0,
+            initializing_shards: 0,
+            unassigned_shards: 0,
+            error: { type: e.class.name, reason: e.message }
+          }
+        ensure
+          response&.close
+        end
+
+        # OpenSearch-specific features
+        def execute_point_in_time_search(query_hash, pit_id:, **_opts)
+          # Point-in-time search for consistent pagination
+          enhanced_query = query_hash.merge(pit: { id: pit_id })
+          body = JSON.generate(enhanced_query)
+
+          response = post_request("/_search", body)
+          parse_json_response!(response, error_class: Noiseless::SearchError, context: "point-in-time search")
+        ensure
+          response&.close
+        end
+
+        def execute_search_template(template_id:, params: {}, **_opts)
+          # OpenSearch search templates
+          template_query = {
+            id: template_id,
+            params: params
+          }
+          body = JSON.generate(template_query)
+
+          response = post_request("/_search/template", body)
+          parse_json_response!(response, error_class: Noiseless::SearchError, context: "search template #{template_id}")
+        ensure
+          response&.close
+        end
+
+        # ============================================
+        # Search Pipeline API (OpenSearch 3.x)
+        # ============================================
+
+        def execute_create_pipeline(name, request_processors:, response_processors:, description: nil)
+          body = {
+            description: description,
+            request_processors: request_processors,
+            response_processors: response_processors
+          }.compact
+
+          response = put_request("/_search/pipeline/#{name}", JSON.generate(body))
+          JSON.parse(response.read)
+        rescue StandardError => e
+          { acknowledged: false, error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_get_pipeline(name)
+          response = get_request("/_search/pipeline/#{name}")
+          JSON.parse(response.read)
+        rescue StandardError => e
+          { error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_list_pipelines
+          response = get_request("/_search/pipeline")
+          JSON.parse(response.read)
+        rescue StandardError => e
+          { error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_delete_pipeline(name)
+          response = delete_request("/_search/pipeline/#{name}")
+          JSON.parse(response.read)
+        rescue StandardError => e
+          { acknowledged: false, error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_pipeline_exists?(name)
+          response = head_request("/_search/pipeline/#{name}")
+          response.success?
+        rescue StandardError
+          false
+        ensure
+          response&.close
+        end
+
+        # ============================================
+        # Query Rules API (OpenSearch 3.x)
+        # ============================================
+
+        def execute_create_rule(feature_type, rule_id, attributes:, feature_value:)
+          body = {
+            match_criteria: {
+              query: attributes
+            },
+            feature_value: feature_value
+          }
+
+          response = put_request("/_rules/#{feature_type}/#{rule_id}", JSON.generate(body))
+          JSON.parse(response.read)
+        rescue StandardError => e
+          { acknowledged: false, error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_get_rule(feature_type, rule_id)
+          response = get_request("/_rules/#{feature_type}/#{rule_id}")
+          JSON.parse(response.read)
+        rescue StandardError => e
+          { error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_list_rules(feature_type, search_after: nil)
+          path = "/_rules/#{feature_type}"
+          path += "?search_after=#{search_after}" if search_after
+
+          response = get_request(path)
+          JSON.parse(response.read)
+        rescue StandardError => e
+          { rules: [], error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_delete_rule(feature_type, rule_id)
+          response = delete_request("/_rules/#{feature_type}/#{rule_id}")
+          JSON.parse(response.read)
+        rescue StandardError => e
+          { acknowledged: false, error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_rule_exists?(feature_type, rule_id)
+          response = head_request("/_rules/#{feature_type}/#{rule_id}")
+          response.success?
+        rescue StandardError
+          false
+        ensure
+          response&.close
+        end
+      end
+    end
+  end
+end

data/lib/noiseless/adapters/execution_modules/pgvector_support.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module Adapters
+    module ExecutionModules
+      # pgvector support for semantic/vector search in PostgreSQL
+      # Provides similarity search using embeddings
+      #
+      # Required:
+      #   CREATE EXTENSION IF NOT EXISTS vector;
+      #
+      # Table setup:
+      #   ALTER TABLE your_table ADD COLUMN embedding vector(1536);
+      #   CREATE INDEX ON your_table USING ivfflat (embedding vector_cosine_ops);
+      #
+      module PgvectorSupport
+        # Perform semantic search using vector similarity
+        #
+        # @param scope [ActiveRecord::Relation] The base scope to search
+        # @param embedding [Array<Float>] The query embedding vector
+        # @param column [Symbol] The column containing embeddings (default: :embedding)
+        # @param limit [Integer] Maximum results to return
+        # @param distance_threshold [Float] Maximum distance threshold (optional)
+        # @param distance_metric [Symbol] :cosine, :l2, or :inner_product
+        # @return [ActiveRecord::Relation] Scope with vector similarity ordering
+        #
+        def vector_search(scope, embedding, column: :embedding, limit: 20, distance_threshold: nil,
+                          distance_metric: :cosine)
+          return scope unless pgvector_available?
+
+          vector_string = "[#{embedding.join(',')}]"
+          distance_op = distance_operator(distance_metric)
+
+          # Build the query with distance calculation
+          scope = scope.select(
+            "#{scope.table_name}.*",
+            "#{quoted_column(column)} #{distance_op} '#{vector_string}' AS vector_distance"
+          )
+
+          # Apply distance threshold if specified
+          if distance_threshold
+            scope = scope.where(
+              "#{quoted_column(column)} #{distance_op} '#{vector_string}' < ?",
+              distance_threshold
+            )
+          end
+
+          # Order by similarity (ascending distance = more similar)
+          scope.order(Arel.sql("#{quoted_column(column)} #{distance_op} '#{vector_string}'"))
+               .limit(limit)
+        end
+
+        # Hybrid search combining text and vector search
+        #
+        # @param scope [ActiveRecord::Relation] Base scope
+        # @param text_query [String] Text query for pg_trgm search
+        # @param embedding [Array<Float>] Query embedding for vector search
+        # @param text_fields [Array<Symbol>] Fields to search with text
+        # @param vector_column [Symbol] Column containing embeddings
+        # @param text_weight [Float] Weight for text similarity (0.0-1.0)
+        # @param vector_weight [Float] Weight for vector similarity (0.0-1.0)
+        # @return [ActiveRecord::Relation]
+        #
+        def hybrid_search(scope, text_query:, embedding:, text_fields:, vector_column: :embedding,
+                          text_weight: 0.5, vector_weight: 0.5, limit: 20)
+          return scope unless pgvector_available?
+
+          vector_string = "[#{embedding.join(',')}]"
+          text_conditions = text_fields.map { |f| "similarity(#{quoted_column(f)}, ?)" }.join(" + ")
+          text_similarity_count = text_fields.size
+
+          # Normalized combined score
+          scope.select(
+            "#{scope.table_name}.*",
+            # Text similarity (0-1 per field, averaged)
+            Arel.sql(
+              "(#{text_conditions}) / #{text_similarity_count} * #{text_weight} AS text_score"
+            ),
+            # Vector similarity (convert distance to similarity: 1 - distance for cosine)
+            "(1 - (#{quoted_column(vector_column)} <=> '#{vector_string}')) * #{vector_weight} AS vector_score",
+            # Combined score
+            "(((#{text_conditions}) / #{text_similarity_count}) * #{text_weight} + " \
+            "(1 - (#{quoted_column(vector_column)} <=> '#{vector_string}')) * #{vector_weight}) AS combined_score"
+          ).where(
+            "#{text_conditions} > 0 OR #{quoted_column(vector_column)} IS NOT NULL",
+            *Array.new(text_similarity_count, text_query)
+          ).order(Arel.sql("combined_score DESC"))
+               .limit(limit)
+               .tap { |s| s.bind_values.concat(Array.new(text_similarity_count, text_query)) }
+        end
+
+        # Execute a KNN (K-Nearest Neighbors) search
+        #
+        # @param model [Class] The ActiveRecord model
+        # @param embedding [Array<Float>] Query embedding
+        # @param k [Integer] Number of nearest neighbors
+        # @param column [Symbol] Embedding column
+        # @param filters [Hash] Additional WHERE conditions
+        # @return [Array<Hash>] Results with distance scores
+        #
+        def knn_search(model, embedding, k: 10, column: :embedding, filters: {})
+          return [] unless pgvector_available?
+
+          vector_string = "[#{embedding.join(',')}]"
+
+          scope = model.all
+          scope = scope.where(filters) if filters.any?
+
+          results = scope.select(
+            "#{model.table_name}.*",
+            "#{quoted_column(column)} <=> '#{vector_string}' AS distance"
+          ).order(Arel.sql("#{quoted_column(column)} <=> '#{vector_string}'"))
+                         .limit(k)
+
+          format_knn_response(results, model)
+        end
+
+        # Store an embedding for a record
+        #
+        # @param record [ActiveRecord::Base] The record to update
+        # @param embedding [Array<Float>] The embedding vector
+        # @param column [Symbol] The column to store the embedding
+        #
+        def store_embedding(record, embedding, column: :embedding)
+          return false unless pgvector_available?
+
+          vector_string = "[#{embedding.join(',')}]"
+          record.update_column(column, vector_string)
+        end
+
+        # Batch store embeddings
+        #
+        # @param model [Class] The ActiveRecord model
+        # @param embeddings [Hash<String, Array<Float>>] Map of ID -> embedding
+        # @param column [Symbol] The column to store embeddings
+        #
+        def batch_store_embeddings(model, embeddings, column: :embedding)
+          return 0 unless pgvector_available?
+
+          # Use UPDATE FROM VALUES for efficient batch update
+          values = embeddings.map do |id, emb|
+            "(#{ActiveRecord::Base.connection.quote(id)}, '[#{emb.join(',')}]'::vector)"
+          end.join(",")
+
+          sql = <<~SQL.squish
+            UPDATE #{model.table_name}
+            SET #{column} = v.embedding
+            FROM (VALUES #{values}) AS v(id, embedding)
+            WHERE #{model.table_name}.id = v.id::uuid
+          SQL
+
+          ActiveRecord::Base.connection.execute(sql)
+          embeddings.size
+        rescue StandardError => e
+          Rails.logger.error("Failed to batch store embeddings: #{e.message}")
+          0
+        end
+
+        # Find similar records to a given record
+        #
+        # @param record [ActiveRecord::Base] The reference record
+        # @param limit [Integer] Number of similar records
+        # @param column [Symbol] Embedding column
+        # @param exclude_self [Boolean] Exclude the reference record
+        # @return [ActiveRecord::Relation]
+        #
+        def find_similar(record, limit: 10, column: :embedding, exclude_self: true)
+          embedding = record.send(column)
+          return record.class.none unless embedding && pgvector_available?
+
+          scope = record.class.where.not(column => nil)
+          scope = scope.where.not(id: record.id) if exclude_self
+
+          vector_search(scope, embedding, column: column, limit: limit)
+        end
+
+        # Check if pgvector is available
+        def pgvector_available?
+          @pgvector_available ||= available_extensions.include?("vector")
+        end
+
+        private
+
+        def distance_operator(metric)
+          case metric
+          when :l2, :euclidean
+            "<->"  # L2/Euclidean distance
+          when :inner_product
+            "<#>"  # Negative inner product
+          else
+            "<=>"  # Cosine distance (default)
+          end
+        end
+
+        def format_knn_response(records, model)
+          hits = records.map do |record|
+            {
+              "_index" => model.table_name,
+              "_id" => record.id.to_s,
+              "_score" => 1.0 - (record.respond_to?(:distance) ? record.distance : 0),
+              "_source" => record.as_json(except: [:distance])
+            }
+          end
+
+          {
+            "took" => 0,
+            "timed_out" => false,
+            "_shards" => { "total" => 1, "successful" => 1, "skipped" => 0, "failed" => 0 },
+            "hits" => {
+              "total" => { "value" => hits.size, "relation" => "eq" },
+              "max_score" => hits.first&.dig("_score"),
+              "hits" => hits
+            }
+          }
+        end
+      end
+    end
+  end
+end