noiseless 0.0.0 → 0.2.0

data/lib/noiseless/adapters/execution_modules/typesense_execution.rb

@@ -0,0 +1,425 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "http_transport"
+
+module Noiseless
+  module Adapters
+    module ExecutionModules
+      module TypesenseExecution
+        include HttpTransport
+
+        private
+
+        # Override AST to Hash conversion for Typesense query format
+        def ast_to_hash(ast_node)
+          result = {}
+
+          # Build search query from match nodes
+          query_parts = build_search_query(ast_node.bool)
+          result[:q] = query_parts unless query_parts.empty?
+
+          # Build query_by from multi_match nodes
+          query_by_fields = build_query_by_fields(ast_node.bool)
+          result[:query_by] = query_by_fields unless query_by_fields.empty?
+
+          # Build filter expressions from filter nodes
+          filter_expr = build_filter_expression(ast_node.bool)
+          result[:filter_by] = filter_expr unless filter_expr.empty?
+
+          # Build sort expressions from sort nodes
+          sort_expr = build_sort_expression(ast_node.sort)
+          result[:sort_by] = sort_expr unless sort_expr.empty?
+
+          # Add pagination
+          pagination = build_pagination_params(ast_node.paginate)
+          result.merge!(pagination)
+
+          # Field collapsing -> Typesense group_by
+          if ast_node.collapse
+            result[:group_by] = ast_node.collapse.field
+            result[:group_limit] = 1 # Collapse shows 1 per group by default
+            if ast_node.collapse.max_concurrent_group_searches
+              # Typesense v30+: improve found accuracy for grouped results up to this threshold.
+              result[:group_max_candidates] = ast_node.collapse.max_concurrent_group_searches
+            end
+          end
+
+          # Aggregations -> Typesense facet_by
+          if ast_node.aggregations.any?
+            facet_fields = ast_node.aggregations
+                                   .select { |agg| agg.type == :terms }
+                                   .filter_map(&:field)
+
+            result[:facet_by] = facet_fields.join(",") if facet_fields.any?
+          end
+
+          # Vector search -> Typesense vector_query
+          if ast_node.vector_search?
+            vector = ast_node.vector
+            # Typesense uses format: "field_name:([vector], k:N)"
+            vector_str = vector.embedding.join(",")
+            result[:vector_query] = "#{vector.field}:([#{vector_str}], k:#{vector.k})"
+          end
+
+          # Hybrid search -> Typesense native hybrid with q + vector_query
+          if ast_node.hybrid_search?
+            hybrid = ast_node.hybrid
+            vector = hybrid.vector
+            vector_str = vector.embedding.join(",")
+
+            # Typesense natively supports hybrid by combining q and vector_query
+            result[:q] = hybrid.text_query
+            result[:vector_query] = "#{vector.field}:([#{vector_str}], k:#{vector.k}, alpha:#{hybrid.vector_weight})"
+          end
+
+          # Image search -> Typesense image embedding search
+          if ast_node.image_search?
+            img = ast_node.image_query
+            # Typesense accepts image URL or base64 directly in vector_query
+            result[:vector_query] = "#{img.field}:(#{img.image_data}, k:#{img.k})"
+          end
+
+          # Conversational/RAG search
+          if ast_node.conversational?
+            conv = ast_node.conversation
+            result[:conversation] = true
+            result[:conversation_model_id] = conv.model_id
+            result[:conversation_id] = conv.conversation_id if conv.conversation_id
+            result[:system_prompt] = conv.system_prompt if conv.system_prompt
+          end
+
+          # JOINs across collections
+          if ast_node.has_joins?
+            include_fields = ast_node.joins.map do |join_node|
+              fields = join_node.include_fields.join(", ")
+              "$#{join_node.collection}(#{fields})"
+            end
+            result[:include_fields] = include_fields.join(", ")
+          end
+
+          # Union-search related options (Typesense v30+).
+          result[:remove_duplicates] = ast_node.remove_duplicates unless ast_node.remove_duplicates.nil?
+          result[:facet_sample_slope] = ast_node.facet_sample_slope unless ast_node.facet_sample_slope.nil?
+          result[:pinned_hits] = ast_node.pinned_hits unless ast_node.pinned_hits.nil?
+
+          result
+        end
+
+        def build_search_query(bool_node)
+          # Combine all match queries into a single search string
+          queries = bool_node.must.filter_map do |node|
+            case node
+            when AST::Match
+              "#{node.field}:#{node.value}"
+            when AST::MultiMatch
+              # For Typesense, multi_match becomes a broader search across fields
+              node.query
+            when AST::Range
+              # Range queries are handled in filters, not search
+              nil
+            else
+              node.respond_to?(:value) ? "#{node.field}:#{node.value}" : nil
+            end
+          end
+          queries.join(" ")
+        end
+
+        def build_query_by_fields(bool_node)
+          # Extract fields from multi_match nodes for Typesense query_by parameter
+          fields = bool_node.must.filter_map do |node|
+            case node
+            when AST::MultiMatch
+              node.fields
+            end
+          end.flatten.uniq
+
+          fields.join(",")
+        end
+
+        def build_filter_expression(bool_node)
+          # Convert filter and range nodes to Typesense filter expressions
+          filters = bool_node.filter.map { |filter| "#{filter.field}:=#{filter.value}" }
+
+          # Add range filters from must clause
+          range_filters = bool_node.must.filter_map do |node|
+            next unless node.is_a?(AST::Range)
+
+            conditions = []
+            conditions << "#{node.field}:>#{node.gt}" if node.gt
+            conditions << "#{node.field}:>=#{node.gte}" if node.gte
+            conditions << "#{node.field}:<#{node.lt}" if node.lt
+            conditions << "#{node.field}:<=#{node.lte}" if node.lte
+            conditions.join(" && ")
+          end
+
+          (filters + range_filters).compact.join(" && ")
+        end
+
+        def build_sort_expression(sort_nodes)
+          # Convert sort nodes to Typesense sort format
+          sorts = sort_nodes.map do |sort|
+            direction = sort.direction == :desc ? "desc" : "asc"
+            "#{sort.field}:#{direction}"
+          end
+          sorts.join(",")
+        end
+
+        def build_pagination_params(paginate_node)
+          return { page: 1, per_page: 20 } unless paginate_node
+
+          {
+            page: paginate_node.page,
+            per_page: paginate_node.per_page
+          }
+        end
+
+        def execute_search(query_hash, collections: [], **_opts)
+          collection_path = collections.any? ? "/collections/#{collections.first}/documents/search" : "/multi_search"
+
+          # Convert query_hash to URL params for Typesense
+          params = query_hash.map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }.join("&")
+          path = "#{collection_path}?#{params}"
+
+          response = get_request(path)
+          result = JSON.parse(response.read)
+
+          # Convert Typesense format to Elasticsearch-like format
+          {
+            took: result["search_time_ms"] || 0,
+            timed_out: false,
+            _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
+            hits: {
+              total: { value: result["found"] || 0, relation: "eq" },
+              max_score: nil,
+              hits: (result["hits"] || []).map do |hit|
+                {
+                  _index: collections.first || "typesense",
+                  _type: "_doc",
+                  _id: hit["document"]["id"],
+                  _score: hit["text_match"] || 1.0,
+                  _source: hit["document"]
+                }
+              end
+            }
+          }
+        rescue StandardError => e
+          # Return empty response on error to maintain compatibility
+          {
+            took: 0,
+            timed_out: false,
+            _shards: { total: 0, successful: 0, skipped: 0, failed: 0 },
+            hits: {
+              total: { value: 0, relation: "eq" },
+              max_score: nil,
+              hits: []
+            },
+            error: {
+              type: e.class.name,
+              reason: e.message
+            }
+          }
+        ensure
+          response&.close
+        end
+
+        def execute_bulk(actions, **_opts)
+          # Typesense uses different endpoints for different operations
+          results = actions.map do |action|
+            if action[:index]
+              collection = action[:index][:_index]
+              id = action[:index][:_id]
+              document = action[:index][:data]
+
+              path = "/collections/#{collection}/documents"
+              body = JSON.generate(document.merge(id: id))
+
+              response = post_request(path, body)
+              result = JSON.parse(response.read)
+              response.close
+
+              { index: { _id: result["id"], status: 201, result: "created" } }
+            elsif action[:delete]
+              collection = action[:delete][:_index]
+              id = action[:delete][:_id]
+
+              path = "/collections/#{collection}/documents/#{id}"
+
+              response = delete_request(path)
+              response.close
+
+              { delete: { _id: id, status: 200, result: "deleted" } }
+            else
+              { error: { status: 400, error: "Unsupported action" } }
+            end
+          end
+
+          { items: results }
+        rescue StandardError => e
+          { items: [], errors: true, error: { type: e.class.name, reason: e.message } }
+        end
+
+        def execute_create_index(collection_name, mappings: nil, **_opts)
+          # Typesense calls indexes "collections"
+          schema = {
+            name: collection_name,
+            fields: []
+          }
+
+          # Convert mappings to Typesense schema if provided
+          if mappings && mappings["properties"]
+            schema[:fields] = mappings["properties"].map do |field_name, field_config|
+              {
+                name: field_name,
+                type: map_type_to_typesense(field_config["type"] || "string"),
+                facet: field_config["facet"] || false
+              }
+            end
+          end
+
+          body = JSON.generate(schema)
+          response = post_request("/collections", body)
+          result = JSON.parse(response.read)
+
+          { acknowledged: true, index: result["name"] }
+        rescue StandardError => e
+          { acknowledged: false, error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_delete_index(collection_name, **_opts)
+          response = delete_request("/collections/#{collection_name}")
+          JSON.parse(response.read)
+
+          { acknowledged: true }
+        rescue StandardError => e
+          { acknowledged: false, error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_index_exists?(collection_name)
+          response = head_request("/collections/#{collection_name}")
+          response.success?
+        rescue StandardError
+          false
+        ensure
+          response&.close
+        end
+
+        def execute_index_document(collection, id, document, **_opts)
+          path = "/collections/#{collection}/documents"
+          body = JSON.generate(document.merge(id: id))
+
+          response = post_request(path, body)
+          result = JSON.parse(response.read)
+
+          { _index: collection, _id: result["id"], result: "created" }
+        rescue StandardError => e
+          { _index: collection, _id: id, result: "error", error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_update_document(collection, id, changes, **_opts)
+          # Typesense doesn't have partial updates, so we need to fetch and merge
+          get_response = get_request("/collections/#{collection}/documents/#{id}")
+          document = JSON.parse(get_response.read)
+          get_response.close
+
+          updated_document = document.merge(changes).merge(id: id)
+          body = JSON.generate(updated_document)
+
+          response = put_request("/collections/#{collection}/documents/#{id}", body)
+          result = JSON.parse(response.read)
+
+          { _index: collection, _id: result["id"], result: "updated" }
+        rescue StandardError => e
+          { _index: collection, _id: id, result: "error", error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close if defined?(response)
+        end
+
+        def execute_delete_document(collection, id, **_opts)
+          response = delete_request("/collections/#{collection}/documents/#{id}")
+
+          { _index: collection, _id: id, result: "deleted" }
+        rescue StandardError => e
+          { _index: collection, _id: id, result: "error", error: { type: e.class.name, reason: e.message } }
+        ensure
+          response&.close
+        end
+
+        def execute_document_exists?(collection, id)
+          response = head_request("/collections/#{collection}/documents/#{id}")
+          response.success?
+        rescue StandardError
+          false
+        ensure
+          response&.close
+        end
+
+        def execute_cluster_health(**_opts)
+          response = get_request("/health")
+          health_data = JSON.parse(response.read)
+
+          # Convert Typesense health format to match expected format
+          {
+            cluster_name: "typesense",
+            status: health_data["ok"] ? "green" : "red",
+            timed_out: false,
+            number_of_nodes: 1,
+            number_of_data_nodes: 1,
+            active_primary_shards: 0,
+            active_shards: 0,
+            typesense_ok: health_data["ok"]
+          }
+        rescue StandardError => e
+          {
+            cluster_name: "unknown",
+            status: "red",
+            timed_out: false,
+            number_of_nodes: 0,
+            number_of_data_nodes: 0,
+            active_primary_shards: 0,
+            active_shards: 0,
+            error: { type: e.class.name, reason: e.message }
+          }
+        ensure
+          response&.close
+        end
+
+        def default_headers
+          headers = super
+
+          # Add Typesense API key if configured
+          if @connection_params && @connection_params[:api_key]
+            headers << ["X-TYPESENSE-API-KEY",
+                        @connection_params[:api_key]]
+          end
+
+          headers
+        end
+
+        # rubocop:disable Lint/DuplicateBranch
+        def map_type_to_typesense(elasticsearch_type)
+          # Map Elasticsearch types to Typesense types
+          case elasticsearch_type
+          when "text", "keyword"
+            "string"
+          when "long", "integer", "short", "byte", "date"
+            "int64" # date uses Unix timestamps
+          when "double", "float", "half_float", "scaled_float"
+            "float"
+          when "boolean"
+            "bool"
+          else
+            "string" # Default to string for unknown types
+          end
+        end
+        # rubocop:enable Lint/DuplicateBranch
+      end
+    end
+  end
+end

data/lib/noiseless/adapters/indices_api.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module Adapters
+    # Indices API - needed for index management operations
+    class IndicesAPI
+      def initialize(adapter)
+        @adapter = adapter
+      end
+
+      def get(index:)
+        @adapter.execute_index_exists?(index) ? { index => {} } : raise("Index not found")
+      end
+
+      def stats(index:)
+        # Return basic stats structure
+        { "indices" => { index => {} } }
+      end
+
+      def refresh(index:)
+        # Refresh the index to make documents immediately searchable
+        @adapter.send(:execute_refresh_index, index)
+      end
+    end
+  end
+end

data/lib/noiseless/adapters/open_search.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require_relative "execution_modules/opensearch_execution"
+
+module Noiseless
+  module Adapters
+    class OpenSearch < Adapter
+      include ExecutionModules::OpensearchExecution
+
+      ClusterAPI = Adapters::ClusterAPI
+      IndicesAPI = Adapters::IndicesAPI
+
+      # OpenSearch-specific features
+      def point_in_time_search(ast_node, pit_id:, **)
+        query_hash = ast_to_hash(ast_node)
+        Async do
+          execute_point_in_time_search(query_hash, pit_id: pit_id, **)
+        end
+      end
+
+      def search_template(template_id:, params: {}, **)
+        Async do
+          execute_search_template(template_id: template_id, params: params, **)
+        end
+      end
+
+      # Cluster health API - needed for Rails healthcheck
+      def cluster
+        @cluster ||= ClusterAPI.new(self)
+      end
+
+      # Indices API - needed for index management operations
+      def indices
+        @indices ||= IndicesAPI.new(self)
+      end
+
+      # Search Pipelines API - OpenSearch 3.x feature
+      def pipelines
+        @pipelines ||= PipelinesAPI.new(self)
+      end
+
+      # Query Rules API - OpenSearch 3.x feature
+      def rules
+        @rules ||= RulesAPI.new(self)
+      end
+
+      # Raw search for CommonShare compatibility
+      def search_raw(query_body, indexes: [], **)
+        Async do
+          execute_search(query_body, indexes: indexes, **)
+        end
+      end
+
+      # Search Pipelines API for OpenSearch 3.x
+      # Pipelines can include request and response processors for neural search, reranking, etc.
+      class PipelinesAPI
+        def initialize(adapter)
+          @adapter = adapter
+        end
+
+        # Create or update a search pipeline
+        # @param name [String] Pipeline name
+        # @param request_processors [Array<Hash>] Request phase processors
+        # @param response_processors [Array<Hash>] Response phase processors
+        # @param description [String, nil] Optional description
+        def create(name, request_processors: [], response_processors: [], description: nil)
+          Sync do
+            @adapter.send(:execute_create_pipeline, name,
+                          request_processors: request_processors,
+                          response_processors: response_processors,
+                          description: description)
+          end
+        end
+
+        alias put create
+
+        # Get a specific pipeline
+        def get(name)
+          Sync do
+            @adapter.send(:execute_get_pipeline, name)
+          end
+        end
+
+        # List all pipelines
+        def list
+          Sync do
+            @adapter.send(:execute_list_pipelines)
+          end
+        end
+
+        alias all list
+
+        # Delete a pipeline
+        def delete(name)
+          Sync do
+            @adapter.send(:execute_delete_pipeline, name)
+          end
+        end
+
+        # Check if a pipeline exists
+        def exists?(name)
+          Sync do
+            @adapter.send(:execute_pipeline_exists?, name)
+          end
+        end
+      end
+
+      # Query Rules API for OpenSearch 3.x
+      # Rules allow pinning, boosting, or hiding specific results based on query patterns
+      class RulesAPI
+        def initialize(adapter)
+          @adapter = adapter
+        end
+
+        # Create or update a rule
+        # @param feature_type [String] Feature type (e.g., 'pinned_queries')
+        # @param rule_id [String] Unique rule identifier
+        # @param attributes [Hash] Rule matching attributes
+        # @param feature_value [Hash] The feature value to apply
+        def create(feature_type, rule_id, attributes:, feature_value:)
+          Sync do
+            @adapter.send(:execute_create_rule, feature_type, rule_id,
+                          attributes: attributes,
+                          feature_value: feature_value)
+          end
+        end
+
+        alias put create
+
+        # Get a specific rule
+        def get(feature_type, rule_id)
+          Sync do
+            @adapter.send(:execute_get_rule, feature_type, rule_id)
+          end
+        end
+
+        # List rules for a feature type
+        def list(feature_type, search_after: nil)
+          Sync do
+            @adapter.send(:execute_list_rules, feature_type, search_after: search_after)
+          end
+        end
+
+        alias all list
+
+        # Delete a rule
+        def delete(feature_type, rule_id)
+          Sync do
+            @adapter.send(:execute_delete_rule, feature_type, rule_id)
+          end
+        end
+
+        # Check if a rule exists
+        def exists?(feature_type, rule_id)
+          Sync do
+            @adapter.send(:execute_rule_exists?, feature_type, rule_id)
+          end
+        end
+      end
+
+      private
+
+      def default_port
+        ENV["OPENSEARCH_PORT"] || 9200
+      end
+    end
+  end
+end