noiseless 0.7.1 → 0.8.0

data/lib/noiseless/adapters/execution_modules/typesense_query.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module Adapters
+    module ExecutionModules
+      # Translates the noiseless AST into Typesense search parameters.
+      module TypesenseQuery
+        private
+
+        def ast_to_hash(ast_node)
+          result = {}
+
+          query_parts = build_search_query(ast_node.bool)
+          result[:q] = query_parts.empty? ? "*" : query_parts
+
+          query_by_fields = build_query_by_fields(ast_node.bool)
+          result[:query_by] = query_by_fields unless query_by_fields.empty?
+
+          filter_expr = build_filter_expression(ast_node.bool)
+          result[:filter_by] = filter_expr unless filter_expr.empty?
+
+          sort_expr = build_sort_expression(ast_node.sort)
+          result[:sort_by] = sort_expr unless sort_expr.empty?
+
+          result.merge!(build_pagination_params(ast_node.paginate))
+          result[:enable_highlight_v1] = false
+
+          if ast_node.collapse
+            result[:group_by] = ast_node.collapse.field
+            result[:group_limit] = 1
+          end
+
+          terms_aggregations = ast_node.aggregations.select { it.type == :terms && it.field }
+          if terms_aggregations.any?
+            result[:facet_by] = terms_aggregations.map(&:field).uniq.join(",")
+            result[:aggregation_names] = terms_aggregations.to_h { [it.field, it.name] }
+          end
+
+          if ast_node.vector_search?
+            vector = ast_node.vector
+            result[:vector_query] = "#{vector.field}:([#{vector.embedding.join(',')}], k:#{vector.k})"
+            result[:exclude_fields] = vector.field.to_s
+          end
+
+          if ast_node.hybrid_search?
+            hybrid = ast_node.hybrid
+            vector = hybrid.vector
+            raise ArgumentError, "Typesense hybrid search needs fields: to run the text query against" if hybrid.fields.empty?
+
+            result[:q] = hybrid.text_query
+            result[:query_by] = hybrid.fields.join(",")
+            result[:vector_query] =
+              "#{vector.field}:([#{vector.embedding.join(',')}], k:#{vector.k}, alpha:#{hybrid.vector_weight})"
+            result[:exclude_fields] = vector.field.to_s
+          end
+
+          if ast_node.image_search?
+            img = ast_node.image_query
+            result[:vector_query] = "#{img.field}:([], image:#{img.image_data}, k:#{img.k})"
+            result[:exclude_fields] = img.field.to_s
+          end
+
+          if ast_node.conversational?
+            conv = ast_node.conversation
+            result[:conversation] = true
+            result[:conversation_model_id] = conv.model_id
+            result[:conversation_id] = conv.conversation_id if conv.conversation_id
+          end
+
+          if ast_node.has_joins?
+            result[:include_fields] = ast_node.joins.map { "$#{it.collection}(#{it.include_fields.presence&.join(', ') || '*'})" }.join(", ")
+            inner = ast_node.joins.select(&:inner_join?).map { "$#{it.collection}(id:*)" }
+            result[:filter_by] = [result[:filter_by], *inner].compact.join(" && ") if inner.any?
+          end
+
+          result[:remove_duplicates] = ast_node.remove_duplicates unless ast_node.remove_duplicates.nil?
+          result[:facet_sample_slope] = ast_node.facet_sample_slope unless ast_node.facet_sample_slope.nil?
+          result[:pinned_hits] = ast_node.pinned_hits unless ast_node.pinned_hits.nil?
+
+          result
+        end
+
+        def build_search_query(bool_node)
+          bool_node.must.filter_map do |node|
+            case node
+            when AST::Match then node.value.to_s
+            when AST::MultiMatch then node.query.to_s
+            end
+          end.join(" ")
+        end
+
+        def build_query_by_fields(bool_node)
+          bool_node.must.flat_map do |node|
+            case node
+            when AST::Match then [node.field.to_s]
+            when AST::MultiMatch then node.fields.map(&:to_s)
+            else []
+            end
+          end.uniq.join(",")
+        end
+
+        def build_filter_expression(bool_node)
+          filters = bool_node.filter.map { |filter| "#{filter.field}:=#{filter_value(filter.value)}" }
+
+          range_filters = bool_node.must.filter_map do |node|
+            next unless node.is_a?(AST::Range)
+
+            conditions = []
+            conditions << "#{node.field}:>#{filter_value(node.gt)}" if node.gt
+            conditions << "#{node.field}:>=#{filter_value(node.gte)}" if node.gte
+            conditions << "#{node.field}:<#{filter_value(node.lt)}" if node.lt
+            conditions << "#{node.field}:<=#{filter_value(node.lte)}" if node.lte
+            conditions.join(" && ")
+          end
+
+          (filters + range_filters).compact.join(" && ")
+        end
+
+        def filter_value(value)
+          return value.to_time.to_i.to_s if value.respond_to?(:to_time) && !value.is_a?(String)
+
+          case value
+          when Array then "[#{value.map { filter_value(it) }.join(',')}]"
+          when Numeric, true, false then value.to_s
+          else
+            string = value.to_s
+            raise ArgumentError, "Typesense filter values cannot contain backticks: #{string.inspect}" if string.include?("`")
+
+            "`#{string}`"
+          end
+        end
+
+        def build_sort_expression(sort_nodes)
+          sort_nodes.map { |sort| "#{sort.field}:#{sort.direction == :desc ? 'desc' : 'asc'}" }.join(",")
+        end
+
+        def build_pagination_params(paginate_node)
+          { page: paginate_node&.page || 1, per_page: paginate_node&.per_page || 20 }
+        end
+      end
+    end
+  end
+end

data/lib/noiseless/adapters/indices_api.rb

@@ -9,7 +9,9 @@ module Noiseless
       end
 
       def get(index:)
-        @adapter.execute_index_exists?(index) ? { index => {} } : raise("Index not found")
+        raise Noiseless::Error, "Index not found: #{index}" unless @adapter.index_exists?(index).wait
+
+        { index => {} }
       end
 
       def stats(index:)

data/lib/noiseless/adapters/open_search.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "execution_modules/opensearch_execution"
 
 module Noiseless
   module Adapters
@@ -10,152 +9,95 @@ module Noiseless
       ClusterAPI = Adapters::ClusterAPI
       IndicesAPI = Adapters::IndicesAPI
 
-      # OpenSearch-specific features
-      def point_in_time_search(ast_node, pit_id:, **)
-        query_hash = ast_to_hash(ast_node)
-        Async do
-          execute_point_in_time_search(query_hash, pit_id: pit_id, **)
-        end
-      end
-
       def search_template(template_id:, params: {}, **)
         Async do
           execute_search_template(template_id: template_id, params: params, **)
         end
       end
 
-      # Cluster health API - needed for Rails healthcheck
       def cluster
         @cluster ||= ClusterAPI.new(self)
       end
 
-      # Indices API - needed for index management operations
       def indices
         @indices ||= IndicesAPI.new(self)
       end
 
-      # Search Pipelines API - OpenSearch 3.x feature
       def pipelines
         @pipelines ||= PipelinesAPI.new(self)
       end
 
-      # Query Rules API - OpenSearch 3.x feature
       def rules
         @rules ||= RulesAPI.new(self)
       end
 
-      # Raw search for CommonShare compatibility
+      def workload_groups
+        @workload_groups ||= WorkloadGroupsAPI.new(self)
+      end
+
       def search_raw(query_body, indexes: [], **)
         Async do
           execute_search(query_body, indexes: indexes, **)
         end
       end
 
-      # Search Pipelines API for OpenSearch 3.x
-      # Pipelines can include request and response processors for neural search, reranking, etc.
       class PipelinesAPI
         def initialize(adapter)
           @adapter = adapter
         end
 
-        # Create or update a search pipeline
-        # @param name [String] Pipeline name
-        # @param request_processors [Array<Hash>] Request phase processors
-        # @param response_processors [Array<Hash>] Response phase processors
-        # @param description [String, nil] Optional description
-        def create(name, request_processors: [], response_processors: [], description: nil)
+        def create(name, request_processors: nil, response_processors: nil, phase_results_processors: nil, description: nil)
           Sync do
-            @adapter.send(:execute_create_pipeline, name,
-                          request_processors: request_processors,
-                          response_processors: response_processors,
-                          description: description)
+            @adapter.send(:execute_create_pipeline, name, request_processors:, response_processors:,
+                                                          phase_results_processors:, description:)
           end
         end
 
         alias put create
 
-        # Get a specific pipeline
-        def get(name)
-          Sync do
-            @adapter.send(:execute_get_pipeline, name)
-          end
-        end
-
-        # List all pipelines
-        def list
-          Sync do
-            @adapter.send(:execute_list_pipelines)
-          end
-        end
+        def get(name) = Sync { @adapter.send(:execute_get_pipeline, name) }
+        def list = Sync { @adapter.send(:execute_list_pipelines) }
+        def delete(name) = Sync { @adapter.send(:execute_delete_pipeline, name) }
+        def exists?(name) = Sync { @adapter.send(:execute_pipeline_exists?, name) }
 
         alias all list
-
-        # Delete a pipeline
-        def delete(name)
-          Sync do
-            @adapter.send(:execute_delete_pipeline, name)
-          end
-        end
-
-        # Check if a pipeline exists
-        def exists?(name)
-          Sync do
-            @adapter.send(:execute_pipeline_exists?, name)
-          end
-        end
       end
 
-      # Query Rules API for OpenSearch 3.x
-      # Rules allow pinning, boosting, or hiding specific results based on query patterns
+      # Rule-based auto-tagging; needs the workload-management plugin.
       class RulesAPI
         def initialize(adapter)
           @adapter = adapter
         end
 
-        # Create or update a rule
-        # @param feature_type [String] Feature type (e.g., 'pinned_queries')
-        # @param rule_id [String] Unique rule identifier
-        # @param attributes [Hash] Rule matching attributes
-        # @param feature_value [Hash] The feature value to apply
-        def create(feature_type, rule_id, attributes:, feature_value:)
-          Sync do
-            @adapter.send(:execute_create_rule, feature_type, rule_id,
-                          attributes: attributes,
-                          feature_value: feature_value)
-          end
+        def create(feature_type, description:, value:, **attributes)
+          Sync { @adapter.send(:execute_create_rule, feature_type, description:, value:, **attributes) }
         end
 
-        alias put create
-
-        # Get a specific rule
-        def get(feature_type, rule_id)
-          Sync do
-            @adapter.send(:execute_get_rule, feature_type, rule_id)
-          end
-        end
+        def update(feature_type, id, **changes) = Sync { @adapter.send(:execute_update_rule, feature_type, id, **changes) }
+        def get(feature_type, id) = Sync { @adapter.send(:execute_get_rule, feature_type, id) }
 
-        # List rules for a feature type
         def list(feature_type, search_after: nil)
-          Sync do
-            @adapter.send(:execute_list_rules, feature_type, search_after: search_after)
-          end
+          Sync { @adapter.send(:execute_list_rules, feature_type, search_after:) }
         end
 
+        def delete(feature_type, id) = Sync { @adapter.send(:execute_delete_rule, feature_type, id) }
+        def exists?(feature_type, id) = Sync { @adapter.send(:execute_rule_exists?, feature_type, id) }
+
         alias all list
+      end
 
-        # Delete a rule
-        def delete(feature_type, rule_id)
-          Sync do
-            @adapter.send(:execute_delete_rule, feature_type, rule_id)
-          end
+      # Workload groups that workload_group rules point at; needs the workload-management plugin.
+      class WorkloadGroupsAPI
+        def initialize(adapter)
+          @adapter = adapter
         end
 
-        # Check if a rule exists
-        def exists?(feature_type, rule_id)
-          Sync do
-            @adapter.send(:execute_rule_exists?, feature_type, rule_id)
-          end
+        def create(name, resource_limits:, resiliency_mode: "soft")
+          Sync { @adapter.send(:execute_create_workload_group, name, resource_limits:, resiliency_mode:) }
         end
+
+        def get(name) = Sync { @adapter.send(:execute_get_workload_group, name) }
+        def delete(name) = Sync { @adapter.send(:execute_delete_workload_group, name) }
       end
 
       private

data/lib/noiseless/adapters/postgresql.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "execution_modules/postgresql_execution"
 
 module Noiseless
   module Adapters
@@ -21,7 +20,7 @@ module Noiseless
 
       attr_reader :model_class_cache
 
-      def initialize(hosts: nil, **connection_params) # rubocop:disable Lint/UnusedMethodArgument
+      def initialize(hosts: nil, **connection_params)
         @connection_params = connection_params
         @model_class_cache = {}
 
@@ -44,7 +43,8 @@ module Noiseless
           sort: ast_node.sort,
           paginate: ast_node.paginate,
           indexes: ast_node.indexes,  # maps to table/model
-          vector: ast_node.vector     # for pgvector semantic search
+          vector: ast_node.vector,    # for pgvector semantic search
+          hybrid: ast_node.hybrid
         }
       end
 
@@ -52,7 +52,7 @@ module Noiseless
       def search(ast_node, model_class: nil, response_type: nil, **)
         query_hash = ast_to_hash(ast_node)
 
-        Async do
+        Async(finished: false) do
           raw_response = instrument(:search, indexes: ast_node.indexes, query: query_hash) do
             execute_search(query_hash, model_class: model_class, **)
           end
@@ -112,7 +112,7 @@ module Noiseless
           { "indices" => { index => {} } }
         end
 
-        def refresh(index:) # rubocop:disable Lint/UnusedMethodArgument
+        def refresh(index:)
           # No-op for PostgreSQL - queries always see latest data
           { "_shards" => { "total" => 1, "successful" => 1, "failed" => 0 } }
         end

data/lib/noiseless/adapters/typesense.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "execution_modules/typesense_execution"
 
 module Noiseless
   module Adapters
@@ -20,7 +19,7 @@ module Noiseless
       end
 
       class IndicesAPI < Adapters::IndicesAPI
-        def refresh(index: nil) # rubocop:disable Lint/UnusedMethodArgument
+        def refresh(index: nil)
           # Typesense doesn't require explicit refresh - documents are immediately available
           { "_shards" => { "total" => 1, "successful" => 1, "failed" => 0 } }
         end

data/lib/noiseless/ast/aggregation.rb

@@ -29,28 +29,5 @@ module Noiseless
         @sub_aggregations << aggregation
       end
     end
-
-    class AggregationBuilder
-      attr_reader :aggregations
-
-      def initialize
-        @aggregations = []
-      end
-
-      def agg(name, type, field: nil, **, &)
-        sub_aggs = []
-        if block_given?
-          sub_builder = AggregationBuilder.new
-          sub_builder.instance_eval(&)
-          sub_aggs = sub_builder.aggregations
-        end
-
-        aggregation = Aggregation.new(name, type, field: field, sub_aggregations: sub_aggs, **)
-        @aggregations << aggregation
-        aggregation
-      end
-
-      alias aggregation agg
-    end
   end
 end

data/lib/noiseless/ast/aggregation_builder.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module AST
+    class AggregationBuilder
+      attr_reader :aggregations
+
+      def initialize
+        @aggregations = []
+      end
+
+      def agg(name, type, field: nil, **, &)
+        sub_aggs = []
+        if block_given?
+          sub_builder = AggregationBuilder.new
+          sub_builder.instance_eval(&)
+          sub_aggs = sub_builder.aggregations
+        end
+
+        aggregation = Aggregation.new(name, type, field: field, sub_aggregations: sub_aggs, **)
+        @aggregations << aggregation
+        aggregation
+      end
+
+      alias aggregation agg
+    end
+  end
+end

data/lib/noiseless/ast/hybrid.rb

@@ -5,16 +5,18 @@ module Noiseless
     # Hybrid search node combining text and vector search
     # Supports weighted combination of BM25 text scores and kNN vector scores
     class Hybrid < Node
-      attr_reader :text_query, :vector, :text_weight, :vector_weight
+      attr_reader :text_query, :vector, :fields, :text_weight, :vector_weight
 
       # @param text_query [String] The text query for BM25 matching
       # @param vector [AST::Vector] The vector search node
+      # @param fields [Array<Symbol>] Fields the text query runs against
       # @param text_weight [Float] Weight for text search score (0.0-1.0)
       # @param vector_weight [Float] Weight for vector search score (0.0-1.0)
-      def initialize(text_query, vector, text_weight: 0.5, vector_weight: 0.5)
+      def initialize(text_query, vector, fields: [], text_weight: 0.5, vector_weight: 0.5)
         super()
         @text_query = text_query
         @vector = vector
+        @fields = Array(fields).map(&:to_s)
         @text_weight = text_weight
         @vector_weight = vector_weight
       end

data/lib/noiseless/ast/join.rb

@@ -3,18 +3,17 @@
 module Noiseless
   module AST
     # Join node for cross-collection queries (Typesense feature)
-    # Allows including related documents from other collections
+    # Allows including related documents from other collections. The relation
+    # comes from a reference field in the collection schema.
     class Join < Node
-      attr_reader :collection, :on, :include_fields, :strategy
+      attr_reader :collection, :include_fields, :strategy
 
       # @param collection [String, Symbol] The collection to join
-      # @param on [Hash] Join conditions (e.g., { foreign_key: :local_key })
       # @param include_fields [Array<String, Symbol>] Fields to include from joined collection
       # @param strategy [Symbol] Join strategy :left or :inner (default: :left)
-      def initialize(collection, on:, include_fields: [], strategy: :left)
+      def initialize(collection, include_fields: [], strategy: :left)
         super()
         @collection = collection.to_s
-        @on = on
         @include_fields = Array(include_fields).map(&:to_s)
         @strategy = strategy
       end

data/lib/noiseless/ast/vector.rb

@@ -5,18 +5,39 @@ module Noiseless
     # Vector search node for semantic/embedding-based search
     # Used with pgvector in PostgreSQL or knn in OpenSearch
     class Vector < Node
+      DISTANCE_METRICS = %i[cosine l2 euclidean inner_product].freeze
+
       attr_reader :field, :embedding, :k, :distance_metric
 
+      # Embeddings are rendered into SQL literals and engine query strings, so
+      # anything that is not a finite number is rejected rather than escaped.
+      def self.coerce_embedding(embedding)
+        values = Array(embedding).map do |value|
+          number = Float(value)
+          raise ArgumentError, "embedding contains a non-finite value: #{value.inspect}" unless number.finite?
+
+          number
+        end
+        raise ArgumentError, "embedding must not be empty" if values.empty?
+
+        values
+      rescue TypeError => e
+        raise ArgumentError, "embedding must contain only numbers (#{e.message})"
+      end
+
       # @param field [Symbol, String] The embedding column/field
       # @param embedding [Array<Float>] The query embedding vector
       # @param k [Integer] Number of nearest neighbors (default: 10)
-      # @param distance_metric [Symbol] :cosine, :l2, or :inner_product (default: :cosine)
+      # @param distance_metric [Symbol] :cosine, :l2, :euclidean or :inner_product (default: :cosine)
       def initialize(field, embedding, k: 10, distance_metric: :cosine)
         super()
         @field = field
-        @embedding = embedding
-        @k = k
-        @distance_metric = distance_metric
+        @embedding = embedding.nil? ? nil : self.class.coerce_embedding(embedding)
+        @k = Integer(k)
+        @distance_metric = distance_metric.to_sym
+        return if DISTANCE_METRICS.include?(@distance_metric)
+
+        raise ArgumentError, "distance_metric must be one of #{DISTANCE_METRICS.join(', ')}, got #{distance_metric.inspect}"
       end
 
       def dimension

data/lib/noiseless/bulk_importer.rb

@@ -19,11 +19,7 @@ module Noiseless
                **)
       @errors.clear
 
-      # Create index if force is true
-      if force
-        delete_index
-        create_index
-      end
+      recreate_index if force
 
       # Get records to import
       records = resolve_records(relation_or_records)
@@ -40,7 +36,7 @@ module Noiseless
         # Execute bulk operation
         begin
           client = Noiseless.connections.client(@connection)
-          response = client.bulk(actions, refresh: refresh, **)
+          response = client.bulk(actions, refresh: refresh, **).wait
 
           # Check for errors in response
           collect_errors(response, processed_batch)
@@ -95,6 +91,8 @@ module Noiseless
         document = transform ? transform.call(record) : default_transform(record)
         next unless document
 
+        document = Embedding.wrap(document, vector_fields)
+
         {
           index: {
             _index: index_name,
@@ -141,7 +139,7 @@ module Noiseless
     end
 
     def collect_errors(response, batch)
-      return unless response.is_a?(Hash) && response["items"]
+      return unless response.is_a?(Hash) && response["errors"] && response["items"]
 
       response["items"].each_with_index do |item, index|
         action = item.keys.first
@@ -166,30 +164,21 @@ module Noiseless
                       end
     end
 
-    def delete_index
+    def recreate_index
       client = Noiseless.connections.client(@connection)
-      client.delete_index(index_name)
-    rescue StandardError => _e
-      # Index might not exist, which is fine
-      nil
-    end
+      client.delete_index(index_name).wait
+      result = client.create_index(index_name, **index_config).wait
+      return if result.with_indifferent_access[:acknowledged]
 
-    def create_index
-      return unless model_class.respond_to?(:mapping)
+      raise Noiseless::Error, "index #{index_name} was deleted but could not be recreated: #{result.inspect}"
+    end
 
-      mapping_block = model_class.mapping
-      return unless mapping_block
+    def index_config
+      model_class.respond_to?(:mapping) ? MappingDefinitionProcessor.process(model_class.mapping) : {}
+    end
 
-      begin
-        _client = Noiseless.connections.client(@connection)
-        # This would need to be implemented in the adapter
-        # client.create_index(index_name, mapping: mapping_block)
-      rescue StandardError => e
-        @errors << {
-          error: "Failed to create index: #{e.message}",
-          index: index_name
-        }
-      end
+    def vector_fields
+      @vector_fields ||= Embedding.vector_fields(model_class)
     end
   end
 end