noiseless 0.7.1 → 0.8.0

data/lib/noiseless/adapters/execution_modules/typesense_execution.rb

@@ -1,361 +1,250 @@
 # frozen_string_literal: true
 
+require "cgi/escape"
 require "json"
-require_relative "http_transport"
 
 module Noiseless
   module Adapters
     module ExecutionModules
       module TypesenseExecution
         include HttpTransport
+        include TypesenseQuery
 
-        private
-
-        # Override AST to Hash conversion for Typesense query format
-        def ast_to_hash(ast_node)
-          result = {}
-
-          # Build search query from match nodes
-          query_parts = build_search_query(ast_node.bool)
-          result[:q] = query_parts unless query_parts.empty?
-
-          # Build query_by from multi_match nodes
-          query_by_fields = build_query_by_fields(ast_node.bool)
-          result[:query_by] = query_by_fields unless query_by_fields.empty?
-
-          # Build filter expressions from filter nodes
-          filter_expr = build_filter_expression(ast_node.bool)
-          result[:filter_by] = filter_expr unless filter_expr.empty?
-
-          # Build sort expressions from sort nodes
-          sort_expr = build_sort_expression(ast_node.sort)
-          result[:sort_by] = sort_expr unless sort_expr.empty?
-
-          # Add pagination
-          pagination = build_pagination_params(ast_node.paginate)
-          result.merge!(pagination)
-
-          # Field collapsing -> Typesense group_by
-          if ast_node.collapse
-            result[:group_by] = ast_node.collapse.field
-            result[:group_limit] = 1 # Collapse shows 1 per group by default
-            if ast_node.collapse.max_concurrent_group_searches
-              # Typesense v30+: improve found accuracy for grouped results up to this threshold.
-              result[:group_max_candidates] = ast_node.collapse.max_concurrent_group_searches
-            end
-          end
-
-          # Aggregations -> Typesense facet_by
-          if ast_node.aggregations.any?
-            facet_fields = ast_node.aggregations
-                                   .select { |agg| agg.type == :terms }
-                                   .filter_map(&:field)
-
-            result[:facet_by] = facet_fields.join(",") if facet_fields.any?
+        # Typesense stores date fields as int64, so times travel as epoch seconds.
+        TYPESENSE_JSON = JSON::Coder.new do |object|
+          if object.respond_to?(:to_time) then object.to_time.to_i
+          elsif object.is_a?(BigDecimal) then object.to_f
+          else object.respond_to?(:as_json) ? object.as_json : object.to_s
           end
+        end
 
-          # Vector search -> Typesense vector_query
-          if ast_node.vector_search?
-            vector = ast_node.vector
-            # Typesense uses format: "field_name:([vector], k:N)"
-            vector_str = vector.embedding.join(",")
-            result[:vector_query] = "#{vector.field}:([#{vector_str}], k:#{vector.k})"
-          end
-
-          # Hybrid search -> Typesense native hybrid with q + vector_query
-          if ast_node.hybrid_search?
-            hybrid = ast_node.hybrid
-            vector = hybrid.vector
-            vector_str = vector.embedding.join(",")
-
-            # Typesense natively supports hybrid by combining q and vector_query
-            result[:q] = hybrid.text_query
-            result[:vector_query] = "#{vector.field}:([#{vector_str}], k:#{vector.k}, alpha:#{hybrid.vector_weight})"
-          end
+        private
 
-          # Image search -> Typesense image embedding search
-          if ast_node.image_search?
-            img = ast_node.image_query
-            # Typesense accepts image URL or base64 directly in vector_query
-            result[:vector_query] = "#{img.field}:(#{img.image_data}, k:#{img.k})"
-          end
+        def dump_json(object) = TYPESENSE_JSON.dump(object)
 
-          # Conversational/RAG search
-          if ast_node.conversational?
-            conv = ast_node.conversation
-            result[:conversation] = true
-            result[:conversation_model_id] = conv.model_id
-            result[:conversation_id] = conv.conversation_id if conv.conversation_id
-            result[:system_prompt] = conv.system_prompt if conv.system_prompt
-          end
+        def execute_search(query_hash, indexes: [], **_opts)
+          collections = Array(indexes).map(&:to_s)
+          raise Noiseless::SearchError, "Typesense search needs a collection" if collections.empty?
 
-          # JOINs across collections
-          if ast_node.has_joins?
-            include_fields = ast_node.joins.map do |join_node|
-              fields = join_node.include_fields.join(", ")
-              "$#{join_node.collection}(#{fields})"
-            end
-            result[:include_fields] = include_fields.join(", ")
-          end
+          aggregation_names = query_hash[:aggregation_names] || {}
+          params = query_hash.except(:aggregation_names, :remove_duplicates)
+          context = "search #{collections.join(',')}"
 
-          # Union-search related options (Typesense v30+).
-          result[:remove_duplicates] = ast_node.remove_duplicates unless ast_node.remove_duplicates.nil?
-          result[:facet_sample_slope] = ast_node.facet_sample_slope unless ast_node.facet_sample_slope.nil?
-          result[:pinned_hits] = ast_node.pinned_hits unless ast_node.pinned_hits.nil?
+          result = if collections.one?
+                     body = { searches: [params.merge(collection: collections.first)] }
+                     single = post_json!("/multi_search", body, context:).fetch("results").first
+                     raise_search_error!(single, context) if single["error"]
+                     single
+                   else
+                     body = { union: true, searches: collections.map { params.except(:page, :per_page).merge(collection: it) } }
+                     body[:remove_duplicates] = query_hash[:remove_duplicates] unless query_hash[:remove_duplicates].nil?
+                     post_json!("/multi_search?page=#{params[:page]}&per_page=#{params[:per_page]}", body, context:)
+                   end
 
-          result
+          search_response(result, collections.first, aggregation_names)
         end
 
-        def build_search_query(bool_node)
-          # Combine all match queries into a single search string
-          queries = bool_node.must.filter_map do |node|
-            case node
-            when AST::Match
-              "#{node.field}:#{node.value}"
-            when AST::MultiMatch
-              # For Typesense, multi_match becomes a broader search across fields
-              node.query
-            when AST::Range
-              # Range queries are handled in filters, not search
-              nil
-            else
-              node.respond_to?(:value) ? "#{node.field}:#{node.value}" : nil
-            end
-          end
-          queries.join(" ")
+        def post_json!(path, body, context:)
+          response = post_request(path, dump_json(body))
+          parse_json_response!(response, error_class: Noiseless::SearchError, context:)
+        ensure
+          response&.close
         end
 
-        def build_query_by_fields(bool_node)
-          # Extract fields from multi_match nodes for Typesense query_by parameter
-          fields = bool_node.must.filter_map do |node|
-            case node
-            when AST::MultiMatch
-              node.fields
-            end
-          end.flatten.uniq
-
-          fields.join(",")
+        def raise_search_error!(result, context)
+          raise Noiseless::SearchError.new("#{context}: #{result['error']}", status: result["code"])
         end
 
-        def build_filter_expression(bool_node)
-          # Convert filter and range nodes to Typesense filter expressions
-          filters = bool_node.filter.map { |filter| "#{filter.field}:=#{filter.value}" }
+        def search_response(result, default_collection, aggregation_names)
+          hits = if result["grouped_hits"]
+                   result["grouped_hits"].filter_map { it["hits"].first }
+                 else
+                   result["hits"] || []
+                 end
 
-          # Add range filters from must clause
-          range_filters = bool_node.must.filter_map do |node|
-            next unless node.is_a?(AST::Range)
+          {
+            "took" => result["search_time_ms"] || 0,
+            "timed_out" => false,
+            "hits" => {
+              "total" => { "value" => result["found"] || 0, "relation" => "eq" },
+              "max_score" => nil,
+              "hits" => hits.map { search_hit(it, default_collection) }
+            },
+            "aggregations" => facet_aggregations(result["facet_counts"], aggregation_names)
+          }
+        end
 
-            conditions = []
-            conditions << "#{node.field}:>#{node.gt}" if node.gt
-            conditions << "#{node.field}:>=#{node.gte}" if node.gte
-            conditions << "#{node.field}:<#{node.lt}" if node.lt
-            conditions << "#{node.field}:<=#{node.lte}" if node.lte
-            conditions.join(" && ")
-          end
+        def search_hit(hit, default_collection)
+          score = hit.dig("hybrid_search_info", "rank_fusion_score") ||
+                  (hit["vector_distance"] && (1.0 - hit["vector_distance"])) ||
+                  hit["text_match"] || 1.0
 
-          (filters + range_filters).compact.join(" && ")
+          {
+            "_index" => hit["collection"] || default_collection,
+            "_id" => hit.dig("document", "id"),
+            "_score" => score,
+            "_source" => hit["document"]
+          }
         end
 
-        def build_sort_expression(sort_nodes)
-          # Convert sort nodes to Typesense sort format
-          sorts = sort_nodes.map do |sort|
-            direction = sort.direction == :desc ? "desc" : "asc"
-            "#{sort.field}:#{direction}"
+        def facet_aggregations(facet_counts, aggregation_names)
+          Array(facet_counts).each_with_object({}) do |facet, aggregations|
+            name = aggregation_names.fetch(facet["field_name"], facet["field_name"])
+            buckets = facet["counts"].map { { "key" => it["value"], "doc_count" => it["count"] } }
+            aggregations[name] = { "buckets" => buckets }
           end
-          sorts.join(",")
         end
 
-        def build_pagination_params(paginate_node)
-          return { page: 1, per_page: 20 } unless paginate_node
-
-          {
-            page: paginate_node.page,
-            per_page: paginate_node.per_page
-          }
-        end
+        def execute_bulk(actions, **_opts)
+          items = Array.new(actions.size)
+          indexed = actions.each_with_index.group_by { |action, _| action.dig(:index, :_index) || action.dig(:delete, :_index) }
 
-        def execute_search(query_hash, collections: [], **_opts)
-          collection_path = collections.any? ? "/collections/#{collections.first}/documents/search" : "/multi_search"
+          indexed.each do |collection, pairs|
+            imports, deletes = pairs.partition { |action, _| action[:index] }
+            import_documents(collection, imports, items) if imports.any?
+            delete_documents(collection, deletes, items) if deletes.any?
+          end
 
-          # Convert query_hash to URL params for Typesense
-          params = query_hash.map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }.join("&")
-          path = "#{collection_path}?#{params}"
+          { "items" => items, "errors" => items.any? { it.values.first["error"] } }
+        end
 
-          response = get_request(path)
-          result = JSON.parse(response.read)
+        def import_documents(collection, pairs, items)
+          jsonl = pairs.map { |action, _| dump_json(typesense_document(collection, action[:index][:data], action[:index][:_id])) }.join("\n")
+          response = post_request("/collections/#{collection}/documents/import?action=upsert", jsonl, content_type: "text/plain")
+          body = response.read
+          unless response.success?
+            raise Noiseless::RequestError.new("import #{collection}: #{error_message(body, response.status)}", status: response.status)
+          end
 
-          # Convert Typesense format to Elasticsearch-like format
-          {
-            took: result["search_time_ms"] || 0,
-            timed_out: false,
-            _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
-            hits: {
-              total: { value: result["found"] || 0, relation: "eq" },
-              max_score: nil,
-              hits: (result["hits"] || []).map do |hit|
-                {
-                  _index: collections.first || "typesense",
-                  _type: "_doc",
-                  _id: hit["document"]["id"],
-                  _score: hit["text_match"] || 1.0,
-                  _source: hit["document"]
-                }
-              end
-            }
-          }
-        rescue StandardError => e
-          # Return empty response on error to maintain compatibility
-          {
-            took: 0,
-            timed_out: false,
-            _shards: { total: 0, successful: 0, skipped: 0, failed: 0 },
-            hits: {
-              total: { value: 0, relation: "eq" },
-              max_score: nil,
-              hits: []
-            },
-            error: {
-              type: e.class.name,
-              reason: e.message
+          body.each_line.zip(pairs).each do |line, (action, position)|
+            result = JSON.parse(line)
+            items[position] = {
+              "index" => {
+                "_index" => collection,
+                "_id" => action[:index][:_id].to_s,
+                "status" => result["success"] ? 201 : result["code"],
+                "error" => (result["error"] unless result["success"])
+              }.compact
             }
-          }
+          end
         ensure
           response&.close
         end
 
-        def execute_bulk(actions, **_opts)
-          # Typesense uses different endpoints for different operations
-          results = actions.map do |action|
-            if action[:index]
-              collection = action[:index][:_index]
-              id = action[:index][:_id]
-              document = action[:index][:data]
-
-              path = "/collections/#{collection}/documents"
-              body = JSON.generate(document.merge(id: id))
-
-              response = post_request(path, body)
-              result = JSON.parse(response.read)
-              response.close
-
-              { index: { _id: result["id"], status: 201, result: "created" } }
-            elsif action[:delete]
-              collection = action[:delete][:_index]
-              id = action[:delete][:_id]
-
-              path = "/collections/#{collection}/documents/#{id}"
-
-              response = delete_request(path)
-              response.close
-
-              { delete: { _id: id, status: 200, result: "deleted" } }
-            else
-              { error: { status: 400, error: "Unsupported action" } }
-            end
+        def delete_documents(collection, pairs, items)
+          ids = pairs.map { |action, _| action[:delete][:_id] }
+          filter = CGI.escape("id:#{filter_value(ids.map(&:to_s))}")
+          response = delete_request("/collections/#{collection}/documents?filter_by=#{filter}")
+          parse_json_response!(response, context: "delete documents #{collection}")
+
+          pairs.each do |action, position|
+            items[position] = { "delete" => { "_index" => collection, "_id" => action[:delete][:_id].to_s, "status" => 200 } }
           end
+        ensure
+          response&.close
+        end
 
-          { items: results }
-        rescue StandardError => e
-          { items: [], errors: true, error: { type: e.class.name, reason: e.message } }
+        def typesense_document(collection, document, id)
+          shape_document(collection, document).merge("id" => id.to_s)
         end
 
-        def execute_create_index(collection_name, mappings: nil, **_opts)
-          # Typesense calls indexes "collections"
-          schema = {
-            name: collection_name,
-            fields: []
-          }
+        def shape_document(collection, document)
+          arrays = array_fields(collection)
+          document.to_h.to_h do |key, value|
+            key = key.to_s
+            [key, arrays.include?(key) && !value.nil? && !value.is_a?(Array) ? [value] : value]
+          end
+        end
 
-          # Convert mappings to Typesense schema if provided
-          if mappings && mappings["properties"]
-            schema[:fields] = mappings["properties"].map do |field_name, field_config|
-              {
-                name: field_name,
-                type: map_type_to_typesense(field_config["type"] || "string"),
-                facet: field_config["facet"] || false
-              }
-            end
+        def array_fields(collection)
+          (@array_fields ||= {})[collection.to_s] ||= begin
+            response = get_request("/collections/#{collection}")
+            fields = response.success? ? JSON.parse(response.read).fetch("fields") : []
+            fields.filter_map { it["name"] if it["type"].end_with?("[]") }.to_set
+          ensure
+            response&.close
           end
+        end
 
-          body = JSON.generate(schema)
-          response = post_request("/collections", body)
-          result = JSON.parse(response.read)
+        def error_message(body, status)
+          parsed = JSON.parse(body)
+          parsed.is_a?(Hash) && parsed["message"] ? parsed["message"] : "HTTP #{status}"
+        rescue JSON::ParserError
+          "HTTP #{status}"
+        end
 
-          { acknowledged: true, index: result["name"] }
-        rescue StandardError => e
-          { acknowledged: false, error: { type: e.class.name, reason: e.message } }
+        def execute_create_index(collection_name, mappings: nil, **_opts)
+          @array_fields&.delete(collection_name.to_s)
+          properties = (mappings || {}).with_indifferent_access[:properties] || {}
+          fields = properties.map { |name, config| typesense_field(name, config) }
+          fields = [{ name: ".*", type: "auto" }] if fields.empty?
+
+          response = post_request("/collections", dump_json(name: collection_name, fields:))
+          result = parse_json_response!(response, context: "create collection #{collection_name}")
+
+          { "acknowledged" => true, "index" => result["name"] }
         ensure
           response&.close
         end
 
+        def typesense_field(name, config)
+          type = map_type_to_typesense(config[:type].to_s)
+          field = { name: name.to_s, type: type, optional: true }
+          field[:num_dim] = config[:dims] || config[:dimension] if type == "float[]"
+          field[:facet] = true if config[:facet] || config[:type].to_s == "keyword"
+          field[:reference] = config[:reference] if config[:reference]
+          field
+        end
+
         def execute_delete_index(collection_name, **_opts)
+          @array_fields&.delete(collection_name.to_s)
           response = delete_request("/collections/#{collection_name}")
-          JSON.parse(response.read)
+          return { "acknowledged" => true, "result" => "not_found" } if response.status == 404
 
-          { acknowledged: true }
-        rescue StandardError => e
-          { acknowledged: false, error: { type: e.class.name, reason: e.message } }
+          parse_json_response!(response, context: "delete collection #{collection_name}")
+          { "acknowledged" => true }
         ensure
           response&.close
         end
 
         def execute_index_exists?(collection_name)
-          response = head_request("/collections/#{collection_name}")
-          response.success?
-        rescue StandardError
-          false
+          response = get_request("/collections/#{collection_name}")
+          exists_response?(response, context: "collection exists #{collection_name}")
         ensure
           response&.close
         end
 
         def execute_index_document(collection, id, document, **_opts)
-          path = "/collections/#{collection}/documents"
-          body = JSON.generate(document.merge(id: id))
-
-          response = post_request(path, body)
-          result = JSON.parse(response.read)
+          response = post_request("/collections/#{collection}/documents?action=upsert",
+                                  dump_json(typesense_document(collection, document, id)))
+          result = parse_json_response!(response, context: "index document #{collection}/#{id}")
 
-          { _index: collection, _id: result["id"], result: "created" }
-        rescue StandardError => e
-          { _index: collection, _id: id, result: "error", error: { type: e.class.name, reason: e.message } }
+          { "_index" => collection, "_id" => result["id"], "result" => "upserted" }
         ensure
           response&.close
         end
 
         def execute_update_document(collection, id, changes, **_opts)
-          # Typesense doesn't have partial updates, so we need to fetch and merge
-          get_response = get_request("/collections/#{collection}/documents/#{id}")
-          document = JSON.parse(get_response.read)
-          get_response.close
-
-          updated_document = document.merge(changes).merge(id: id)
-          body = JSON.generate(updated_document)
-
-          response = put_request("/collections/#{collection}/documents/#{id}", body)
-          result = JSON.parse(response.read)
+          response = patch_request("/collections/#{collection}/documents/#{id}", dump_json(shape_document(collection, changes)))
+          result = parse_json_response!(response, context: "update document #{collection}/#{id}")
 
-          { _index: collection, _id: result["id"], result: "updated" }
-        rescue StandardError => e
-          { _index: collection, _id: id, result: "error", error: { type: e.class.name, reason: e.message } }
+          { "_index" => collection, "_id" => result["id"], "result" => "updated" }
         ensure
-          response&.close if defined?(response)
+          response&.close
         end
 
         def execute_delete_document(collection, id, **_opts)
-          response = delete_request("/collections/#{collection}/documents/#{id}")
+          response = delete_request("/collections/#{collection}/documents/#{id}?ignore_not_found=true")
+          parse_json_response!(response, context: "delete document #{collection}/#{id}")
 
-          { _index: collection, _id: id, result: "deleted" }
-        rescue StandardError => e
-          { _index: collection, _id: id, result: "error", error: { type: e.class.name, reason: e.message } }
+          { "_index" => collection, "_id" => id.to_s, "result" => "deleted" }
         ensure
           response&.close
         end
 
         def execute_document_exists?(collection, id)
-          response = head_request("/collections/#{collection}/documents/#{id}")
-          response.success?
-        rescue StandardError
-          false
+          response = get_request("/collections/#{collection}/documents/#{id}?include_fields=id")
+          exists_response?(response, context: "document exists #{collection}/#{id}")
         ensure
           response&.close
         end
@@ -364,7 +253,6 @@ module Noiseless
           response = get_request("/health")
           health_data = JSON.parse(response.read)
 
-          # Convert Typesense health format to match expected format
           {
             cluster_name: "typesense",
             status: health_data["ok"] ? "green" : "red",
@@ -392,30 +280,18 @@ module Noiseless
 
         def default_headers
           headers = super
-
-          # Add Typesense API key if configured
-          if @connection_params && @connection_params[:api_key]
-            headers << ["X-TYPESENSE-API-KEY",
-                        @connection_params[:api_key]]
-          end
-
+          headers << ["X-TYPESENSE-API-KEY", @connection_params[:api_key]] if @connection_params&.dig(:api_key)
           headers
         end
 
-        # rubocop:disable-next Lint/DuplicateBranch
         def map_type_to_typesense(elasticsearch_type)
-          # Map Elasticsearch types to Typesense types
           case elasticsearch_type
-          when "text", "keyword"
-            "string"
-          when "long", "integer", "short", "byte", "date"
-            "int64" # date uses Unix timestamps
-          when "double", "float", "half_float", "scaled_float"
-            "float"
-          when "boolean"
-            "bool"
-          else
-            "string" # Default to string for unknown types
+          when "long", "integer", "short", "byte", "date" then "int64"
+          when "double", "float", "half_float", "scaled_float" then "float"
+          when "boolean" then "bool"
+          when "dense_vector", "knn_vector" then "float[]"
+          when "keyword" then "string[]"
+          else "string"
           end
         end
       end