noiseless 0.7.2 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a2305879802aa874c213e1a5a87707ec7222af979d21834f21f4fd2c994cb48
-  data.tar.gz: 065cf12ecb097ac17ee1e33d99d5df0ccb79ca4ea6713640f82f6bcac2fad51a
+  metadata.gz: 195219b6de1210cb1f888a5a322e41c624403b7cab4f07cbd700ca9f5f573196
+  data.tar.gz: b92048b781e491ae6c8f0d9114dbf4a4094ba04b7e06bf8394c3a6164e18521a
 SHA512:
-  metadata.gz: dc5302df3e73a9a413e1a7382db1334e11dd28ee2564bb5ae9e0f5d6a2d62f7b241584463678ef6f0db6112740c70d5687fdc425903979056c9d61ff10140bc6
-  data.tar.gz: baba1a26426413fef74f6f900d3146aa9c7bcda385030dd3b722e8277c5dc5b8a25bb79adbf6dc9d0c695e4d3b43e4353e4fe99e3bffb7ec5fc5a0d18ff566cf
+  metadata.gz: afc592e3f9e505ff2272845eb4b8cddaae23543400fe83b1c11c70895fd0b8a537e670c1cc78103df93999f72f51cc0cdb8c008772a59e1e881df024e5a9cf24
+  data.tar.gz: 7b4a4113ea30314fbed156161dfa031344881ba17f422ac233cd5a92b91f2b63ff4f7e07282e3a993e98bf42d48d87813b8c4e3aea23ece3365695a1779b382e

data/README.md

@@ -6,7 +6,7 @@ Async-first search abstraction for Rails with multi-backend support (OpenSearch,
 
 - **Chainable DSL** — fluent query builder with runtime validation
 - **Multi-backend** — OpenSearch, Elasticsearch, Typesense, PostgreSQL adapters
-- **Async-first** — built on Ruby 3.4+ fiber scheduler with non-blocking I/O
+- **Async-first** — built on Ruby 4.0+ fiber scheduler with non-blocking I/O
 - **HTTP/2 connection pooling** — persistent connections via `Async::Pool`
 - **Rails integration** — Railtie with log subscriber and controller runtime tracking
 - **Lazy loading** — adapters loaded on-demand, test files excluded from production
@@ -17,7 +17,7 @@ Async-first search abstraction for Rails with multi-backend support (OpenSearch,
 gem "noiseless"
 ```
 
-`noiseless` is a Rails gem. It requires Ruby >= 3.4 and Rails >= 8.1.
+`noiseless` is a Rails gem. It requires Ruby >= 4.0 and Rails >= 8.1.
 
 ## Configuration
 
@@ -114,6 +114,51 @@ end
 
 For best performance, run independent searches concurrently within a single `Async` block rather than creating separate `Sync` blocks per search.
 
+### Loading Database Records
+
+Search classes return search results by default. To load database records, configure
+the record model explicitly and request `response_type: :records`:
+
+```ruby
+class Company::Search < Noiseless::Model
+  index_name 'companies'
+  record_class 'Company' # A Class is also accepted; strings resolve lazily.
+end
+
+records = Company::Search.search_sync(response_type: :records) do |s|
+  s.match(:name, 'tech')
+end
+records.to_a # Company records in search relevance order
+
+# The instance API supports the same option.
+Company::Search.new.match(:name, 'tech').execute_sync(response_type: :records)
+```
+
+`record_class` is inherited and does not change the default response type. Index
+names are not used to infer record classes: aliases and versioned indexes need
+not match table names. Requesting `:records` without a model supporting `.where`
+raises `ArgumentError`, even when the search has no hits. Records missing from
+the database are omitted; search totals still describe backend hits.
+
+### Pagination and Multi-model Search
+
+The paginator executes lazily and waits for its response when results or totals
+are accessed. Multi-search also waits and returns a `MultiSearchResponse`, not
+an `Async::Task`. Both work inside an existing async context.
+
+```ruby
+page = Company::Search.page(2).per(10)
+page.to_a
+page.total_count
+
+response = Noiseless.multi_search(models: [Company::Search, Product::Search]) do |s|
+  s.match(:name, 'tech')
+  s.limit(10)
+end
+response.results_for_model(Company::Search)
+response.records_for_model(Company::Search) # Uses its configured record_class.
+```
+
 ### Advanced Queries
 
 ```ruby
@@ -179,6 +224,15 @@ end
 
 ### Running Tests Locally
 
+Run the pagination, multi-search, and record-loading unit tests without services:
+
+```bash
+bundle exec ruby -Itest -Ilib -e 'ARGV.each { require_relative it }' \
+  test/{search_pagination,multi_search,record_response,active_record_response,postgresql_search_contract,query_contract}_test.rb
+```
+
+For the full integration suite:
+
 ```bash
 docker compose up -d postgres elasticsearch opensearch typesense
 bin/test

data/lib/noiseless/adapter.rb

@@ -2,7 +2,6 @@
 
 require "async"
 require "json"
-require_relative "introspection"
 
 module Noiseless
   class Adapter
@@ -127,6 +126,8 @@ module Noiseless
 
     private
 
+    def instance_variables_to_inspect = instance_variables - %i[@connection_params]
+
     # Convert AST to Hash - override in subclasses for adapter-specific format
     def ast_to_hash(ast_node)
       result = {}
@@ -153,14 +154,8 @@ module Noiseless
       # Aggregations
       result[:aggs] = build_aggregations_hash(ast_node.aggregations) if ast_node.aggregations.any?
 
-      # Vector/kNN search (OpenSearch/Elasticsearch compatible)
-      result[:knn] = build_knn_query(ast_node.vector) if ast_node.vector_search?
-
-      # Hybrid search (combines text + vector with RRF or weighted scoring)
-      if ast_node.hybrid_search?
-        hybrid_config = build_hybrid_query(ast_node.hybrid)
-        result.merge!(hybrid_config)
-      end
+      result[:knn] = build_knn_query(ast_node.vector, filters: result.dig(:query, :bool, :filter)) if ast_node.vector_search?
+      result[:query] = build_hybrid_query(ast_node.hybrid, result[:query]) if ast_node.hybrid_search?
 
       # Search pipeline (OpenSearch only)
       result[:search_pipeline] = ast_node.pipeline if ast_node.has_pipeline?
@@ -168,36 +163,27 @@ module Noiseless
       result
     end
 
-    def build_knn_query(vector_node)
+    def build_knn_query(vector_node, filters: nil)
       {
         field: vector_node.field.to_s,
-        query_vector: vector_node.embedding,
+        query_vector: encode_vector(vector_node.embedding),
         k: vector_node.k,
-        num_candidates: vector_node.k * 10
-      }
+        filter: filters.presence
+      }.compact
     end
 
-    # Build hybrid query using RRF (Reciprocal Rank Fusion) for OpenSearch/Elasticsearch
-    def build_hybrid_query(hybrid_node)
-      {
-        query: {
-          bool: {
-            should: [
-              {
-                match: {
-                  _all: hybrid_node.text_query
-                }
-              }
-            ]
-          }
-        },
-        knn: build_knn_query(hybrid_node.vector),
-        rank: {
-          rrf: {
-            window_size: hybrid_node.vector.k * 2
-          }
-        }
-      }
+    def build_hybrid_query(hybrid_node, query)
+      bool = query&.fetch(:bool, nil)&.dup || {}
+      text = { multi_match: { query: hybrid_node.text_query, fields: hybrid_node.fields.presence, boost: hybrid_node.text_weight }.compact }
+      vector = { knn: build_knn_query(hybrid_node.vector, filters: bool[:filter]).merge(boost: hybrid_node.vector_weight) }
+
+      { bool: bool.merge(should: [text, vector], minimum_should_match: 1) }
+    end
+
+    def encode_vector(values) = [values.pack("g*")].pack("m0")
+
+    def encode_document(document)
+      document.to_h.transform_values { it.is_a?(Noiseless::Embedding) ? encode_vector(it.values) : it }
     end
 
     def build_query_hash(bool_node)
@@ -241,12 +227,7 @@ module Noiseless
     end
 
     def build_pagination_hash(paginate_node)
-      return { from: 0, size: 20 } unless paginate_node
-
-      {
-        from: (paginate_node.page - 1) * paginate_node.per_page,
-        size: paginate_node.per_page
-      }
+      Pagination.query_options(paginate_node)
     end
 
     # Parses a backend HTTP response, raising when the backend reported an error.
@@ -262,7 +243,7 @@ module Noiseless
       rescue JSON::ParserError, TypeError
         nil
       end
-      error = payload.is_a?(Hash) ? payload["error"] : nil
+      error = payload.is_a?(Hash) ? payload["error"] || payload["message"] : nil
       reason = if error.is_a?(Hash)
                  [error["type"], error["reason"]].compact.join(": ")
                elsif error
@@ -274,6 +255,13 @@ module Noiseless
       raise error_class.new(message, status: response.status, error_type: error.is_a?(Hash) ? error["type"] : nil)
     end
 
+    def exists_response?(response, context:)
+      return true if response.success?
+      return false if response.status == 404
+
+      raise Noiseless::RequestError.new("#{context}: HTTP #{response.status}", status: response.status)
+    end
+
     # Override in subclasses
     def execute_search(_query_hash, **_opts)
       {

data/lib/noiseless/adapters/elasticsearch.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "execution_modules/elasticsearch_execution"
 
 module Noiseless
   module Adapters

data/lib/noiseless/adapters/execution_modules/elasticsearch_execution.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "json"
-require_relative "es_compatible_execution"
 
 module Noiseless
   module Adapters
@@ -13,7 +12,7 @@ module Noiseless
 
         def execute_search(query_hash, indexes: [], **_opts)
           path = indexes.any? ? "/#{indexes.join(',')}/_search" : "/_search"
-          body = JSON.generate(query_hash)
+          body = dump_json(query_hash)
 
           response = post_request(path, body)
           parse_json_response!(response, error_class: Noiseless::SearchError, context: "search")
@@ -26,15 +25,15 @@ module Noiseless
           body[:mappings] = mappings if mappings
           body[:settings] = settings if settings
 
-          response = put_request("/#{index_name}", body.any? ? JSON.generate(body) : nil)
+          response = put_request("/#{index_name}", body.any? ? dump_json(body) : nil)
           parse_json_response!(response, context: "create index #{index_name}")
         ensure
           response&.close
         end
 
-        def execute_index_document(index, id, document, **_opts)
-          path = id ? "/#{index}/_doc/#{id}" : "/#{index}/_doc"
-          body = JSON.generate(document)
+        def execute_index_document(index, id, document, refresh: nil, **_opts)
+          path = (id ? "/#{index}/_doc/#{id}" : "/#{index}/_doc") + query_string(refresh:)
+          body = dump_json(encode_document(document))
 
           response = id ? put_request(path, body) : post_request(path, body)
           parse_json_response!(response, context: "index document #{index}/#{id}")
@@ -42,6 +41,9 @@ module Noiseless
           response&.close
         end
 
+        def pit_path = "_pit"
+        def pit_close_body(pit_id) = { id: pit_id }
+
         def execute_cluster_health(**_opts)
           response = get_request("/_cluster/health")
           JSON.parse(response.read)

data/lib/noiseless/adapters/execution_modules/es_compatible_execution.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "json"
-require_relative "http_transport"
 
 module Noiseless
   module Adapters
@@ -11,20 +10,67 @@ module Noiseless
       module EsCompatibleExecution
         include HttpTransport
 
+        # Pages through every hit of a query with a point in time and
+        # search_after, which unlike from/size has no depth limit and sees a
+        # consistent snapshot. The point in time is always closed.
+        def each_page(ast_node, keep_alive: "1m", model_class: nil, response_type: nil)
+          return enum_for(__method__, ast_node, keep_alive:, model_class:, response_type:) unless block_given?
+
+          Sync do
+            query = ast_to_hash(ast_node).except(:from)
+            query[:sort] = [*query[:sort], { _shard_doc: "asc" }]
+            pit_id = execute_open_pit(ast_node.indexes, keep_alive:)
+
+            loop do
+              raw = execute_pit_search(query.merge(pit: { id: pit_id, keep_alive: }))
+              hits = raw.dig("hits", "hits")
+              break if hits.empty?
+
+              yield ResponseFactory.create(raw, model_class:, response_type:)
+              pit_id = raw["pit_id"] || pit_id
+              query[:search_after] = hits.last["sort"]
+            end
+          ensure
+            execute_close_pit(pit_id) if pit_id
+          end
+        end
+
         private
 
-        def execute_bulk(actions, **_opts)
+        def execute_open_pit(indexes, keep_alive:)
+          response = post_request("/#{indexes.join(',')}/#{pit_path}?keep_alive=#{keep_alive}", nil)
+          parse_json_response!(response, context: "open point in time").then { it["pit_id"] || it["id"] }
+        ensure
+          response&.close
+        end
+
+        def execute_close_pit(pit_id)
+          response = delete_request("/#{pit_path}", dump_json(pit_close_body(pit_id)))
+          parse_json_response!(response, context: "close point in time")
+        ensure
+          response&.close
+        end
+
+        def execute_pit_search(query_hash)
+          response = post_request("/_search", dump_json(query_hash))
+          parse_json_response!(response, error_class: Noiseless::SearchError, context: "point-in-time search")
+        ensure
+          response&.close
+        end
+
+        def execute_bulk(actions, refresh: nil, **_opts)
           body = actions.map do |action|
             if action[:index]
               action_line = { index: { _index: action[:index][:_index], _id: action[:index][:_id] } }
-              data_line = action[:index][:data]
-              "#{JSON.generate(action_line)}\n#{JSON.generate(data_line)}\n"
+              data_line = encode_document(action[:index][:data])
+              "#{dump_json(action_line)}\n#{dump_json(data_line)}\n"
             else
-              "#{JSON.generate(action)}\n"
+              "#{dump_json(action)}\n"
             end
           end.join
 
-          response = post_request("/_bulk", body, content_type: "application/x-ndjson")
+          path = "/_bulk#{query_string(refresh:, filter_path: 'errors,items.*.status,items.*.error')}"
+          response = post_request(path, body, content_type: "application/x-ndjson")
           parse_json_response!(response, context: "bulk")
         ensure
           response&.close
@@ -41,6 +87,12 @@ module Noiseless
           response&.close
         end
 
+        def query_string(refresh: nil, **params)
+          params = params.compact
+          params[:refresh] = refresh if refresh
+          params.empty? ? "" : "?#{params.map { |key, value| "#{key}=#{value}" }.join('&')}"
+        end
+
         def execute_refresh_index(index_name)
           response = post_request("/#{index_name}/_refresh", nil)
           parse_json_response!(response, context: "refresh index #{index_name}")
@@ -50,24 +102,22 @@ module Noiseless
 
         def execute_index_exists?(index_name)
           response = head_request("/#{index_name}")
-          response.success?
-        rescue StandardError
-          false
+          exists_response?(response, context: "index exists #{index_name}")
         ensure
           response&.close
         end
 
-        def execute_update_document(index, id, changes, **_opts)
-          body = JSON.generate(doc: changes)
+        def execute_update_document(index, id, changes, refresh: nil, **_opts)
+          body = dump_json(doc: encode_document(changes))
 
-          response = post_request("/#{index}/_update/#{id}", body)
+          response = post_request("/#{index}/_update/#{id}#{query_string(refresh:)}", body)
           parse_json_response!(response, context: "update document #{index}/#{id}")
         ensure
           response&.close
         end
 
-        def execute_delete_document(index, id, **_opts)
-          response = delete_request("/#{index}/_doc/#{id}")
+        def execute_delete_document(index, id, refresh: nil, **_opts)
+          response = delete_request("/#{index}/_doc/#{id}#{query_string(refresh:)}")
           # 404 covers both a missing document and a missing index; either way
           # the delete is idempotent.
           return { "_index" => index, "_id" => id, "result" => "not_found" } if response.status == 404
@@ -79,9 +129,7 @@ module Noiseless
 
         def execute_document_exists?(index, id)
           response = head_request("/#{index}/_doc/#{id}")
-          response.success?
-        rescue StandardError
-          false
+          exists_response?(response, context: "document exists #{index}/#{id}")
         ensure
           response&.close
         end

data/lib/noiseless/adapters/execution_modules/http_transport.rb

@@ -19,7 +19,8 @@ module Noiseless
           SocketError,
           IOError,
           IO::TimeoutError,
-          Timeout::Error
+          Timeout::Error,
+          Protocol::HTTP::Error
         ].freeze
 
         # Default per-operation IO timeout (seconds) for the search backend.
@@ -39,13 +40,29 @@ module Noiseless
         # deadline (long bulk imports may need a higher value or nil).
         DEFAULT_REQUEST_TIMEOUT = 30
 
-        BufferedResponse = Struct.new(:status, :body) do
+        # Default cap on concurrent connections per host. Without it the pool
+        # opens one socket per concurrent request. Waiting for a free
+        # connection counts against request_timeout. Override per-connection
+        # with +pool_limit:+.
+        DEFAULT_POOL_LIMIT = 16
+
+        # json 3 serializes objects it does not know with to_s, so a Time
+        # became "2025-12-11 14:18:40 UTC", which search engines reject.
+        # Falling back to as_json keeps ActiveSupport's ISO 8601 times.
+        JSON_CODER = JSON::Coder.new { |object| object.respond_to?(:as_json) ? object.as_json : object.to_s }
+
+        BufferedResponse = Data.define(:status, :body) do
           def read = body
           def success? = (200..299).cover?(status)
           def close = nil
         end
 
-        def initialize(hosts: [], timeout: DEFAULT_TIMEOUT, request_timeout: DEFAULT_REQUEST_TIMEOUT, **connection_params)
+        def initialize(hosts: [], timeout: DEFAULT_TIMEOUT, request_timeout: DEFAULT_REQUEST_TIMEOUT,
+                       pool_limit: DEFAULT_POOL_LIMIT, **connection_params)
+          unless request_timeout.nil? || (request_timeout.is_a?(Numeric) && request_timeout.positive? && request_timeout.finite?)
+            raise ArgumentError, "request_timeout must be a positive finite number or nil, got #{request_timeout.inspect}"
+          end
+
           # Ensure we always have at least one host
           hosts_array = Array(hosts)
           @hosts = hosts_array.empty? ? ["http://localhost:#{default_port}"] : hosts_array
@@ -59,7 +76,7 @@ module Noiseless
           @clients = {}
           @hosts.each do |host|
             endpoint = Async::HTTP::Endpoint.parse(host, timeout: @timeout)
-            @clients[host] = Async::HTTP::Client.new(endpoint)
+            @clients[host] = Async::HTTP::Client.new(endpoint, limit: pool_limit)
           end
 
           super(hosts: @hosts, **connection_params)
@@ -78,27 +95,19 @@ module Noiseless
           end
         end
 
-        def post_request(path, body, content_type: "application/json")
-          headers = body ? default_headers + [["content-type", content_type]] : default_headers
+        def post_request(path, body, content_type: "application/json") = body_request(:post, path, body, content_type)
+        def put_request(path, body, content_type: "application/json") = body_request(:put, path, body, content_type)
+        def patch_request(path, body, content_type: "application/json") = body_request(:patch, path, body, content_type)
 
-          with_client do |client|
-            client.post(path, headers, body)
-          end
-        end
-
-        def put_request(path, body, content_type: "application/json")
+        def body_request(verb, path, body, content_type)
           headers = body ? default_headers + [["content-type", content_type]] : default_headers
 
           with_client do |client|
-            client.put(path, headers, body)
+            client.public_send(verb, path, headers, body)
           end
         end
 
-        def delete_request(path)
-          with_client do |client|
-            client.delete(path, default_headers)
-          end
-        end
+        def delete_request(path, body = nil) = body_request(:delete, path, body, "application/json")
 
         def head_request(path)
           with_client do |client|
@@ -146,6 +155,8 @@ module Noiseless
                 "search backend unreachable#{location} (#{e.class}: #{e.message})"
         end
 
+        def dump_json(object) = JSON_CODER.dump(object)
+
         def default_headers
           [
             ["accept", "application/json"],