esse 0.4.0.rc4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b78836cea90af6c95e5bad1fec44c734733e21dab660cbb338a79546f2539d5
-  data.tar.gz: 0d1e073bbcc3ab1576d134ab4c70e2231767b836f59652f2f41b56e5943cc585
+  metadata.gz: dc6a554b787bfb51e5568bd590790171cc88ad8c5ce432e1de2c3ef68a5d1def
+  data.tar.gz: cac82a25d6e8dd6dad3667a734d58be098f4b7020db76472359f7bdb4fe2067c
 SHA512:
-  metadata.gz: 4118d2d6d2d163456f09c487b391abfb7522d4c1900e1517d94a8414b3f14587abf7e3fbb8292321933c767cb21b981c03f829ec2bd12f4cbc7327aa34cbe3e4
-  data.tar.gz: 80cdb7b8ccdb6ca158ca95207b0f9363b3edd19edddc6e9d29c168cb03ae04d520f110c752dd604b19665891cde2ce787db9eb18ca0119d155b90f1150527d38
+  metadata.gz: f26afad34715ed37cec3f1d1cad11ca6e05d2921a6f31884d9ea9f1136ef239ba7661e29796c37abf69ea19aa5f5407884cf88b108bbfa211e5639e371d492ed
+  data.tar.gz: 13b1921d85723d3c33bacc54c7bcf95885a196c3a62c095a5ea3be5a5d5061b8cd4eb8fbb1bdbf1bb91c141ed4457f9a1baab5ec121dd7e4f4dc1c1c423277d2

data/lib/esse/cli/index/update_lazy_attributes.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative 'base_operation'
+
+module Esse
+  module CLI
+    class Index::UpdateLazyAttributes < Index::BaseOperation
+      attr_reader :attributes
+
+      def initialize(indices:, attributes: nil, **options)
+        super(indices: indices, **options)
+        @attributes = Array(attributes)
+      end
+
+      def run
+        validate_options!
+        indices.each do |index|
+          repos = if (repo = @options[:repo])
+            [index.repo(repo)]
+          else
+            index.repo_hash.values
+          end
+
+          repos.each do |repo|
+            attrs = repo_attributes(repo)
+            next unless attrs.any?
+
+            repo.send(:each_batch_ids, **context_options) do |ids|
+              attrs.each do |attribute|
+                repo.update_documents_attribute(attribute, ids, bulk_options)
+              end
+            end
+          end
+        end
+      end
+
+      private
+
+      def bulk_options
+        @bulk_options ||= (@options[:bulk_options] || {}).transform_values do |value|
+          value.is_a?(String) ? Hstring.new(value).coerce_type : value
+        end
+      end
+
+      def context_options
+        @context_options ||= (@options[:context] || {}).transform_values do |value|
+          value.is_a?(String) ? Hstring.new(value).coerce_type : value
+        end
+      end
+
+      def validate_options!
+        validate_indices_option!
+      end
+
+      def repo_attributes(repo)
+        return repo.lazy_document_attribute_names(true) if attributes.empty?
+
+        repo.lazy_document_attribute_names(attributes)
+      end
+    end
+  end
+end

data/lib/esse/cli/index.rb

@@ -19,6 +19,10 @@ module Esse
       option :reindex, desc: 'Use _reindex API to import documents from the old index to the new index'
       option :optimize, type: :boolean, default: true, desc: 'Optimize index before import documents by disabling refresh_interval and setting number_of_replicas to 0'
       option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=refresh_interval:1s,number_of_replicas:0'
+
+      option :preload_lazy_attributes, type: :string, default: nil, desc: 'Command separated list of lazy document attributes to preload using search API before the bulk import. Or pass `true` to preload all lazy attributes'
+      option :eager_load_lazy_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to include to the bulk index request. Or pass `true` to include all lazy attributes'
+      option :update_lazy_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to bulk update after the bulk index request Or pass `true` to include all lazy attributes'
       def reset(*index_classes)
         require_relative 'index/reset'
         opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
@@ -27,6 +31,22 @@ module Esse
         if opts[:import] && opts[:reindex]
           raise ArgumentError, 'You cannot use --import and --reindex together'
         end
+
+        %i[preload_lazy_attributes eager_load_lazy_attributes update_lazy_attributes].each do |key|
+          val = opts.delete(key)
+          val = 'true' if val == key.to_s
+          next if val.nil? || val == 'false'
+
+          if opts[:reindex]
+            raise ArgumentError, "You cannot use --#{key}=#{val} with --reindex"
+          elsif opts[:import] == false
+            raise ArgumentError, "You cannot use --#{key}=#{val} with --import=false"
+          end
+
+          opts[:import] = {} if opts[:import] == true
+          opts[:import][key] = (val == 'true') ? true : val.split(',')
+        end
+
         Reset.new(indices: index_classes, **opts).run
       end
 
@@ -39,7 +59,7 @@ module Esse
       DESC
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :alias, type: :boolean, default: false, aliases: '-a', desc: 'Update alias after create index'
-      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=index.refresh_interval:-1,index.number_of_replicas:0'
+      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=index.refresh_interval:-1 index.number_of_replicas:0'
       def create(*index_classes)
         require_relative 'index/create'
         opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
@@ -66,7 +86,7 @@ module Esse
       desc 'update_settings *INDEX_CLASS', 'Closes the index for read/write operations, updates the index settings, and open it again'
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :type, type: :string, default: nil, aliases: '-t', desc: 'Document Type to update mapping for'
-      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=index.refresh_interval:-1,index.number_of_replicas:0'
+      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=index.refresh_interval:-1 index.number_of_replicas:0'
       def update_settings(*index_classes)
         require_relative 'index/update_settings'
         opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
@@ -107,11 +127,23 @@ module Esse
         opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
         %i[preload_lazy_attributes eager_load_lazy_attributes update_lazy_attributes].each do |key|
           if (val = opts.delete(key)) && val != 'false'
+            val = 'true' if val == key.to_s
             opts[key] = (val == 'true') ? true : val.split(',')
           end
         end
         Import.new(indices: index_classes, **opts).run
       end
+
+      desc "update_lazy_attributes INDEX_CLASS", "Async update lazy attributes for the given index"
+      option :repo, type: :string, default: nil, alias: "-r", desc: "Repository to use for import"
+      option :suffix, type: :string, default: nil, aliases: "-s", desc: "Suffix to append to index name"
+      option :context, type: :hash, default: {}, required: true, desc: "List of options to pass to the index class"
+      option :bulk_options, type: :hash, default: nil, desc: 'List of options to pass to the bulk update request. Example: --bulk-options=timeout:30s refresh:true retry_on_conflict:3'
+      def update_lazy_attributes(index_class, *attributes)
+        require_relative "index/update_lazy_attributes"
+        opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
+        UpdateLazyAttributes.new(indices: [index_class], attributes: attributes, **opts).run
+      end
     end
   end
 end

data/lib/esse/collection.rb

@@ -14,5 +14,11 @@ module Esse
     def each
       raise NotImplementedError, 'Override this method to iterate over the collection'
     end
+
+    # @yield [<Array>] A batch of document IDs to be processed.
+    # @abstract Override this method to yield each chunk of document IDs
+    def each_batch_ids
+      raise NotImplementedError, 'Override this method to iterate over the collection in batches of IDs'
+    end
   end
 end

data/lib/esse/document.rb

@@ -1,7 +1,23 @@
 # frozen_string_literal: true
 
+# Esse::Document is the base class for all documents in Esse.
+# It provides methods to access the document ID, type, routing, meta, and source.
+# It also provides methods to convert the document to a hash or bulk format.
+#
+# @example
+#   class UserDocument < Esse::Document
+#     def id
+#       object.id
+#     end
+#
+#     def source
+#       { name: object.name, email: object.email }
+#     end
+#   end
+
 module Esse
   class Document
+
     MUTATIONS_FALLBACK = {}.freeze
 
     attr_reader :object, :options
@@ -39,11 +55,6 @@ module Esse
       !routing.nil?
     end
 
-    # @TODO allow import, index, bulk to accept a suffix to tell which index to use
-    # def index_suffix
-    #   nil
-    # end
-
     # @return [Hash] the document meta
     # @abstract Override this method to return the document meta
     def meta
@@ -108,7 +119,7 @@ module Esse
       DocumentForPartialUpdate.new(self, source: source)
     end
 
-    def inspect
+    def to_s
       attributes = {id: :id, routing: :routing, source: :memoized_source}.map do |attr_name, attr_src|
         value = send(attr_src)
         next unless value

data/lib/esse/events/bus.rb

@@ -92,9 +92,14 @@ module Esse
 
       # @api private
       def process(event_id, payload)
+        # Build the event once and reuse for all listeners. Previously, a new
+        # Event was created per listener via Event#payload which does
+        # @payload.merge(data) — a full hash copy each time. With 24+
+        # esse-rails subscribers, each search was creating 24+ Event objects
+        # with 24+ hash merges of the full payload (including Response with
+        # the entire OpenSearch JSON). Now it's 1 Event + 1 merge total.
+        event = events[event_id].payload(payload)
         listeners[event_id].each do |listener|
-          event = events[event_id].payload(payload)
-
           yield(event, listener)
         end
       end

data/lib/esse/events/publisher.rb

@@ -60,6 +60,11 @@ module Esse
           if publish_event
             payload[:runtime] ||= Time.now - payload.delete(:__started_at__) if payload[:__started_at__]
             __bus__.publish(event_id, payload)
+            # Release references to large objects (Query, Response with full
+            # OpenSearch JSON) so they become GC-eligible immediately after
+            # event dispatch, rather than being held until the caller's stack
+            # frame unwinds.
+            payload.clear
           end
         end
 

data/lib/esse/events.rb

@@ -56,6 +56,7 @@ module Esse
     register_event 'elasticsearch.exist'
     register_event 'elasticsearch.count'
     register_event 'elasticsearch.get'
+    register_event 'elasticsearch.mget'
     register_event 'elasticsearch.reindex'
     register_event 'elasticsearch.update_by_query'
     register_event 'elasticsearch.delete_by_query'

data/lib/esse/import/bulk.rb

@@ -1,30 +1,6 @@
 module Esse
   module Import
     class Bulk
-      def self.build_from_documents(type: nil, index: nil, delete: nil, create: nil, update: nil)
-        index = Array(index).select(&Esse.method(:document?)).reject(&:ignore_on_index?).map do |doc|
-          value = doc.to_bulk
-          value[:_type] ||= type if type
-          value
-        end
-        create = Array(create).select(&Esse.method(:document?)).reject(&:ignore_on_index?).map do |doc|
-          value = doc.to_bulk
-          value[:_type] ||= type if type
-          value
-        end
-        update = Array(update).select(&Esse.method(:document?)).reject(&:ignore_on_index?).map do |doc|
-          value = doc.to_bulk(operation: :update)
-          value[:_type] ||= type if type
-          value
-        end
-        delete = Array(delete).select(&Esse.method(:document?)).reject(&:ignore_on_delete?).map do |doc|
-          value = doc.to_bulk(data: false)
-          value[:_type] ||= type if type
-          value
-        end
-        new(index: index, delete: delete, create: create, update: update)
-      end
-
       def initialize(index: nil, delete: nil, create: nil, update: nil)
         @index = Esse::ArrayUtils.wrap(index).map { |payload| { index: payload } }
         @create = Esse::ArrayUtils.wrap(create).map { |payload| { create: payload } }

data/lib/esse/index/documents.rb

@@ -29,6 +29,37 @@ module Esse
         cluster.api.get(**options)
       end
 
+      # Retrieves multiple JSON documents by ID from an index.
+      #
+      #   UsersIndex.mget(ids: [1, 2, 3])
+      #   UsersIndex.mget(ids: [Esse::HashDocument.new(id: 1), Esse::HashDocument.new(id: 2)])
+      #
+      # @param ids [Array<Esse::Document, Hash, String, Integer>] documents, hashes, or IDs to retrieve
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      # @return [Hash] The elasticsearch response with 'docs' array
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-multi-get.html
+      def mget(ids:, suffix: nil, **options)
+        options[:body] = {
+          docs: ids.map do |doc|
+            if document?(doc)
+              datum = { _id: doc.id }
+              datum[:_type] = doc.type if doc.type?
+              datum[:routing] = doc.routing if doc.routing?
+              datum
+            elsif doc.is_a?(Hash)
+              doc
+            else
+              { _id: doc }
+            end
+          end,
+        }
+        options[:index] = index_name(suffix: suffix)
+        cluster.may_update_type!(options)
+        cluster.api.mget(**options)
+      end
+
       # Check if a JSON document exists
       #
       #   UsersIndex.exist?(id: 1) # true
@@ -88,6 +119,7 @@ module Esse
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-delete.html
       def delete(doc = nil, suffix: nil, **options)
         if document?(doc)
+          options = request_params_for(:delete, doc).merge(options) if request_params_for?(:delete)
           options[:id] = doc.id
           options[:type] = doc.type if doc.type?
           options[:routing] = doc.routing if doc.routing?
@@ -113,6 +145,7 @@ module Esse
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-update.html
       def update(doc = nil, suffix: nil, **options)
         if document?(doc)
+          options = request_params_for(:update, doc).merge(options) if request_params_for?(:update)
           options[:id] = doc.id
           options[:body] = { doc: doc.mutated_source }
           options[:type] = doc.type if doc.type?
@@ -139,6 +172,7 @@ module Esse
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-index_.html
       def index(doc = nil, suffix: nil, **options)
         if document?(doc)
+          options = request_params_for(:index, doc).merge(options) if request_params_for?(:index)
           options[:id] = doc.id
           options[:body] = doc.mutated_source
           options[:type] = doc.type if doc.type?
@@ -181,6 +215,7 @@ module Esse
           elsif Esse.document?(doc) && !doc.ignore_on_index?
             hash = doc.to_bulk
             hash[:_type] ||= type if type
+            hash = request_params_for(:index, doc, bulk: true).merge(hash) if request_params_for?(:index)
             to_index << hash
           end
         end
@@ -190,6 +225,7 @@ module Esse
           elsif Esse.document?(doc) && !doc.ignore_on_index?
             hash = doc.to_bulk
             hash[:_type] ||= type if type
+            hash = request_params_for(:create, doc, bulk: true).merge(hash) if request_params_for?(:create)
             to_create << hash
           end
         end
@@ -199,6 +235,7 @@ module Esse
           elsif Esse.document?(doc) && !doc.ignore_on_index?
             hash = doc.to_bulk(operation: :update)
             hash[:_type] ||= type if type
+            hash = request_params_for(:update, doc, bulk: true).merge(hash) if request_params_for?(:update)
             to_update << hash
           end
         end
@@ -208,6 +245,7 @@ module Esse
           elsif Esse.document?(doc) && !doc.ignore_on_delete?
             hash = doc.to_bulk(data: false)
             hash[:_type] ||= type if type
+            hash = request_params_for(:delete, doc, bulk: true).merge(hash) if request_params_for?(:delete)
             to_delete << hash
           end
         end

data/lib/esse/index/indices.rb

@@ -30,12 +30,23 @@ module Esse
         options = CREATE_INDEX_RESERVED_KEYWORDS.merge(options)
         name = build_real_index_name(suffix)
         definition = body || [settings_hash(settings: settings), mappings_hash].reduce(&:merge)
+        index_alias = options.delete(:alias)
 
-        if options.delete(:alias) && name != index_name
+        if index_alias && name != index_name
           definition[:aliases] = { index_name => {} }
         end
 
-        cluster.api.create_index(index: name, body: definition, **options)
+        retried = false
+        begin
+          cluster.api.create_index(index: name, body: definition, **options)
+        rescue Esse::Transport::BadRequestError => e
+          if retried == false && e.message.include?('exists with the same name') && index_alias == :force
+            cluster.api.delete_index(index: index_name)
+            retried = true
+            retry
+          end
+          raise
+        end
       end
 
       # Deletes, creates and imports data to the index. Performs zero-downtime index resetting.
@@ -59,8 +70,8 @@ module Esse
         optimized_creation = optimize && syncronous_import && (import || reindex)
         if optimized_creation
           definition = [settings_hash(settings: settings), mappings_hash].reduce(&:merge)
-          number_of_replicas = definition.dig(Esse::SETTING_ROOT_KEY, :index, :number_of_replicas)
-          refresh_interval = definition.dig(Esse::SETTING_ROOT_KEY, :index, :refresh_interval)
+          number_of_replicas = definition.dig(Esse::SETTING_ROOT_KEY, :index, :number_of_replicas) || 1
+          refresh_interval = definition.dig(Esse::SETTING_ROOT_KEY, :index, :refresh_interval) || '1s'
           new_number_of_replicas = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:number_of_replicas] = 0
           new_refresh_interval = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:refresh_interval] = '-1'
           create_index(**options, suffix: suffix, alias: false, body: definition)
@@ -88,6 +99,10 @@ module Esse
         end
 
         if optimized_creation && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval
+          settings ||= {}
+          settings[:index] ||= {}
+          settings[:index][:number_of_replicas] = number_of_replicas
+          settings[:index][:refresh_interval] = refresh_interval
           update_settings(suffix: suffix, settings: settings)
           refresh(suffix: suffix)
         end

data/lib/esse/index/inheritance.rb

@@ -39,6 +39,7 @@ module Esse
           :@mapping => nil,
           :@cluster_id => :dup,
           :@plugins => :dup,
+          :@request_params => :dup,
         }
       end
     end

data/lib/esse/index/request_configurable.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    module RequestConfigurable
+      OPERATIONS = %i[index create update delete].freeze
+      BULK_OPERATIONS_AND_PARAMS = {
+        index: %i[_index _type routing if_primary_term if_seq_no version version_type dynamic_templates pipeline require_alias],
+        create: %i[_index _type routing if_primary_term if_seq_no version version_type dynamic_templates pipeline require_alias],
+        update: %i[_index _type routing if_primary_term if_seq_no version version_type require_alias retry_on_conflict],
+        delete: %i[_index _type routing if_primary_term if_seq_no version version_type],
+      }.freeze
+
+      def self.extended(base)
+        base.extend DSL
+      end
+
+      class RequestParams
+        attr_reader :operation, :hash, :block
+
+        def initialize(operation, hash = {}, &block)
+          @operation = operation
+          @hash = hash.transform_keys(&:to_sym)
+          @block = block
+        end
+
+        # @param doc [Esse::Document] the document to apply the request parameters to
+        # @return [Hash] the request parameters for the operation
+        # @raise [ArgumentError] if the result of the block is not a Hash
+        def call(doc)
+          return hash unless block
+
+          result = block.call(doc) || {}
+          raise ArgumentError, "Expected a Hash, got #{result.class}" unless result.is_a?(Hash)
+
+          hash.merge(result.transform_keys(&:to_sym))
+        end
+      end
+
+      class Container
+        def initialize
+          @mutex = Mutex.new
+          @entries = {}.freeze
+        end
+
+        def add(operation, entry)
+          @mutex.synchronize do
+            hash = @entries.dup
+            arr = (hash[operation] || []).dup
+            arr << entry
+            hash[operation] = arr.freeze
+            @entries = hash.freeze
+          end
+        end
+
+        def key?(operation)
+          @entries.key?(operation)
+        end
+
+        def retrieve(operation, doc)
+          return {} unless @entries[operation]
+
+          @entries[operation].each_with_object({}) do |entry, hash|
+            hash.merge!(entry.call(doc))
+          end
+        end
+      end
+
+      module DSL
+        def request_params(*operations, **params, &block)
+          operations.each do |operation|
+            raise ArgumentError, "Invalid operation: #{operation}" unless OPERATIONS.include?(operation)
+
+            @request_params ||= Container.new
+            @request_params.add(operation, RequestParams.new(operation, params, &block))
+          end
+
+          self
+        end
+
+        def request_params_for(operation, doc, bulk: false)
+          return {} unless request_params_for?(operation)
+
+          params = @request_params.retrieve(operation, doc)
+
+          if bulk && BULK_OPERATIONS_AND_PARAMS.key?(operation)
+            params.slice(*BULK_OPERATIONS_AND_PARAMS[operation])
+          else
+            params
+          end
+        end
+
+        def request_params_for?(operation)
+          return false unless @request_params
+
+          @request_params.key?(operation)
+        end
+      end
+    end
+
+    extend RequestConfigurable
+  end
+end

data/lib/esse/index/settings.rb

@@ -11,6 +11,7 @@ module Esse
         number_of_shards
         number_of_replicas
         refresh_interval
+        mapping
       ].freeze
 
       def settings_hash(settings: nil)
@@ -22,9 +23,17 @@ module Esse
         end
         INDEX_SIMPLIFIED_SETTINGS.each do |key|
           next unless values.key?(key)
+          value = values.delete(key)
+          next if value.nil?
 
-          (values[:index] ||= {}).merge!(key => values.delete(key))
+          (values[:index] ||= {}).merge!(key => value)
         end
+
+        if values[:index].is_a?(Hash)
+          INDEX_SIMPLIFIED_SETTINGS.each { |key| values[:index].delete(key) if values[:index][key].nil? }
+          values.delete(:index) if values[:index].empty?
+        end
+
         { Esse::SETTING_ROOT_KEY => values }
       end
 

data/lib/esse/index.rb

@@ -23,5 +23,6 @@ module Esse
     require_relative 'index/indices'
     require_relative 'index/search'
     require_relative 'index/documents'
+    require_relative 'index/request_configurable'
   end
 end

data/lib/esse/primitives/hstring.rb

@@ -82,5 +82,19 @@ module Esse
       @value
     end
     def_conventional :presence!
+
+    def coerce_type
+      if @value =~ /\A-?\d+\z/
+        return @value.to_i
+      elsif @value =~ /\A-?\d+\.\d+\z/
+        return @value.to_f
+      elsif @value == 'true'
+        return true
+      elsif @value == 'false'
+        return false
+      end
+
+      @value
+    end
   end
 end

data/lib/esse/repository/documents.rb

@@ -10,6 +10,7 @@ module Esse
       def update_documents_attribute(name, ids_or_doc_headers = [], kwargs = {})
         batch = documents_for_lazy_attribute(name, ids_or_doc_headers)
         return if batch.empty?
+
         kwargs = kwargs.transform_keys(&:to_sym)
 
         if kwargs.delete(:index_on_missing) { true }

data/lib/esse/repository/object_document_mapper.rb

@@ -157,6 +157,66 @@ module Esse
         @document_proc.call(model, **kwargs)
       end
 
+      # Used to fetch batches of ids from the collection that implement the `each_batch_ids` method.
+      #
+      # @param [Hash] kwargs The context
+      # @yield [Array] A batch of document IDs to be processed.
+      # @raise [NotImplementedError] if the collection does not implement the `each_batch_ids` method.
+      # @raise [NotImplementedError] if the collection is not defined.
+      # @return [Enumerator] The enumerator
+      # @example
+      #   each_batch_ids(active: true) do |ids|
+      #     puts ids.size
+      #   end
+      def each_batch_ids(*args, **kwargs)
+        if @collection_proc.nil?
+          raise NotImplementedError, format('there is no %<t>p collection defined for the %<k>p index', t: repo_name, k: index.to_s)
+        end
+
+        if @collection_proc.is_a?(Class) && @collection_proc.method_defined?(:each_batch_ids)
+          colection_instance = @collection_proc.new(*args, **kwargs)
+          if block_given?
+            colection_instance.each_batch_ids { |ids| yield ids }
+          else
+            Enumerator.new do |yielder|
+              colection_instance.each_batch_ids { |ids| yielder.yield ids }
+            end
+          end
+        else
+          Kernel.warn(<<~MSG)
+            The public `#each_batch_ids' method is not available for the collection defined in the #{repo_name} index.
+
+            The `#each' method will be used instead, which may lead to performance degradation because it serializes the entire document
+            to only obtain the IDs. Consider implementing a public `#each_batch_ids' method in your collection class for better performance.
+
+            Example implementation taking into account you are dealing with an ActiveRecord model:
+              class UserCollection < Esse::Collection
+                # ....
+
+                def each_batch_ids
+                  user_query.except(:includes, :preload, :eager_load).in_batches do |batch|
+                    yield batch.pluck(:id)
+                  end
+                end
+              end
+          MSG
+
+          enumerator = Enumerator.new do |yielder|
+            each_batch(*args, **kwargs) do |*batch_args|
+              batch, collection_context = batch_args
+              collection_context ||= {}
+              ids = [*batch].map { |entry| serialize(entry, **collection_context)&.id }.compact
+              yielder.yield(ids) if ids.any?
+            end
+          end
+          return enumerator unless block_given?
+
+          enumerator.each { |ids| yield ids }
+        end
+      rescue LocalJumpError
+        raise(SyntaxError, 'block must be explicitly declared in the collection definition')
+      end
+
       protected
 
       def coerce_to_document(value)

data/lib/esse/search/query.rb

@@ -40,7 +40,7 @@ module Esse
 
       def scroll_hits(batch_size: 1_000, scroll: '1m')
         response = execute_search_query!(size: batch_size, scroll: scroll)
-        scroll_id = nil
+        scroll_id = response.raw_response['scroll_id'] || response.raw_response['_scroll_id']
         fetched = 0
         total = response.total
 
@@ -48,9 +48,9 @@ module Esse
           fetched += response.hits.size
           yield(response.hits) if response.hits.any?
           break if fetched >= total
-          scroll_id = response.raw_response['scroll_id'] || response.raw_response['_scroll_id']
           break unless scroll_id
           response = execute_scroll_query(scroll: scroll, scroll_id: scroll_id)
+          scroll_id = response.raw_response['scroll_id'] || response.raw_response['_scroll_id']
         end
       ensure
         begin
@@ -59,12 +59,40 @@ module Esse
         end
       end
 
+      def search_after_hits(batch_size: 1_000)
+        body = HashUtils.deep_dup(definition.fetch(:body, {}))
+        body[:size] = batch_size
+        body.delete(:from)
+        body.delete('from')
+
+        unless body.key?(:sort) || body.key?('sort')
+          raise ArgumentError, 'The query body must include a :sort to use search_after'
+        end
+
+        loop do
+          response = execute_search_query!(body: body)
+          break if response.hits.empty?
+
+          yield(response.hits)
+
+          last_sort = response.hits.last['sort']
+          break unless last_sort
+          break if response.hits.size < batch_size
+
+          body[:search_after] = last_sort
+        end
+      end
+
+      def reset!
+        @response = nil
+      end
+
       private
 
       def execute_search_query!(**execution_options)
         resp, err = nil
         Esse::Events.instrument('elasticsearch.execute_search_query') do |payload|
-          payload[:query] = self
+          payload[:query_definition] = definition
           begin
             resp = Response.new(self, transport.search(**definition, **execution_options))
           rescue => e
@@ -81,7 +109,7 @@ module Esse
       def execute_scroll_query(scroll:, scroll_id:)
         resp, err = nil
         Esse::Events.instrument('elasticsearch.execute_search_query') do |payload|
-          payload[:query] = self
+          payload[:query_definition] = definition
           begin
             resp = Response.new(self, transport.scroll(scroll: scroll, body: { scroll_id: scroll_id }))
           rescue => e
@@ -94,10 +122,6 @@ module Esse
 
         resp
       end
-
-      def reset!
-        @response = nil
-      end
     end
   end
 end

data/lib/esse/search/response.rb

@@ -7,13 +7,13 @@ module Esse
       extend Forwardable
 
       def_delegators :hits, :each, :size, :empty?
-      attr_reader :query, :raw_response, :options
+      attr_reader :query_definition, :raw_response, :options
 
-      # @param [Esse::Search::Query] query The search query
+      # @param [Esse::Search::Query, Hash] query The search query or its definition hash
       # @param [Hash] raw_response The raw response from Elasticsearch
       # @param [Hash] options The options passed to the search
       def initialize(query, raw_response, **options)
-        @query = query
+        @query_definition = query.is_a?(Hash) ? query : query.definition
         @raw_response = raw_response
         @options = options
       end

data/lib/esse/transport/documents.rb

@@ -29,6 +29,30 @@ module Esse
         end
       end
 
+      # Allows to get multiple documents in a single request.
+      #
+      # @option [String] :index The name of the index
+      # @option [Hash] :body Document identifiers; can be either `docs` (specifying full document information)
+      #   or `ids` (when index is provided). (*Required*)
+      # @option [String] :preference Specify the node or shard the operation should be performed on (default: random)
+      # @option [Boolean] :realtime Specify whether to perform the operation in realtime or search mode
+      # @option [Boolean] :refresh Refresh the shard containing the document before performing the operation
+      # @option [String] :routing Specific routing value
+      # @option [List] :stored_fields A comma-separated list of stored fields to return in the response
+      # @option [List] :_source True or false to return the _source field or not, or a list of fields to return
+      # @option [List] :_source_excludes A list of fields to exclude from the returned _source field
+      # @option [List] :_source_includes A list of fields to extract and return from the _source field
+      # @option [Hash] :headers Custom HTTP headers
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html
+      #
+      def mget(index:, body:, **options)
+        Esse::Events.instrument('elasticsearch.mget') do |payload|
+          payload[:request] = opts = options.merge(index: index, body: body)
+          payload[:response] = coerce_exception { client.mget(**opts) }
+        end
+      end
+
       # Returns information about whether a document exists in an index.
       #
       # @option [String] :id The document ID

data/lib/esse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Esse
-  VERSION = '0.4.0.rc4'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: esse
 version: !ruby/object:Gem::Version
-  version: 0.4.0.rc4
+  version: 0.4.0
 platform: ruby
 authors:
 - Marcos G. Zimmermann
 autorequire:
 bindir: exec
 cert_chain: []
-date: 2024-10-07 00:00:00.000000000 Z
+date: 2026-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -226,6 +226,7 @@ files:
 - lib/esse/cli/index/open.rb
 - lib/esse/cli/index/reset.rb
 - lib/esse/cli/index/update_aliases.rb
+- lib/esse/cli/index/update_lazy_attributes.rb
 - lib/esse/cli/index/update_mapping.rb
 - lib/esse/cli/index/update_settings.rb
 - lib/esse/cli/parser/bool_or_hash.rb
@@ -272,6 +273,7 @@ files:
 - lib/esse/index/mappings.rb
 - lib/esse/index/object_document_mapper.rb
 - lib/esse/index/plugins.rb
+- lib/esse/index/request_configurable.rb
 - lib/esse/index/search.rb
 - lib/esse/index/settings.rb
 - lib/esse/index/type.rb
@@ -317,14 +319,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Pure Ruby and framework-agnostic ElasticSearch/OpenSearch toolkit for building