esse 0.2.6 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c4a9e11bda74a05216a06279503d853bccc254354ab41d41080a1b19da2058d
-  data.tar.gz: ef29051789f406601d0bf0df066f22091cb096403c5acc7eedffcbe3b91a54bf
+  metadata.gz: 46abc9b3347ca852c1f270910e21582c8f17186b554d1a7f667bafee8f84a152
+  data.tar.gz: baf633b7147e8cf48c704c15fc5f4a2a52c8d08f671f05e6c5577f3d92ef2901
 SHA512:
-  metadata.gz: 9b23aa1caa4cd2d9b71d9fca7d4fce63740d02d564ffb27d056bfbf4fd868595e4207fde138476a5837d99a80bb00bcc90198bf1d69d3230b5e24af1565742f9
-  data.tar.gz: 034e4b02602c55b52ac261ff07b30000b7448eac3ec55531dacf426eefeebd6b9e56bc6381731ff06a864803ddff81e93af4215491a29dacf36eb230f7472f8c
+  metadata.gz: 129c4475225760da6620d990a1065795d6c7f1248b01745105715b8d01f4c11c500b253701f573eac5071abeeeaed83e6be1c5bf8f22b5f2fab819587c96c595
+  data.tar.gz: 93f66e7d04a3857c3daf9b6daaa72a39e6a3027b80e464e53a97325f74666324451f93a914d9daa31864a77a93c7fb37735e12395536f395133135042a752f21

data/lib/esse/cli/index.rb

@@ -88,9 +88,20 @@ module Esse
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :context, type: :hash, default: {}, required: true, desc: 'List of options to pass to the index class'
       option :repo, type: :string, default: nil, alias: '-r', desc: 'Repository to use for import'
+      option :eager_include_document_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to include to the bulk index request'
+      option :lazy_update_document_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to bulk update after the bulk index request'
       def import(*index_classes)
         require_relative 'index/import'
-        Import.new(indices: index_classes, **HashUtils.deep_transform_keys(options.to_h, &:to_sym)).run
+        opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
+        opts.delete(:lazy_update_document_attributes) if opts[:lazy_update_document_attributes] == 'false'
+        opts.delete(:eager_include_document_attributes) if opts[:eager_include_document_attributes] == 'false'
+        if (val = opts[:eager_include_document_attributes])
+          opts[:eager_include_document_attributes] = (val == 'true') ? true : val.split(',')
+        end
+        if (val = opts[:lazy_update_document_attributes])
+          opts[:lazy_update_document_attributes] = (val == 'true') ? true : val.split(',')
+        end
+        Import.new(indices: index_classes, **opts).run
       end
     end
   end

data/lib/esse/core.rb

@@ -6,6 +6,8 @@ module Esse
   require_relative 'primitives'
   require_relative 'collection'
   require_relative 'document'
+  require_relative 'document_lazy_attribute'
+  require_relative 'lazy_document_header'
   require_relative 'hash_document'
   require_relative 'null_document'
   require_relative 'repository'

data/lib/esse/document.rb

@@ -56,7 +56,7 @@ module Esse
 
     # @return [Hash] the document data
     def to_h
-      source.merge(
+      mutated_source.merge(
         _id: id,
       ).tap do |hash|
         hash[:_type] = type if type
@@ -65,11 +65,13 @@ module Esse
       end
     end
 
-    def to_bulk(data: true)
-      { _id: id }.tap do |h|
-        h[:data] = source&.to_h if data
-        h[:_type] = type if type
-        h[:routing] = routing if routing?
+    def to_bulk(data: true, operation: nil)
+      doc_header.tap do |h|
+        if data && operation == :update
+          h[:data] = { doc: mutated_source }
+        elsif data
+          h[:data] = mutated_source
+        end
         h.merge!(meta)
       end
     end
@@ -87,5 +89,35 @@ module Esse
         id == other.id && type == other.type && routing == other.routing && meta == other.meta && source == other.source
       )
     end
+
+    def doc_header
+      { _id: id }.tap do |h|
+        h[:_type] = type if type
+        h[:routing] = routing if routing?
+      end
+    end
+
+    def inspect
+      attributes = %i[id routing source].map do |attr|
+        value = send(attr)
+        "#{attr}: #{value.inspect}" if value
+      end.compact.join(', ')
+      attributes << " mutations: #{@__mutations__.inspect}" if @__mutations__
+      "#<#{self.class.name || 'Esse::Document'} #{attributes}>"
+    end
+
+    def mutate(key)
+      @__mutations__ ||= {}
+      @__mutations__[key] = yield
+      instance_variable_set(:@__mutated_source__, nil)
+    end
+
+    protected
+
+    def mutated_source
+      return source unless @__mutations__
+
+      @__mutated_source__ ||= source.merge(@__mutations__)
+    end
   end
 end

data/lib/esse/document_lazy_attribute.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Esse
+  class DocumentLazyAttribute
+    attr_reader :options
+
+    def initialize(**kwargs)
+      @options = kwargs
+    end
+
+    # Returns an Hash with the document ID as key and attribute data as value.
+    # @param doc_headers [Array<Esse::LazyDocumentHeader>] the document headers
+    # @return [Hash] An Hash with the instance of document header as key and the attribute data as value.
+    def call(doc_headers)
+      raise NotImplementedError, 'Override this method to return the document attribute data'
+    end
+  end
+end

data/lib/esse/import/bulk.rb

@@ -1,7 +1,7 @@
 module Esse
   module Import
     class Bulk
-      def initialize(type: nil, index: nil, delete: nil, create: nil)
+      def initialize(type: nil, index: nil, delete: nil, create: nil, update: nil)
         @index = Array(index).select(&method(:valid_doc?)).reject(&:ignore_on_index?).map do |doc|
           value = doc.to_bulk
           value[:_type] ||= type if type
@@ -12,6 +12,11 @@ module Esse
           value[:_type] ||= type if type
           { create: value }
         end
+        @update = Array(update).select(&method(:valid_doc?)).reject(&:ignore_on_index?).map do |doc|
+          value = doc.to_bulk(operation: :update)
+          value[:_type] ||= type if type
+          { update: value }
+        end
         @delete = Array(delete).select(&method(:valid_doc?)).reject(&:ignore_on_delete?).map do |doc|
           value = doc.to_bulk(data: false)
           value[:_type] ||= type if type
@@ -22,12 +27,12 @@ module Esse
       # Return an array of RequestBody instances
       #
       # In case of timeout error, will retry with an exponential backoff using the following formula:
-      #  wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1)) seconds. It will retry up to max_retries times that is default 3.
+      #  wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1)) seconds. It will retry up to max_retries times that is default 4.
       #
       # Too large bulk requests will be split into multiple requests with only one attempt.
       #
       # @yield [RequestBody] A request body instance
-      def each_request(max_retries: 3)
+      def each_request(max_retries: 4, last_retry_in_small_chunks: true)
         # @TODO create indexes when by checking all the index suffixes (if mapping is not empty)
         requests = [optimistic_request]
         retry_count = 0
@@ -43,6 +48,8 @@ module Esse
         rescue Faraday::TimeoutError, Esse::Transport::RequestTimeoutError => e
           retry_count += 1
           raise Esse::Transport::RequestTimeoutError.new(e.message) if retry_count >= max_retries
+          # Timeout error may be caused by a too large request, so we split the requests in small chunks as a last attempt
+          requests = requests_in_small_chunks if last_retry_in_small_chunks && max_retries > 2 && retry_count == max_retries - 2
           wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1))
           Esse.logger.warn "Timeout error, retrying in #{wait_interval} seconds"
           sleep(wait_interval)
@@ -67,16 +74,30 @@ module Esse
 
       def optimistic_request
         request = Import::RequestBodyAsJson.new
-        request.delete = @delete
         request.create = @create
         request.index = @index
+        request.update = @update
+        request.delete = @delete
         request
       end
 
+      def requests_in_small_chunks(chunk_size: 1)
+        arr = []
+        @create.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.create = slice } }
+        @index.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.index = slice } }
+        @update.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.update = slice } }
+        @delete.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.delete = slice } }
+        Esse.logger.warn <<~MSG
+          Retrying the last request in small chunks of #{chunk_size} documents.
+          This is a last resort to avoid timeout errors, consider increasing the bulk size or reducing the batch size.
+        MSG
+        arr
+      end
+
       # @return [Array<RequestBody>]
       def balance_requests_size(err)
         if (bulk_size = err.message.scan(/exceeded.(\d+).bytes/).dig(0, 0).to_i) > 0
-          requests = (@delete + @create + @index).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
+          requests = (@create + @index + @update + @delete).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
             operation, meta = as_json.to_a.first
             meta = meta.dup
             data = meta.delete(:data)

data/lib/esse/import/request_body.rb

@@ -5,7 +5,7 @@ module Esse
 
       def initialize(body:)
         @body = body # body may be String or Array<Hash>
-        @stats = { index: 0, create: 0, delete: 0 }
+        @stats = { index: 0, create: 0, delete: 0, update: 0 }
       end
 
       def body?
@@ -46,6 +46,11 @@ module Esse
         @stats[:index] += docs.size
       end
 
+      def update=(docs)
+        @body += docs
+        @stats[:update] += docs.size
+      end
+
       def create=(docs)
         @body += docs
         @stats[:create] += docs.size

data/lib/esse/index/documents.rb

@@ -164,7 +164,7 @@ module Esse
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-bulk.html
       # @see https://github.com/elastic/elasticsearch-ruby/blob/main/elasticsearch-api/lib/elasticsearch/api/utils.rb
       # @see https://github.com/elastic/elasticsearch-ruby/blob/main/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
-      def bulk(index: nil, delete: nil, create: nil, type: nil, suffix: nil, **options)
+      def bulk(create: nil, delete: nil, index: nil, update: nil, type: nil, suffix: nil, **options)
         definition = {
           index: index_name(suffix: suffix),
           type: type,
@@ -174,9 +174,10 @@ module Esse
         # @TODO Wrap the return in a some other Stats object with more information
         Esse::Import::Bulk.new(
           **definition.slice(:type),
-          index: index,
-          delete: delete,
           create: create,
+          delete: delete,
+          index: index,
+          update: update,
         ).each_request do |request_body|
           cluster.api.bulk(**definition, body: request_body.body) do |event_payload|
             event_payload[:body_stats] = request_body.stats
@@ -198,20 +199,37 @@ module Esse
       # @option [Hash] :context The collection context. This value will be passed as argument to the collection
       #   May be SQL condition or any other filter you have defined on the collection.
       # @return [Numeric] The number of documents imported
-      def import(*repo_types, context: {}, suffix: nil, **options)
+      def import(*repo_types, context: {}, eager_include_document_attributes: false, lazy_update_document_attributes: false, suffix: nil, **options)
         repo_types = repo_hash.keys if repo_types.empty?
         count = 0
+
         repo_hash.slice(*repo_types).each do |repo_name, repo|
-          repo.each_serialized_batch(**(context || {})) do |batch|
+          doc_attrs = {eager: [], lazy: []}
+          doc_attrs[:eager] = repo.lazy_document_attribute_names(eager_include_document_attributes)
+          doc_attrs[:lazy] = repo.lazy_document_attribute_names(lazy_update_document_attributes)
+          doc_attrs[:lazy] -= doc_attrs[:eager]
+
+          context ||= {}
+          context[:lazy_attributes] = doc_attrs[:eager] if doc_attrs[:eager].any?
+          repo.each_serialized_batch(**context) do |batch|
             # Elasticsearch 6.x and older have multiple types per index.
             # This gem supports multiple types per index for backward compatibility, but we recommend to update
             # your elasticsearch to a at least 7.x version and use a single type per index.
             #
             # Note that the repository name will be used as the document type.
             # mapping_default_type
-            kwargs = { index: batch, suffix: suffix, type: repo_name, **options }
+            kwargs = { suffix: suffix, type: repo_name, **options }
             cluster.may_update_type!(kwargs)
-            bulk(**kwargs)
+
+            bulk(**kwargs, index: batch)
+
+            doc_attrs[:lazy].each do |attr_name|
+              partial_docs = repo.documents_for_lazy_attribute(attr_name, batch.reject(&:ignore_on_index?))
+              next if partial_docs.empty?
+
+              bulk(**kwargs, update: partial_docs)
+            end
+
             count += batch.size
           end
         end

data/lib/esse/index/indices.rb

@@ -26,10 +26,10 @@ module Esse
       #
       # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
       # @see Esse::Transport#create_index
-      def create_index(suffix: nil, **options)
+      def create_index(suffix: nil, body: nil, **options)
         options = CREATE_INDEX_RESERVED_KEYWORDS.merge(options)
         name = build_real_index_name(suffix)
-        definition = [settings_hash, mappings_hash].reduce(&:merge)
+        definition = body || [settings_hash, mappings_hash].reduce(&:merge)
 
         if options.delete(:alias) && name != index_name
           definition[:aliases] = { index_name => {} }
@@ -48,13 +48,23 @@ module Esse
       # @return [Hash] the elasticsearch response
       #
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html
-      def reset_index(suffix: index_suffix, import: true, reindex: false, **options)
+      def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: false, **options)
         cluster.throw_error_when_readonly!
-        existing = []
+
         suffix ||= Esse.timestamp
-        suffix = Esse.timestamp while index_exist?(suffix: suffix).tap { |exist| existing << suffix if exist }
+        suffix = Esse.timestamp while index_exist?(suffix: suffix)
+
+        if optimize
+          definition = [settings_hash, mappings_hash].reduce(&:merge)
+          number_of_replicas = definition.dig(Esse::SETTING_ROOT_KEY, :index, :number_of_replicas)
+          refresh_interval = definition.dig(Esse::SETTING_ROOT_KEY, :index, :refresh_interval)
+          new_number_of_replicas = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:number_of_replicas] = 0
+          new_refresh_interval = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:refresh_interval] = '-1'
+          create_index(**options, suffix: suffix, alias: false, body: definition)
+        else
+          create_index(**options, suffix: suffix, alias: false)
+        end
 
-        create_index(**options, suffix: suffix, alias: false)
         if index_exist? && aliases.none?
           cluster.api.delete_index(index: index_name)
         end
@@ -63,8 +73,13 @@ module Esse
         elsif reindex && (_from = indices_pointing_to_alias).any?
           # @TODO: Reindex using the reindex API
         end
+
+        if optimize && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval
+          update_settings(suffix: suffix)
+        end
+
         update_aliases(suffix: suffix)
-        existing.each { |_s| delete_index!(**options, suffix: suffix) }
+
         true
       end
 

data/lib/esse/lazy_document_header.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Esse
+  class LazyDocumentHeader
+    def self.coerce_each(values)
+      arr = []
+      Esse::ArrayUtils.wrap(values).map do |value|
+        instance = coerce(value)
+        arr << instance if instance&.valid?
+      end
+      arr
+    end
+
+    def self.coerce(value)
+      return unless value
+
+      if value.is_a?(Esse::LazyDocumentHeader)
+        value
+      elsif value.is_a?(Esse::Document)
+        new(value.doc_header)
+      elsif value.is_a?(Hash)
+        resp = value.transform_keys do |key|
+          case key
+          when :_id, :id, '_id', 'id'
+            :_id
+          when :_routing, :routing, '_routing', 'routing'
+            :routing
+          when :_type, :type, '_type', 'type'
+            :_type
+          else
+            key.to_sym
+          end
+        end
+        new(resp)
+      elsif String === value || Integer === value
+        new(_id: value)
+      end
+    end
+
+    def initialize(attributes)
+      @attributes = attributes
+    end
+
+    def valid?
+      !@attributes[:_id].nil?
+    end
+
+    def to_h
+      @attributes
+    end
+
+    def id
+      @attributes.fetch(:_id)
+    end
+
+    def type
+      @attributes[:_type]
+    end
+
+    def routing
+      @attributes[:routing]
+    end
+
+    def to_doc(source = {})
+      HashDocument.new(source.merge(@attributes))
+    end
+
+    def eql?(other)
+      self.class == other.class && @attributes == other.instance_variable_get(:@attributes)
+    end
+    alias_method :==, :eql?
+  end
+end

data/lib/esse/primitives/array_utils.rb

@@ -0,0 +1,17 @@
+module Esse
+  # The idea here is to add useful methods to the ruby standard objects without
+  # monkey patching them
+  module ArrayUtils
+    module_function
+
+    def wrap(object)
+      if object.nil?
+        []
+      elsif object.respond_to?(:to_ary)
+        object.to_ary || [object]
+      else
+        [object]
+      end
+    end
+  end
+end

data/lib/esse/primitives.rb

@@ -2,3 +2,4 @@
 
 require_relative 'primitives/hstring'
 require_relative 'primitives/hash_utils'
+require_relative 'primitives/array_utils'

data/lib/esse/repository/documents.rb

@@ -6,6 +6,43 @@ module Esse
       def import(**kwargs)
         index.import(repo_name, **kwargs)
       end
+
+      def update_documents_attribute(name, ids_or_doc_headers = [], kwargs = {})
+        batch = documents_for_lazy_attribute(name, ids_or_doc_headers)
+        return if batch.empty?
+
+        index.bulk(**kwargs.transform_keys(&:to_sym), update: batch)
+      end
+
+      def documents_for_lazy_attribute(name, ids_or_doc_headers)
+        retrieve_lazy_attribute_values(name, ids_or_doc_headers).map do |doc_header, datum|
+          doc_header.to_doc(name => datum)
+        end
+      end
+
+      def retrieve_lazy_attribute_values(name, ids_or_doc_headers)
+        unless lazy_document_attribute?(name)
+          raise ArgumentError, <<~MSG
+            The attribute `#{name}` is not defined as a lazy document attribute.
+
+            Define the attribute as a lazy document attribute using the `lazy_document_attribute` method.
+          MSG
+        end
+
+        docs = LazyDocumentHeader.coerce_each(ids_or_doc_headers)
+        return [] if docs.empty?
+
+        result = fetch_lazy_document_attribute(name).call(docs)
+        return [] unless result.is_a?(Hash)
+
+        result.each_with_object({}) do |(key, value), memo|
+          if key.is_a?(LazyDocumentHeader) && (doc = docs.find { |d| d == key || d.id == key.id })
+            memo[doc] = value
+          elsif (doc = docs.find { |d| d.id == key })
+            memo[doc] = value
+          end
+        end
+      end
     end
 
     extend ClassMethods

data/lib/esse/repository/lazy_document_attributes.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Esse
+  # Definition for the lazy document attributes
+  class Repository
+    module ClassMethods
+      def lazy_document_attributes
+        @lazy_document_attributes ||= {}.freeze
+      end
+
+      def lazy_document_attribute_names(all = true)
+        case all
+        when false
+          []
+        when true
+          lazy_document_attributes.keys
+        else
+          filtered = Array(all).map(&:to_s)
+          lazy_document_attributes.keys.select { |name| filtered.include?(name.to_s) }
+        end
+      end
+
+      def fetch_lazy_document_attribute(attr_name)
+        klass, kwargs = lazy_document_attributes.fetch(attr_name)
+        klass.new(**kwargs)
+      rescue KeyError
+        raise ArgumentError, format('Attribute %<attr>p is not defined as a lazy document attribute', attr: attr_name)
+      end
+
+      def lazy_document_attribute(attr_name, klass = nil, **kwargs, &block)
+        if attr_name.nil?
+          raise ArgumentError, 'Attribute name is required to define a lazy document attribute'
+        end
+        if lazy_document_attribute?(attr_name.to_sym) || lazy_document_attribute?(attr_name.to_s)
+          raise ArgumentError, format('Attribute %<attr>p is already defined as a lazy document attribute', attr: attr_name)
+        end
+
+        @lazy_document_attributes = lazy_document_attributes.dup
+        if block
+          klass = Class.new(Esse::DocumentLazyAttribute) do
+            define_method(:call, &block)
+          end
+          @lazy_document_attributes[attr_name] = [klass, kwargs]
+        elsif klass.is_a?(Class) && klass <= Esse::DocumentLazyAttribute
+          @lazy_document_attributes[attr_name] = [klass, kwargs]
+        elsif klass.is_a?(Class) && klass.instance_methods.include?(:call)
+          @lazy_document_attributes[attr_name] = [klass, kwargs]
+        elsif klass.nil?
+          raise ArgumentError, format('A block or a class that responds to `call` is required to define a lazy document attribute')
+        else
+          raise ArgumentError, format('%<arg>p is not a valid lazy document attribute. Class should inherit from Esse::DocumentLazyAttribute or respond to `call`', arg: klass)
+        end
+      ensure
+        @lazy_document_attributes&.freeze
+      end
+
+      protected
+
+      def lazy_document_attribute?(attr_name)
+        lazy_document_attributes.key?(attr_name)
+      end
+    end
+
+    extend ClassMethods
+  end
+end

data/lib/esse/repository/object_document_mapper.rb

@@ -6,8 +6,8 @@ module Esse
   # @see ObjectDocumentMapper
   class Repository
     module ClassMethods
-      # Convert ruby object to json. Arguments will be same of passed through the
-      # collection. It's allowed a block or a class with the `to_h` instance method.
+      # Define the document type that will be used to serialize the data.
+      # Arguments will be same of passed through the collection. It's allowed a block or a class with the `to_h` instance method.
       # Example with block
       #   document do |model, **context|
       #     {
@@ -38,31 +38,6 @@ module Esse
         end
       end
 
-      def coerce_to_document(value)
-        case value
-        when Esse::Document
-          value
-        when Hash
-          Esse::HashDocument.new(value)
-        when NilClass, FalseClass
-          Esse::NullDocument.new
-        else
-          raise ArgumentError, format('%<arg>p is not a valid document. The document should be a hash or an instance of Esse::Document', arg: value)
-        end
-      end
-
-      # Convert ruby object to json by using the document of the given document type.
-      # @param [Object] model The ruby object
-      # @param [Hash] kwargs The context
-      # @return [Esse::Document] The serialized document
-      def serialize(model, **kwargs)
-        if @document_proc.nil?
-          raise NotImplementedError, format('there is no %<t>p document defined for the %<k>p index', t: repo_name, k: index.to_s)
-        end
-
-        @document_proc.call(model, **kwargs)
-      end
-
       # Used to define the source of data. A block is required. And its
       # content should yield an array of each object that should be serialized.
       # The list of arguments will be passed throught the document method.
@@ -94,6 +69,71 @@ module Esse
         @collection_proc = collection_klass || block
       end
 
+      # Wrap collection data into serialized batches
+      #
+      # @param [Hash] kwargs The context
+      # @return [Enumerator] The enumerator
+      # @yield [Array, **context] serialized collection and the optional context from the collection
+      def each_serialized_batch(lazy_attributes: false, **kwargs)
+        each_batch(**kwargs) do |*args|
+          batch, collection_context = args
+          collection_context ||= {}
+          entries = [*batch].map { |entry| serialize(entry, **collection_context) }.compact
+          if lazy_attributes
+            attrs = lazy_attributes.is_a?(Array) ? lazy_attributes : lazy_document_attribute_names(lazy_attributes)
+            attrs.each do |attr_name|
+              retrieve_lazy_attribute_values(attr_name, entries).each do |doc_header, value|
+                doc = entries.find { |d| doc_header.id.to_s == d.id.to_s && doc_header.type == d.type && doc_header.routing == d.routing }
+                doc&.mutate(attr_name) { value }
+              end
+            end
+          end
+
+          yield entries, **kwargs
+        end
+      end
+
+      # Wrap collection data into serialized documents
+      #
+      # Example:
+      #    GeosIndex.documents(id: 1).first
+      #
+      # @return [Enumerator] All serialized entries
+      def documents(**kwargs)
+        Enumerator.new do |yielder|
+          each_serialized_batch(**kwargs) do |docs, **_collection_kargs|
+            docs.each { |document| yielder.yield(document) }
+          end
+        end
+      end
+
+      # Convert ruby object to json by using the document of the given document type.
+      # @param [Object] model The ruby object
+      # @param [Hash] kwargs The context
+      # @return [Esse::Document] The serialized document
+      def serialize(model, **kwargs)
+        if @document_proc.nil?
+          raise NotImplementedError, format('there is no %<t>p document defined for the %<k>p index', t: repo_name, k: index.to_s)
+        end
+
+        @document_proc.call(model, **kwargs)
+      end
+
+      protected
+
+      def coerce_to_document(value)
+        case value
+        when Esse::Document
+          value
+        when Hash
+          Esse::HashDocument.new(value)
+        when NilClass, FalseClass
+          Esse::NullDocument.new
+        else
+          raise ArgumentError, format('%<arg>p is not a valid document. The document should be a hash or an instance of Esse::Document', arg: value)
+        end
+      end
+
       # Used to fetch all batch of data defined on the collection model.
       # Arguments can be anything. They will just be passed through the block.
       # Useful when the collection depends on scope or any other conditions
@@ -122,34 +162,6 @@ module Esse
       rescue LocalJumpError
         raise(SyntaxError, 'block must be explicitly declared in the collection definition')
       end
-
-      # Wrap collection data into serialized batches
-      #
-      # @param [Hash] kwargs The context
-      # @return [Enumerator] The enumerator
-      # @yield [Array, **context] serialized collection and the optional context from the collection
-      def each_serialized_batch(**kwargs, &block)
-        each_batch(**kwargs) do |*args|
-          batch, collection_context = args
-          collection_context ||= {}
-          entries = [*batch].map { |entry| serialize(entry, **collection_context) }.compact
-          block.call(entries, **kwargs)
-        end
-      end
-
-      # Wrap collection data into serialized documents
-      #
-      # Example:
-      #    GeosIndex.documents(id: 1).first
-      #
-      # @return [Enumerator] All serialized entries
-      def documents(**kwargs)
-        Enumerator.new do |yielder|
-          each_serialized_batch(**kwargs) do |docs, **_collection_kargs|
-            docs.each { |document| yielder.yield(document) }
-          end
-        end
-      end
     end
 
     extend ClassMethods

data/lib/esse/repository.rb

@@ -13,5 +13,6 @@ module Esse
     require_relative 'repository/actions'
     require_relative 'repository/documents'
     require_relative 'repository/object_document_mapper'
+    require_relative 'repository/lazy_document_attributes'
   end
 end

data/lib/esse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Esse
-  VERSION = '0.2.6'
+  VERSION = '0.3.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: esse
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.2
 platform: ruby
 authors:
 - Marcos G. Zimmermann
 autorequire:
 bindir: exec
 cert_chain: []
-date: 2023-11-27 00:00:00.000000000 Z
+date: 2024-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -248,6 +248,7 @@ files:
 - lib/esse/deprecations/repository_backend_delegator.rb
 - lib/esse/deprecations/serializer.rb
 - lib/esse/document.rb
+- lib/esse/document_lazy_attribute.rb
 - lib/esse/dynamic_template.rb
 - lib/esse/errors.rb
 - lib/esse/events.rb
@@ -274,16 +275,19 @@ files:
 - lib/esse/index/type.rb
 - lib/esse/index_mapping.rb
 - lib/esse/index_setting.rb
+- lib/esse/lazy_document_header.rb
 - lib/esse/logging.rb
 - lib/esse/null_document.rb
 - lib/esse/plugins.rb
 - lib/esse/primitives.rb
+- lib/esse/primitives/array_utils.rb
 - lib/esse/primitives/hash_utils.rb
 - lib/esse/primitives/hstring.rb
 - lib/esse/primitives/output.rb
 - lib/esse/repository.rb
 - lib/esse/repository/actions.rb
 - lib/esse/repository/documents.rb
+- lib/esse/repository/lazy_document_attributes.rb
 - lib/esse/repository/object_document_mapper.rb
 - lib/esse/search/query.rb
 - lib/esse/search/query/dsl.rb
@@ -318,7 +322,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.0.3.1
 signing_key:
 specification_version: 4
 summary: Pure Ruby and framework-agnostic ElasticSearch/OpenSearch toolkit for building