esse 0.2.0 → 0.2.2

data/lib/esse/import/bulk.rb

@@ -0,0 +1,96 @@
+module Esse
+  module Import
+    class Bulk
+      def initialize(index: nil, delete: nil, create: nil)
+        @index = Array(index).select(&method(:valid_doc?)).reject(&:ignore_on_index?).map do |doc|
+          { index: doc.to_bulk }
+        end
+        @create = Array(create).select(&method(:valid_doc?)).reject(&:ignore_on_index?).map do |doc|
+          { create: doc.to_bulk }
+        end
+        @delete = Array(delete).select(&method(:valid_doc?)).reject(&:ignore_on_delete?).map do |doc|
+          { delete: doc.to_bulk(data: false) }
+        end
+      end
+
+      # Return an array of RequestBody instances
+      #
+      # In case of timeout error, will retry with an exponential backoff using the following formula:
+      #  wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1)) seconds. It will retry up to max_retries times that is default 3.
+      #
+      # Too large bulk requests will be split into multiple requests with only one attempt.
+      #
+      # @yield [RequestBody] A request body instance
+      def each_request(max_retries: 3)
+        requests = [optimistic_request]
+        retry_count = 0
+
+        begin
+          requests.each do |request|
+            yield(request) if request.body?
+          end
+        rescue Faraday::TimeoutError, Esse::Backend::RequestTimeoutError => e
+          retry_count += 1
+          raise Esse::Backend::RequestTimeoutError.new(e.message) if retry_count >= max_retries
+          wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1))
+          Esse.logger.warn "Timeout error, retrying in #{wait_interval} seconds"
+          sleep(wait_interval)
+          retry
+        rescue Esse::Backend::RequestEntityTooLargeError => e
+          retry_count += 1
+          raise e if retry_count > 1 # only retry once on this error
+          requests = balance_requests_size(e)
+          Esse.logger.warn <<~MSG
+            Request entity too large, retrying with a bulk with: #{requests.map(&:bytesize).join(" + ")}.
+            Note that this cause performance degradation, consider adjusting the batch_size of the index or increasing the bulk size.
+          MSG
+          retry
+        end
+      end
+
+      private
+
+      def valid_doc?(doc)
+        doc && doc.is_a?(Esse::Serializer) && doc.id
+      end
+
+      def optimistic_request
+        request = Import::RequestBodyAsJson.new
+        request.delete = @delete
+        request.create = @create
+        request.index = @index
+        request
+      end
+
+      # @return [Array<RequestBody>]
+      def balance_requests_size(err)
+        if (bulk_size = err.message.scan(/exceeded.(\d+).bytes/).dig(0, 0).to_i) > 0
+          requests = (@delete + @create + @index).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
+            operation, meta = as_json.to_a.first
+            meta = meta.dup
+            data = meta.delete(:data)
+            piece = MultiJson.dump(operation => meta)
+            piece << "\n" << MultiJson.dump(data) if data
+            if piece.bytesize > bulk_size
+              Esse.logger.warn <<~MSG
+                The document #{meta.inspect} size is #{piece.bytesize} bytes, which exceeds the maximum bulk size of #{bulk_size} bytes.
+                Consider increasing the bulk size or reducing the document size. The document will be ignored during this import.
+              MSG
+              next
+            end
+
+            if result.last.body.bytesize + piece.bytesize > bulk_size
+              result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
+            else
+              result[-1].add(operation, piece)
+            end
+          end
+          requests.each(&:finalize)
+        else
+          raise err
+        end
+      end
+
+    end
+  end
+end

data/lib/esse/import/request_body.rb

@@ -0,0 +1,60 @@
+module Esse
+  module Import
+    class RequestBody
+      attr_reader :body, :stats
+
+      def initialize(body:)
+        @body = body # body may be String or Array<Hash>
+        @stats = { index: 0, create: 0, delete: 0 }
+      end
+
+      def body?
+        !body.empty?
+      end
+    end
+
+    class RequestBodyRaw < RequestBody
+      def initialize
+        super(body: '')
+      end
+
+      def bytesize
+        body.bytesize
+      end
+
+      def add(operation, payload)
+        stats[operation] += 1
+        if @body.empty?
+          @body = payload
+        else
+          @body << "\n" << payload
+        end
+      end
+
+      def finalize
+        @body << "\n"
+      end
+    end
+
+    class RequestBodyAsJson < RequestBody
+      def initialize
+        super(body: [])
+      end
+
+      def index=(docs)
+        @body += docs
+        @stats[:index] += docs.size
+      end
+
+      def create=(docs)
+        @body += docs
+        @stats[:create] += docs.size
+      end
+
+      def delete=(docs)
+        @body += docs
+        @stats[:delete] += docs.size
+      end
+    end
+  end
+end

data/lib/esse/index/attributes.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    module ClassMethods
+      TEMPLATE_DIRS = [
+        '%<dirname>s/templates',
+        '%<dirname>s'
+      ].freeze
+
+      def index_name=(value)
+        @index_name = Hstring.new(value.to_s).underscore.presence
+      end
+
+      def index_name(suffix: nil)
+        iname = index_prefixed_name(@index_name || normalized_name)
+        suffix = Hstring.new(suffix).underscore.presence
+        return iname if !iname || !suffix
+
+        [iname, suffix].join('_')
+      end
+
+      def index_name?
+        !index_name.nil?
+      end
+
+      def index_prefix
+        return @index_prefix if defined? @index_prefix
+
+        cluster.index_prefix
+      end
+
+      def index_prefix=(value)
+        return @index_prefix = nil if value == false
+
+        @index_prefix = Hstring.new(value.to_s).underscore.presence
+      end
+
+      def index_version=(value)
+        @index_version = Hstring.new(value.to_s).underscore.presence
+      end
+
+      def index_version
+        @index_version
+      end
+
+      def uname
+        Hstring.new(name).underscore.presence
+      end
+
+      def index_directory
+        return unless uname
+        return if uname == 'Esse::Index'
+
+        Esse.config.indices_directory.join(uname).to_s
+      end
+
+      def template_dirs
+        return [] unless index_directory
+
+        TEMPLATE_DIRS.map { |term| format(term, dirname: index_directory) }
+      end
+
+      def bulk_wait_interval
+        @bulk_wait_interval || Esse.config.bulk_wait_interval
+      end
+
+      def bulk_wait_interval=(value)
+        @bulk_wait_interval = value.to_f
+      end
+
+      def mapping_single_type=(value)
+        @mapping_single_type = !!value
+      end
+
+      def mapping_single_type?
+        return @mapping_single_type if defined? @mapping_single_type
+
+        @mapping_single_type = cluster.engine.mapping_single_type?
+      end
+
+      protected
+
+      def index_prefixed_name(value)
+        return if value == '' || value.nil?
+        return value.to_s unless index_prefix
+
+        [index_prefix, value].join('_')
+      end
+
+      def normalized_name
+        Hstring.new(name).underscore.tr('/', '_').sub(/_(index)$/, '')
+      end
+    end
+
+    extend ClassMethods
+  end
+end

data/lib/esse/index/base.rb

@@ -67,7 +67,7 @@ module Esse
           raise ArgumentError.new, format(msg, arg: source, ids: valid_ids.map(&:inspect).join(', '))
         end
 
-        klass.type_hash = {}
+        klass.repo_hash = {}
         klass
       end
 

data/lib/esse/index/inheritance.rb

@@ -11,6 +11,36 @@ module Esse
 
         !index_name?
       end
+
+      def inherited(subclass)
+        super
+
+        inherited_instance_variables.each do |variable_name, should_duplicate|
+          if (variable_value = instance_variable_get(variable_name)) && should_duplicate
+            value = case variable_value
+            when Hash
+              h = {}
+              variable_value.each { |k, v| h[k] = v.dup }
+              h
+            else
+              variable_value.dup
+            end
+          end
+          subclass.instance_variable_set(variable_name, value)
+        end
+      end
+
+      private
+
+      def inherited_instance_variables
+        {
+          :@repo_hash => nil,
+          :@setting => nil,
+          :@mapping => nil,
+          :@cluster_id => :dup,
+          :@plugins => :dup,
+        }
+      end
     end
 
     extend ClassMethods

data/lib/esse/index/mappings.rb

@@ -7,36 +7,23 @@
 module Esse
   class Index
     module ClassMethods
-      # This is the actually content that will be passed through the ES api
-      def mappings_hash
-        { Esse::MAPPING_ROOT_KEY => (index_mapping || type_mapping) }
-      end
-
       # This method is only used to define mapping
       def mappings(hash = {}, &block)
-        @mapping = Esse::IndexMapping.new(body: hash, paths: template_dirs)
+        @mapping = Esse::IndexMapping.new(body: hash, paths: template_dirs, globals: -> { cluster.mappings })
         return unless block
 
         @mapping.define_singleton_method(:to_h, &block)
       end
 
-      private
-
-      def mapping
-        @mapping ||= Esse::IndexMapping.new(paths: template_dirs)
-      end
-
-      def index_mapping
-        return if mapping.empty?
-
+      def mappings_hash
         hash = mapping.body
-        hash.key?(Esse::MAPPING_ROOT_KEY) ? hash[Esse::MAPPING_ROOT_KEY] : hash
+        { Esse::MAPPING_ROOT_KEY => (hash.key?(Esse::MAPPING_ROOT_KEY) ? hash[Esse::MAPPING_ROOT_KEY] : hash) }
       end
 
-      def type_mapping
-        return {} if type_hash.empty?
+      private
 
-        type_hash.values.map(&:mappings_hash).reduce(&:merge)
+      def mapping
+        @mapping ||= Esse::IndexMapping.new(paths: template_dirs, globals: -> { cluster.mappings })
       end
     end
 

data/lib/esse/index/object_document_mapper.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    module ObjectDocumentMapper
+      # Convert ruby object to json. Arguments will be same of passed through the
+      # collection. It's allowed a block or a class with the `to_h` instance method.
+      # Example with block
+      #   serializer :user do |model, **context|
+      #     {
+      #       id: model.id,
+      #       admin: context[:is_admin],
+      #     }
+      #   end
+      # Example with serializer class
+      #   serializer UserSerializer
+      def serializer(*args, &block)
+        repo_name, klass = args
+        # >> Backward compatibility for the old collection syntax without explicit repo_name
+        if repo_name && klass.nil? && !repo_name.is_a?(String) && !repo_name.is_a?(Symbol)
+          klass = repo_name
+          repo_name = DEFAULT_REPO_NAME
+        end
+        repo_name = repo_name&.to_s || DEFAULT_REPO_NAME
+        # <<
+        find_or_define_repo(repo_name).serializer(klass, &block)
+      end
+
+      # Used to define the source of data. A block is required. And its
+      # content should yield an array of each object that should be serialized.
+      # The list of arguments will be passed throught the serializer method.
+      #
+      # Example:
+      #   collection :admin, AdminStore
+      #   collection :user do |**conditions, &block|
+      #     User.where(conditions).find_in_batches(batch_size: 5000) do |batch|
+      #       block.call(batch, conditions)
+      #     end
+      #   end
+      #
+      # @param [String] name The identification of the collection.
+      # @param [Class] klass The class of the collection. (Optional when block is passed)
+      # @param [Proc] block The block that will be used to iterate over the collection. (Optional when using a class)
+      # @return [void]
+      def collection(*args, **kwargs, &block)
+        repo_name, collection_klass = args
+        # >> Backward compatibility for the old collection syntax without explicit repo_name
+        if repo_name && !repo_name.is_a?(Symbol) && !repo_name.is_a?(String) && collection_klass.nil?
+          collection_klass = repo_name
+          repo_name = DEFAULT_REPO_NAME
+        end
+        repo_name = repo_name&.to_s || DEFAULT_REPO_NAME
+        # <<
+        find_or_define_repo(repo_name).collection(collection_klass, **kwargs, &block)
+      end
+
+      # Wrap collection data into serialized batches
+      #
+      # @param [String, NilClass] repo_name The repository identifier
+      # @param [Hash] kwargs The context
+      # @return [Enumerator] The enumerator
+      # @yield [Array, **context] serialized collection and the optional context from the collection
+      def each_serialized_batch(repo_name = nil, **kwargs, &block)
+        (repo_name ? [repo(repo_name)] : repo_hash.values).each do |repo|
+          repo.each_serialized_batch(**kwargs, &block)
+        end
+      end
+
+      # Wrap collection data into serialized documents
+      #
+      # Example:
+      #    GeosIndex.documents(id: 1).first
+      #
+      # @param [String, NilClass] repo_name The repository identifier
+      # @return [Enumerator] All serialized entries
+      def documents(repo_name = nil, **kwargs)
+        Enumerator.new do |yielder|
+          each_serialized_batch(repo_name, **kwargs) do |documents, **_collection_kargs|
+            documents.each { |document| yielder.yield(document) }
+          end
+        end
+      end
+
+      private
+
+      def find_or_define_repo(repo_name)
+        return repo_hash[repo_name] if repo_hash.key?(repo_name)
+
+        repository(repo_name) {}
+      end
+    end
+
+    extend ObjectDocumentMapper
+  end
+end

data/lib/esse/index/plugins.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    module ClassMethods
+      attr_reader :plugins
+
+      def plugin(plugin, **kwargs, &block)
+        mod = plugin.is_a?(Module) ? plugin : load_plugin_module(plugin)
+
+        unless @plugins.include?(mod)
+          @plugins << mod
+          mod.apply(self, **kwargs, &block) if mod.respond_to?(:apply)
+          extend(mod::IndexClassMethods) if mod.const_defined?(:IndexClassMethods, false)
+          if mod.const_defined?(:RepositoryClassMethods, false)
+            repo_hash.each_value.each { |repo| repository_plugin_extend(repo, mod::RepositoryClassMethods) }
+          end
+        end
+
+        mod.configure(self, **kwargs, &block) if mod.respond_to?(:configure)
+      end
+
+      private
+
+      def repository_plugin_extend(repo_class, mod)
+        return if repo_class.singleton_class.included_modules.include?(mod)
+
+        repo_class.extend(mod)
+      end
+
+      def load_plugin_module(name)
+        module_name = Hstring.new(name)
+        unless Esse::Plugins.const_defined?(module_name.camelize.to_s, false)
+          require "esse/plugins/#{module_name.underscore}"
+        end
+        Esse::Plugins.const_get(module_name.camelize.to_s)
+      end
+    end
+
+    extend ClassMethods
+  end
+end

data/lib/esse/index/search.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    module ClassMethods
+      # @param query_or_payload [String,Hash] The search request definition or query in the Lucene query string syntax
+      # @param kwargs [Hash] The options to pass to the search.
+      def search(*args, &block)
+        query_or_payload = args.shift
+        kwargs = args.last.is_a?(Hash) ? args.pop : {}
+
+        if query_or_payload.respond_to?(:to_hash) && (hash = query_or_payload.to_hash) && (hash.key?(:body) || hash.key?('body') || hash.key?(:q) || hash.key?('q'))
+          kwargs.merge!(hash.transform_keys(&:to_sym))
+        elsif query_or_payload.respond_to?(:to_hash)
+          kwargs[:body] = query_or_payload.to_hash
+        elsif query_or_payload.is_a?(String) && query_or_payload =~ /^\s*{/
+          kwargs[:body] = MultiJson.load(query_or_payload)
+        elsif query_or_payload.is_a?(String)
+          kwargs[:q] = query_or_payload
+        end
+        cluster.search(self, **kwargs, &block)
+      end
+    end
+
+    extend ClassMethods
+  end
+end

data/lib/esse/index/settings.rb

@@ -28,7 +28,7 @@ module Esse
       #     end
       #   end
       def settings(hash = {}, &block)
-        @setting = Esse::IndexSetting.new(body: hash, paths: template_dirs, globals: -> { cluster.index_settings })
+        @setting = Esse::IndexSetting.new(body: hash, paths: template_dirs, globals: -> { cluster.settings })
         return unless block
 
         @setting.define_singleton_method(:to_h, &block)
@@ -37,7 +37,7 @@ module Esse
       private
 
       def setting
-        @setting ||= Esse::IndexSetting.new(paths: template_dirs, globals: -> { cluster.index_settings })
+        @setting ||= Esse::IndexSetting.new(paths: template_dirs, globals: -> { cluster.settings })
       end
     end
 

data/lib/esse/index/type.rb

@@ -3,26 +3,67 @@
 module Esse
   class Index
     module ClassMethods
-      attr_writer :type_hash
+      attr_writer :repo_hash
 
-      def type_hash
-        @type_hash ||= {}
+      def repo_hash
+        @repo_hash ||= {}
       end
 
-      def define_type(type_name, &block)
-        type_class = Class.new(Esse::IndexType)
+      def repo(name = nil)
+        if name.nil? && repo_hash.size == 1
+          name = repo_hash.keys.first
+        elsif name.nil? && repo_hash.size > 1
+          raise ArgumentError, "You can only call `repo' with a name when there is only one type defined."
+        end
+        name ||= DEFAULT_REPO_NAME
 
-        const_set(Hstring.new(type_name).camelize.demodulize.to_s, type_class)
+        repo_hash.fetch(name.to_s)
+      rescue KeyError
+        raise ArgumentError, <<~MSG
+          No repo named "#{name}" found. Use the `repository' method to define one:
+
+            repository :#{name} do
+              # collection ...
+              # serializer ...
+            end
+        MSG
+      end
+
+      def repo?(name = nil)
+        return repo_hash.size > 0 if name.nil?
+
+        repo_hash.key?(name.to_s)
+      end
+
+      def repository(repo_name, *_args, **kwargs, &block)
+        repo_class = Class.new(Esse::Repository)
+        kwargs[:const] ||= true # TODO Change this to false to avoid collisions with application classes
+        kwargs[:lazy_evaluate] ||= false
+
+        if kwargs[:const]
+          const_set(Hstring.new(repo_name).camelize.demodulize.to_s, repo_class)
+        end
 
         index = self
 
-        type_class.send(:define_singleton_method, :index) { index }
-        type_class.send(:define_singleton_method, :type_name) { type_name.to_s }
+        repo_class.send(:define_singleton_method, :index) { index }
+        repo_class.send(:define_singleton_method, :repo_name) { repo_name.to_s }
+        repo_class.document_type = (kwargs[:document_type] || repo_name).to_s
+
+        plugins.each do |mod|
+          next unless mod.const_defined?(:RepositoryClassMethods, false)
+
+          repository_plugin_extend(repo_class, mod::RepositoryClassMethods)
+        end
+
+        if kwargs[:lazy_evaluate]
 
-        type_class.class_eval(&block) if block
+        elsif block
+          repo_class.class_eval(&block)
+        end
 
-        self.type_hash = type_hash.merge(type_class.type_name => type_class)
-        type_class
+        self.repo_hash = repo_hash.merge(repo_class.repo_name => repo_class)
+        repo_class
       end
     end
 

data/lib/esse/index.rb

@@ -1,21 +1,25 @@
 # frozen_string_literal: true
 
-require_relative 'object_document_mapper'
-
 module Esse
   class Index
+    @repo_hash = {}
+    @setting = {}
+    @mapping = {}
+    @plugins = []
+    @cluster_id = nil
+
+    require_relative 'index/plugins'
     require_relative 'index/base'
     require_relative 'index/inheritance'
     require_relative 'index/actions'
-    require_relative 'index/naming'
+    require_relative 'index/attributes'
     require_relative 'index/type'
     require_relative 'index/settings'
     require_relative 'index/mappings'
     require_relative 'index/descendants'
     require_relative 'index/backend'
-    extend ObjectDocumentMapper
-
-    @cluster_id = nil
+    require_relative 'index/object_document_mapper'
+    require_relative 'index/search'
 
     def_Index(::Esse)
   end

data/lib/esse/index_mapping.rb

@@ -4,10 +4,11 @@ module Esse
   class IndexMapping
     FILENAMES = %w[mapping mappings].freeze
 
-    def initialize(body: {}, paths: [], filenames: FILENAMES)
+    def initialize(body: {}, paths: [], filenames: FILENAMES, globals: nil)
       @paths = Array(paths)
       @filenames = Array(filenames)
       @mappings = body
+      @globals = globals || -> { {} }
     end
 
     # This method will be overwrited when passing a block during the
@@ -19,7 +20,14 @@ module Esse
     end
 
     def body
-      to_h
+      global = HashUtils.deep_transform_keys(@globals.call, &:to_sym)
+      local = HashUtils.deep_transform_keys(to_h.dup, &:to_sym)
+      dynamic_template = DynamicTemplate.new(global[:dynamic_templates])
+      dynamic_template.merge!(local.delete(:dynamic_templates))
+      if dynamic_template.any?
+        global[:dynamic_templates] = dynamic_template.to_a
+      end
+      HashUtils.deep_merge(global, local)
     end
 
     def empty?

data/lib/esse/index_setting.rb

@@ -30,7 +30,9 @@ module Esse
     end
 
     def body
-      HashUtils.deep_merge(@globals.call, to_h)
+      global = HashUtils.deep_transform_keys(@globals.call, &:to_sym)
+      local = HashUtils.deep_transform_keys(to_h, &:to_sym)
+      HashUtils.deep_merge(global, local)
     end
 
     protected