esse 0.2.0 → 0.2.3

data/lib/esse/import/bulk.rb

@@ -0,0 +1,106 @@
+module Esse
+  module Import
+    class Bulk
+      def initialize(type: nil, index: nil, delete: nil, create: nil)
+        @index = Array(index).select(&method(:valid_doc?)).reject(&:ignore_on_index?).map do |doc|
+          value = doc.to_bulk
+          value[:_type] ||= type if type
+          { index: value }
+        end
+        @create = Array(create).select(&method(:valid_doc?)).reject(&:ignore_on_index?).map do |doc|
+          value = doc.to_bulk
+          value[:_type] ||= type if type
+          { create: value }
+        end
+        @delete = Array(delete).select(&method(:valid_doc?)).reject(&:ignore_on_delete?).map do |doc|
+          value = doc.to_bulk(data: false)
+          value[:_type] ||= type if type
+          { delete: value }
+        end
+      end
+
+      # Return an array of RequestBody instances
+      #
+      # In case of timeout error, will retry with an exponential backoff using the following formula:
+      #  wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1)) seconds. It will retry up to max_retries times that is default 3.
+      #
+      # Too large bulk requests will be split into multiple requests with only one attempt.
+      #
+      # @yield [RequestBody] A request body instance
+      def each_request(max_retries: 3)
+        # @TODO create indexes when by checking all the index suffixes (if mapping is not empty)
+        requests = [optimistic_request]
+        retry_count = 0
+
+        begin
+          requests.each do |request|
+            next unless request.body?
+            resp = yield request
+            if resp&.[]('errors')
+              raise resp&.fetch('items', [])&.select { |item| item.values.first['error'] }&.join("\n")
+            end
+          end
+        rescue Faraday::TimeoutError, Esse::Transport::RequestTimeoutError => e
+          retry_count += 1
+          raise Esse::Transport::RequestTimeoutError.new(e.message) if retry_count >= max_retries
+          wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1))
+          Esse.logger.warn "Timeout error, retrying in #{wait_interval} seconds"
+          sleep(wait_interval)
+          retry
+        rescue Esse::Transport::RequestEntityTooLargeError => e
+          retry_count += 1
+          raise e if retry_count > 1 # only retry once on this error
+          requests = balance_requests_size(e)
+          Esse.logger.warn <<~MSG
+            Request entity too large, retrying with a bulk with: #{requests.map(&:bytesize).join(' + ')}.
+            Note that this cause performance degradation, consider adjusting the batch_size of the index or increasing the bulk size.
+          MSG
+          retry
+        end
+      end
+
+      private
+
+      def valid_doc?(doc)
+        Esse.document?(doc)
+      end
+
+      def optimistic_request
+        request = Import::RequestBodyAsJson.new
+        request.delete = @delete
+        request.create = @create
+        request.index = @index
+        request
+      end
+
+      # @return [Array<RequestBody>]
+      def balance_requests_size(err)
+        if (bulk_size = err.message.scan(/exceeded.(\d+).bytes/).dig(0, 0).to_i) > 0
+          requests = (@delete + @create + @index).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
+            operation, meta = as_json.to_a.first
+            meta = meta.dup
+            data = meta.delete(:data)
+            piece = MultiJson.dump(operation => meta)
+            piece << "\n" << MultiJson.dump(data) if data
+            if piece.bytesize > bulk_size
+              Esse.logger.warn <<~MSG
+                The document #{meta.inspect} size is #{piece.bytesize} bytes, which exceeds the maximum bulk size of #{bulk_size} bytes.
+                Consider increasing the bulk size or reducing the document size. The document will be ignored during this import.
+              MSG
+              next
+            end
+
+            if result.last.body.bytesize + piece.bytesize > bulk_size
+              result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
+            else
+              result[-1].add(operation, piece)
+            end
+          end
+          requests.each(&:finalize)
+        else
+          raise err
+        end
+      end
+    end
+  end
+end

data/lib/esse/import/request_body.rb

@@ -0,0 +1,60 @@
+module Esse
+  module Import
+    class RequestBody
+      attr_reader :body, :stats
+
+      def initialize(body:)
+        @body = body # body may be String or Array<Hash>
+        @stats = { index: 0, create: 0, delete: 0 }
+      end
+
+      def body?
+        !body.empty?
+      end
+    end
+
+    class RequestBodyRaw < RequestBody
+      def initialize
+        super(body: '')
+      end
+
+      def bytesize
+        body.bytesize
+      end
+
+      def add(operation, payload)
+        stats[operation] += 1
+        if @body.empty?
+          @body = payload
+        else
+          @body << "\n" << payload
+        end
+      end
+
+      def finalize
+        @body << "\n"
+      end
+    end
+
+    class RequestBodyAsJson < RequestBody
+      def initialize
+        super(body: [])
+      end
+
+      def index=(docs)
+        @body += docs
+        @stats[:index] += docs.size
+      end
+
+      def create=(docs)
+        @body += docs
+        @stats[:create] += docs.size
+      end
+
+      def delete=(docs)
+        @body += docs
+        @stats[:delete] += docs.size
+      end
+    end
+  end
+end

data/lib/esse/index/aliases.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    module ClassMethods
+      # Get the aliases for the index.
+      def aliases(**options)
+        response = cluster.api.aliases(**options, index: index_name, name: '*')
+        idx_name = response.keys.find { |idx| idx.start_with?(index_name) }
+        return [] unless idx_name
+
+        response.dig(idx_name, 'aliases')&.keys || []
+      rescue Esse::Transport::NotFoundError
+        []
+      end
+
+      # Return list of real index names for the virtual index name(alias)
+      def indices_pointing_to_alias(**options)
+        cluster.api.aliases(**options, name: index_name).keys
+      rescue Esse::Transport::NotFoundError
+        []
+      end
+
+      # Replaces all existing aliases by the respective suffixed index from argument.
+      #
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [Array<String>] :suffix One or more index suffixes to point the alias to.
+      # @raise [Esse::Transport::ServerError] in case of failure
+      # @return [Hash] the elasticsearch response
+      def update_aliases(suffix:, **options)
+        cluster.throw_error_when_readonly!
+        raise(ArgumentError, 'index suffix cannot be nil') if suffix.nil?
+
+        options[:body] = {
+          actions: [
+            *indices_pointing_to_alias.map do |index|
+              { remove: { index: index, alias: index_name } }
+            end,
+            *Array(suffix).map do |value|
+              { add: { index: build_real_index_name(value), alias: index_name } }
+            end,
+          ],
+        }
+        cluster.api.update_aliases(**options)
+      end
+    end
+
+    extend ClassMethods
+  end
+end

data/lib/esse/index/attributes.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    module ClassMethods
+      TEMPLATE_DIRS = [
+        '%<dirname>s/templates',
+        '%<dirname>s'
+      ].freeze
+
+      def index_name=(value)
+        @index_name = Hstring.new(value.to_s).underscore.presence
+      end
+
+      def index_name(suffix: nil)
+        iname = index_prefixed_name(@index_name || normalized_name)
+        suffix = Hstring.new(suffix).underscore.presence
+        return iname if !iname || !suffix
+
+        [iname, suffix].join('_')
+      end
+
+      def index_name?
+        !index_name.nil?
+      end
+
+      def index_prefix
+        return @index_prefix if defined? @index_prefix
+
+        cluster.index_prefix
+      end
+
+      def index_prefix=(value)
+        if value == false
+          @index_prefix = nil
+          return
+        end
+
+        @index_prefix = Hstring.new(value.to_s).underscore.presence
+      end
+
+      def index_suffix=(value)
+        @index_suffix = Hstring.new(value.to_s).underscore.presence
+      end
+
+      def index_suffix
+        @index_suffix
+      end
+
+      def uname
+        Hstring.new(name).underscore.presence
+      end
+
+      def index_directory
+        return unless uname
+        return if uname == 'Esse::Index'
+
+        Esse.config.indices_directory.join(uname).to_s
+      end
+
+      def template_dirs
+        return [] unless index_directory
+
+        TEMPLATE_DIRS.map { |term| format(term, dirname: index_directory) }
+      end
+
+      def bulk_wait_interval
+        @bulk_wait_interval || Esse.config.bulk_wait_interval
+      end
+
+      def bulk_wait_interval=(value)
+        @bulk_wait_interval = value.to_f
+      end
+
+      def mapping_single_type=(value)
+        @mapping_single_type = !!value
+      end
+
+      def mapping_single_type?
+        return @mapping_single_type if defined? @mapping_single_type
+
+        @mapping_single_type = cluster.engine.mapping_single_type?
+      end
+
+      protected
+
+      def index_prefixed_name(value)
+        return if value == '' || value.nil?
+        return value.to_s unless index_prefix
+
+        [index_prefix, value].join('_')
+      end
+
+      def normalized_name
+        Hstring.new(name).underscore.tr('/', '_').sub(/_(index)$/, '')
+      end
+
+      def build_real_index_name(suffix = nil)
+        suffix = Hstring.new(suffix).underscore.presence || index_suffix || Esse.timestamp
+
+        index_name(suffix: suffix)
+      end
+    end
+
+    extend ClassMethods
+  end
+end

data/lib/esse/index/base.rb

@@ -3,48 +3,23 @@
 module Esse
   class Index
     module ClassMethods
-      # Define a Index method on the given module that calls the Index
-      # method on the receiver. This is how the Esse::Index() method is
-      # defined, and allows you to define Index() methods on other modules,
-      # making it easier to have custom index settings for all indices under
-      # a namespace.  Example:
-      #
-      #   module V1
-      #     EsIndex = Class.new(Esse::Index)
-      #     EsIndex.def_Index(self)
-      #
-      #     class Bar < EsIndex
-      #       # Uses :default elasticsearch client connection
-      #     end
-      #
-      #     class Baz < EsIndex(:v1)
-      #       # Uses :v1 elasticsearch client connection
-      #     end
-      #   end
-      def def_Index(index_module) # rubocop:disable Naming/MethodName
-        tap do |model|
-          index_module.define_singleton_method(:Index) do |source|
-            model.Index(source)
-          end
+      # Sets the client_id associated with the Index class.
+      # This can be used directly on Esse::Index to set the :default es cluster
+      # to be used by subclasses, or to override the es client used for specific indices:
+      #   Esse::Index.cluster_id = :v1
+      #   ArtistIndex = Class.new(Esse::Index)
+      #   ArtistIndex.cluster_id = :v2
+      # @param [Symbol, Esse::Cluster, NilClass] source the cluster id or the cluster instance
+      # @return [Symbol] the cluster id
+      # @raise [ArgumentError] if the cluster id is not defined in the Esse.config
+      def cluster_id=(source)
+        if source.nil?
+          @cluster_id = nil
+          return
         end
-      end
-
-      # Lets you create a Index subclass with its elasticsearch cluster
-      #
-      # Example:
-      #   # Using a custom cluster
-      #   Esse.config.cluster(:v1).client = Elasticsearch::Client.new
-      #   class UsersIndex < Esse::Index(:v1)
-      #   end
-      #
-      #   # Using :default cluster
-      #   class UsersIndex < Esse::Index
-      #   end
-      def Index(source) # rubocop:disable Naming/MethodName
-        klass = Class.new(self)
 
         valid_ids = Esse.config.cluster_ids
-        klass.cluster_id = \
+        new_id = \
           case source
           when Esse::Cluster
             source.id
@@ -55,7 +30,7 @@ module Esse
 
         msg = <<~MSG
           We could not resolve the index cluster using the argument %<arg>p. \n
-          It must be previously defined in the `Esse.config' settings. \n
+          It must be previously defined in the `Esse.config.cluster(%<arg>p) { ... }' settings. \n
           Here is the list of cluster ids we have configured: %<ids>s\n
 
           You can ignore this cluster id entirely. That way the :default id will be used.\n
@@ -63,22 +38,11 @@ module Esse
             class UsersIndex < Esse::Index\n
             end\n
         MSG
-        unless klass.cluster_id
+        unless new_id
           raise ArgumentError.new, format(msg, arg: source, ids: valid_ids.map(&:inspect).join(', '))
         end
 
-        klass.type_hash = {}
-        klass
-      end
-
-      # Sets the client_id associated with the Index class.
-      # This can be used directly on Esse::Index to set the :default es cluster
-      # to be used by subclasses, or to override the es client used for specific indices:
-      #   Esse::Index.cluster_id = :v1
-      #   ArtistIndex = Class.new(Esse::Index)
-      #   ArtistIndex.cluster_id = :v2
-      def cluster_id=(cluster_id)
-        @cluster_id = cluster_id
+        @cluster_id = new_id
       end
 
       # @return [Symbol] reads the @cluster_id instance variable or :default

data/lib/esse/index/documents.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    module ClassMethods
+      # Retrieves the specified JSON document from an index.
+      #
+      #   UsersIndex.get(id: 1) # { '_id' => 1, ... }
+      #   UsersIndex.get(id: 'missing') # raise Esse::Transport::NotFoundError
+      #
+      # @param doc [Esse::Document] the document to retrieve
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, Integer] :id The `_id` of the elasticsearch document
+      # @option [String, NilClass] :type The type of the document (Optional for elasticsearch >= 7)
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      # @raise [Esse::Transport::NotFoundError] when the doc does not exist
+      # @return [Hash] The elasticsearch document.
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-get.html
+      def get(doc = nil, suffix: nil, **options)
+        if document?(doc)
+          options[:id] = doc.id
+          options[:type] = doc.type if doc.type?
+          options[:routing] = doc.routing if doc.routing?
+        end
+        require_kwargs!(options, :id)
+        options[:index] = index_name(suffix: suffix)
+        cluster.may_update_type!(options)
+        cluster.api.get(**options)
+      end
+
+      # Check if a JSON document exists
+      #
+      #   UsersIndex.exist?(id: 1) # true
+      #   UsersIndex.exist?(id: 'missing') # false
+      #
+      # @param doc [Esse::Document] the document to retrieve
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, Integer] :id The `_id` of the elasticsearch document
+      # @option [String, NilClass] :type The type of the document (Optional for elasticsearch >= 7)
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      # @return [Boolean] true if the document exists
+      def exist?(doc = nil, suffix: nil, **options)
+        if document?(doc)
+          options[:id] = doc.id
+          options[:type] = doc.type if doc.type?
+          options[:routing] = doc.routing if doc.routing?
+        end
+        require_kwargs!(options, :id)
+        options[:index] = index_name(suffix: suffix)
+        cluster.may_update_type!(options)
+        cluster.api.exist?(**options)
+      end
+
+      # Gets the number of matches for a search query.
+      #
+      #   UsersIndex.count # 999
+      #   UsersIndex.count(body: { ... }) # 32
+      #
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [Hash] :body A query to restrict the results specified with the Query DSL (optional)
+      # @option [String, NilClass] :type The type of the document (Optional for elasticsearch >= 7)
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      # @return [Integer] amount of documents found
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/search-count.html
+      def count(type: nil, suffix: nil, **options)
+        params = {
+          index: index_name(suffix: suffix),
+          type: type,
+        }
+        cluster.may_update_type!(params)
+        cluster.api.count(**options, **params)['count']
+      end
+
+      # Removes a JSON document from the specified index.
+      #
+      #   UsersIndex.delete(id: 1) # true
+      #   UsersIndex.delete(id: 'missing') # false
+      #
+      # @param doc [Esse::Document] the document to retrieve
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, Integer] :id The `_id` of the elasticsearch document
+      # @option [String, NilClass] :type The type of the document (Optional for elasticsearch >= 7)
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      # @raise [Esse::Transport::NotFoundError] when the doc does not exist
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-delete.html
+      def delete(doc = nil, suffix: nil, **options)
+        if document?(doc)
+          options[:id] = doc.id
+          options[:type] = doc.type if doc.type?
+          options[:routing] = doc.routing if doc.routing?
+        end
+        require_kwargs!(options, :id)
+        options[:index] = index_name(suffix: suffix)
+        cluster.may_update_type!(options)
+        cluster.api.delete(**options)
+      end
+
+      # Updates a document using the specified script.
+      #
+      #   UsersIndex.update(id: 1, body: { doc: { ... } }) # { '_id' => 1, ...}
+      #
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, Integer] :id The `_id` of the elasticsearch document
+      # @option [Hash] :body the body of the request
+      # @option [String, NilClass] :type The type of the document (Optional for elasticsearch >= 7)
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      # @raise [Esse::Transport::NotFoundError] when the doc does not exist
+      # @return [Hash] elasticsearch response hash
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-update.html
+      def update(doc = nil, suffix: nil, **options)
+        if document?(doc)
+          options[:id] = doc.id
+          options[:body] = { doc: doc.source }
+          options[:type] = doc.type if doc.type?
+          options[:routing] = doc.routing if doc.routing?
+        end
+        require_kwargs!(options, :id, :body)
+        options[:index] = index_name(suffix: suffix)
+        cluster.may_update_type!(options)
+        cluster.api.update(**options)
+      end
+
+      # Adds a JSON document to the specified index and makes it searchable. If the document
+      # already exists, updates the document and increments its version.
+      #
+      #   UsersIndex::User.index(id: 1, body: { name: 'name' }) # { '_id' => 1, ...}
+      #
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, Integer] :id The `_id` of the elasticsearch document
+      # @option [Hash] :body The JSON document that will be indexed (Required)
+      # @option [String, NilClass] :type The type of the document (Optional for elasticsearch >= 7)
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      # @return [Hash] the elasticsearch response Hash
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-index_.html
+      def index(doc = nil, suffix: nil, **options)
+        if document?(doc)
+          options[:id] = doc.id
+          options[:body] = doc.source
+          options[:type] = doc.type if doc.type?
+          options[:routing] = doc.routing if doc.routing?
+        end
+        require_kwargs!(options, :id, :body)
+        options[:index] = index_name(suffix: suffix)
+        cluster.may_update_type!(options)
+        cluster.api.index(**options)
+      end
+
+      # Performs multiple indexing or delete operations in a single API call.
+      # This reduces overhead and can greatly increase indexing speed.
+      #
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      # @option [Array<Esse::Document>] :index list of documents to be indexed(Optional)
+      # @option [Array<Esse::Document>] :delete list of documents to be deleted(Optional)
+      # @option [Array<Esse::Document>] :create list of documents to be created(Optional)
+      # @option [String, NilClass] :type The type of the document (Optional for elasticsearch >= 7)
+      # @return [Array<Esse::Import::RequestBody>] The list of request bodies. @TODO Change this to a Stats object
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.5/docs-bulk.html
+      # @see https://github.com/elastic/elasticsearch-ruby/blob/main/elasticsearch-api/lib/elasticsearch/api/utils.rb
+      # @see https://github.com/elastic/elasticsearch-ruby/blob/main/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
+      def bulk(index: nil, delete: nil, create: nil, type: nil, suffix: nil, **options)
+        definition = {
+          index: index_name(suffix: suffix),
+          type: type,
+        }.merge(options)
+        cluster.may_update_type!(definition)
+
+        # @TODO Wrap the return in a some other Stats object with more information
+        Esse::Import::Bulk.new(
+          **definition.slice(:type),
+          index: index,
+          delete: delete,
+          create: create,
+        ).each_request do |request_body|
+          cluster.api.bulk(**definition, body: request_body.body) do |event_payload|
+            event_payload[:body_stats] = request_body.stats
+            if bulk_wait_interval > 0
+              event_payload[:wait_interval] = bulk_wait_interval
+              sleep(bulk_wait_interval)
+            else
+              event_payload[:wait_interval] = 0.0
+            end
+          end
+        end
+      end
+
+      # Resolve collection and index data
+      #
+      # @param repos [Array<String>] List of repo types. Defaults to all types.
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      # @option [Hash] :context The collection context. This value will be passed as argument to the collection
+      #   May be SQL condition or any other filter you have defined on the collection.
+      # @return [Numeric] The number of documents imported
+      def import(*repo_types, context: {}, suffix: nil, **options)
+        repo_types = repo_hash.keys if repo_types.empty?
+        count = 0
+        repo_hash.slice(*repo_types).each do |repo_name, repo|
+          repo.each_serialized_batch(**(context || {})) do |batch|
+            # Elasticsearch 6.x and older have multiple types per index.
+            # This gem supports multiple types per index for backward compatibility, but we recommend to update
+            # your elasticsearch to a at least 7.x version and use a single type per index.
+            #
+            # Note that the repository name will be used as the document type.
+            # mapping_default_type
+            kwargs = { index: batch, suffix: suffix, type: repo_name, **options }
+            cluster.may_update_type!(kwargs)
+            bulk(**kwargs)
+            count += batch.size
+          end
+        end
+        count
+      end
+
+      protected
+
+      def document?(doc)
+        Esse.document?(doc)
+      end
+
+      def require_kwargs!(options, *keys)
+        keys.each do |key|
+          raise ArgumentError, "missing keyword: #{key}" unless options.key?(key)
+        end
+      end
+    end
+
+    extend ClassMethods
+  end
+end