esse 0.0.3 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d7f91a4ecb008fe3d8849175cb6f1bb08605d56432b7bc7f52ca9df0c55e77e
-  data.tar.gz: a9b4143597ca2ca571360814b381e8398bd3ac8b45134fc5b3086dfc8a33f0a1
+  metadata.gz: 4dff3560f305debeee691263267e43df0e70a17591f6ecfc897025a5c07ada8a
+  data.tar.gz: a8ba452b0bfc81210f104293f852ecacaef585ea23b2e8f24177197fa2555e69
 SHA512:
-  metadata.gz: ef25dc5238beea72d67eeaf146dad16184cd8d78c76fc84516154e8ddd5cb93f2c5f1fd84574359147d6428dbdb1d25f037d0dce004f79e4e22cbbf1dcc2a30b
-  data.tar.gz: 6c5237fc9f5b59376ef6e948132c5a78200f4f849acab591ba603b3c5173d95c6179dfec3844331a6ba30f41ae519515b12c3f3396d80e4bff5dd3445525faad
+  metadata.gz: 73bfc1a130c18df947cd02120087eed2b571fa174b84009b687dc0e343697aec1a1a20ae00eb0fc8465a9ad603d95318c02d324b98fe811baddc0b08d3533e28
+  data.tar.gz: 160aaf9fdef9da05108c1e966cfc03a45f9357aeb4682d551458546f66dc067d0585e52f0836aa0b4ee8fb5d634d3b705bced6340e91a99013d571fcded5f574

data/exec/esse

@@ -6,4 +6,6 @@ $LOAD_PATH.unshift File.expand_path('../lib', __dir__)
 require 'esse'
 require 'esse/cli'
 
-Esse::CLI.start(ARGV)
+Esse::CLI.with_friendly_errors do
+  Esse::CLI.start(ARGV)
+end

data/lib/esse/backend/index/aliases.rb

@@ -44,10 +44,14 @@ module Esse
               *indices.map do |index|
                 { remove: { index: index, alias: index_name } }
               end,
-              { add: {index: real_index_name(suffix), alias: index_name } }
+              { add: {index: build_real_index_name(suffix), alias: index_name } }
             ],
           }
-          client.indices.update_aliases(options)
+
+          Esse::Events.instrument('elasticsearch.update_aliases') do |payload|
+            payload[:request] = options
+            payload[:response] = client.indices.update_aliases(options)
+          end
         end
 
         # Replaces all existing aliases by the respective suffixed index from argument.
@@ -55,11 +59,11 @@ module Esse
         # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
         # @option [String] :suffix The suffix of the index used for versioning.
         # @raise [Elasticsearch::Transport::Transport::Errors::NotFound] in case of failure
-        # @return [Hash, false] the elasticsearch response, or false in case of failure
+        # @return [Hash] the elasticsearch response, or an hash with 'errors' as true in case of failure
         def update_aliases(suffix:, **options)
           update_aliases!(suffix: suffix, **options)
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          false
+          { 'errors' => true }
         end
       end
 

data/lib/esse/backend/index/close.rb

@@ -21,9 +21,10 @@ module Esse
         #
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html
         def close!(suffix: index_version, **options)
-          name = suffix ? real_index_name(suffix) : index_name
-
-          client.indices.close(options.merge(index: name))
+          Esse::Events.instrument('elasticsearch.close') do |payload|
+            payload[:request] = attributes = options.merge(index: index_name(suffix: suffix))
+            payload[:response] = client.indices.close(**attributes)
+          end
         end
 
         # Close an index (keep the data on disk, but deny operations with the index).
@@ -37,13 +38,13 @@ module Esse
         # @option options [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when
         #   unavailable (missing, closed, etc)
         # @option options [Time] :timeout Explicit operation timeout
-        # @return [Hash, false] the elasticsearch response, or false in case of failure
+        # @return [Hash] the elasticsearch response, or an hash with 'errors' as true in case of failure
         #
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html
         def close(suffix: index_version, **options)
           close!(suffix: suffix, **options)
         rescue Elasticsearch::Transport::Transport::ServerError
-          false
+          { 'errors' => true }
         end
       end
 

data/lib/esse/backend/index/create.rb

@@ -10,43 +10,54 @@ module Esse
 
         # Creates index and applies mappings and settings.
         #
-        #   UsersIndex.backend.create_index # creates index named `<prefix_>users_<suffix|index_version|timestamp>`
+        #   UsersIndex.elasticsearch.create_index # creates index named `<prefix_>users_<suffix|index_version|timestamp>`
         #
         # @param options [Hash] Options hash
         # @option options [Boolean] :alias Update `index_name` alias along with the new index
         # @option options [String] :suffix The index suffix. Defaults to the `IndexClass#index_version` or
         #   `Esse.timestamp`. Suffixed index names might be used for zero-downtime mapping change.
-        # @return [Hash, false] the elasticsearch response or false in case of unsuccessful creation.
+        # @return [Hash] the elasticsearch response, or an hash with 'errors' as true in case of failure
         #
         # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
-        def create_index(suffix: nil, **options)
+        def create_index(suffix: index_version, **options)
           create_index!(suffix: suffix, **options)
         rescue Elasticsearch::Transport::Transport::Errors::BadRequest
-          false
+          { 'errors' => true }
         end
 
         # Creates index and applies mappings and settings.
         #
-        #   UsersIndex.backend.create_index! # creates index named `<prefix_>users_<suffix|index_version|timestamp>`
+        #   UsersIndex.elasticsearch.create_index! # creates index named `<prefix_>users_<suffix|index_version|timestamp>`
         #
         # @param options [Hash] Options hash
         # @option options [Boolean] :alias Update `index_name` alias along with the new index
         # @option options [String] :suffix The index suffix. Defaults to the `IndexClass#index_version` or
         #   `Esse.timestamp`. Suffixed index names might be used for zero-downtime mapping change.
+        # @option arguments [String] :wait_for_active_shards Set the number of active shards
+        #    to wait for before the operation returns.
+        # @option arguments [Time] :timeout Explicit operation timeout
+        # @option arguments [Time] :master_timeout Specify timeout for connection to master
+        # @option arguments [Hash] :headers Custom HTTP headers
+        # @option arguments [Hash] :body The configuration for the index (`settings` and `mappings`)
         # @raise [Elasticsearch::Transport::Transport::Errors::NotFound] when index already exists
         # @return [Hash] the elasticsearch response
         #
         # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
-        def create_index!(suffix: nil, **options)
+        def create_index!(suffix: index_version, **options)
           options = DEFAULT_OPTIONS.merge(options)
-          name = real_index_name(suffix)
+          name = build_real_index_name(suffix)
           definition = [settings_hash, mappings_hash].reduce(&:merge)
 
-          if options[:alias] && name != index_name
+          if options.delete(:alias) && name != index_name
             definition[:aliases] = { index_name => {} }
           end
 
-          client.indices.create(index: name, body: definition)
+          Esse::Events.instrument('elasticsearch.create_index') do |payload|
+            payload[:request] = opts = options.merge(index: name, body: definition)
+            payload[:response] = response = client.indices.create(**opts)
+            cluster.wait_for_status! if response
+            response
+          end
         end
       end
 

data/lib/esse/backend/index/delete.rb

@@ -6,29 +6,30 @@ module Esse
       module InstanceMethods
         # Deletes ES index
         #
-        #   UsersIndex.backend.delete_index! # deletes `<prefix_>users<_suffix|_index_version|_timestamp>` index
+        #   UsersIndex.elasticsearch.delete_index! # deletes `<prefix_>users<_suffix|_index_version|_timestamp>` index
         #
-        # @param options [Hash] Options hash
-        # @option [String, nil] :suffix The index suffix Use nil if you want to delete the current index.
+        # @param suffix [String, nil] The index suffix Use nil if you want to delete the current index.
         # @raise [Elasticsearch::Transport::Transport::Errors::NotFound] when index does not exists
         # @return [Hash] elasticsearch response
-        def delete_index!(suffix:)
-          name = suffix ? real_index_name(suffix) : index_name
-
-          client.indices.delete(index: name)
+        def delete_index!(suffix: index_version, **options)
+          Esse::Events.instrument('elasticsearch.delete_index') do |payload|
+            payload[:request] = opts = options.merge(index: index_name(suffix: suffix))
+            payload[:response] = response = client.indices.delete(**opts)
+            cluster.wait_for_status! if response
+            response
+          end
         end
 
         # Deletes ES index
         #
-        #   UsersIndex.backend.delete_index # deletes `<prefix_>users<_suffix|_index_version|_timestamp>` index
+        #   UsersIndex.elasticsearch.delete_index # deletes `<prefix_>users<_suffix|_index_version|_timestamp>` index
         #
-        # @param options [Hash] Options hash
-        # @option [String] :suffix The index suffix. Use nil if you want to delete the current index.
-        # @return [Hash, false] elasticsearch response, of false in case of error.
-        def delete_index(suffix: index_version)
-          delete_index!(suffix: suffix)
+        # @param suffix [String, nil] The index suffix Use nil if you want to delete the current index.
+        # @return [Hash] the elasticsearch response, or an hash with 'errors' as true in case of failure
+        def delete_index(suffix: index_version, **options)
+          delete_index!(suffix: suffix, **options)
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          false
+          { 'errors' => true }
         end
       end
 

data/lib/esse/backend/index/documents.rb

@@ -6,13 +6,13 @@ module Esse
       module InstanceMethods
         def import!(**options)
           type_hash.each_value do |type|
-            type.backend.import!(**options)
+            type.elasticsearch.import!(**options)
           end
         end
 
         def import(**options)
           type_hash.each_value do |type|
-            type.backend.import(**options)
+            type.elasticsearch.import(**options)
           end
         end
       end

data/lib/esse/backend/index/existance.rb

@@ -6,14 +6,13 @@ module Esse
       module InstanceMethods
         # Checks the index existance. Returns true or false
         #
-        #   UsersIndex.backend.exist? #=> true
+        #   UsersIndex.elasticsearch.exist? #=> true
         #
         # @param options [Hash] Options hash
         # @option options [String, nil] :suffix The index suffix. Defaults to the index_version.
         #   Use nil if you want to check existence of the `index_name` index or alias.
         def exist?(suffix: index_version)
-          name = suffix ? real_index_name(suffix) : index_name
-          client.indices.exists(index: name)
+          client.indices.exists(index: index_name(suffix: suffix))
         end
       end
 

data/lib/esse/backend/index/open.rb

@@ -21,9 +21,10 @@ module Esse
         #
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-open.html
         def open!(suffix: index_version, **options)
-          name = suffix ? real_index_name(suffix) : index_name
-
-          client.indices.open(options.merge(index: name))
+          Esse::Events.instrument('elasticsearch.open') do |payload|
+            payload[:request] = attributes = options.merge(index: index_name(suffix: suffix))
+            payload[:response] = client.indices.open(**attributes)
+          end
         end
 
         # Open a previously closed index
@@ -37,13 +38,13 @@ module Esse
         # @option options [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when
         #   unavailable (missing, closed, etc)
         # @option options [Time] :timeout Explicit operation timeout
-        # @return [Hash, false] the elasticsearch response, or false in case of failure
+        # @return [Hash] the elasticsearch response, or an hash with 'errors' as true in case of failure
         #
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-open.html
         def open(suffix: index_version, **options)
           open!(suffix: suffix, **options)
         rescue Elasticsearch::Transport::Transport::ServerError
-          false
+          { 'errors' => true }
         end
       end
 

data/lib/esse/backend/index/refresh.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Esse
+  module Backend
+    class Index
+      module InstanceMethods
+        # Performs the refresh operation in one or more indices.
+        #
+        # @note The refresh operation can adversely affect indexing throughput when used too frequently.
+        # @param :suffix [String, nil] :suffix The index suffix. Defaults to the index_version.
+        #   A uniq index name will be generated if one index already exist with the given alias.
+        # @param options [Hash] Options hash
+        # @raise [Elasticsearch::Transport::Transport::Errors::BadRequest, Elasticsearch::Transport::Transport::Errors::NotFound]
+        #   in case of failure
+        # @return [Hash] the elasticsearch response
+        #
+        # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-refresh.html
+        def refresh!(suffix: index_version, **options)
+          client.indices.refresh(
+            options.merge(index: index_name(suffix: suffix)),
+          )
+        end
+
+        # Performs the refresh operation in one or more indices.
+        #
+        # @note The refresh operation can adversely affect indexing throughput when used too frequently.
+        # @param :suffix [String, nil] :suffix The index suffix. Defaults to the index_version.
+        #   A uniq index name will be generated if one index already exist with the given alias.
+        # @param options [Hash] Options hash
+        # @return [Hash] the elasticsearch response, or an hash with 'errors' as true in case of failure
+        #
+        # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-refresh.html
+        def refresh(suffix: index_version, **options)
+          refresh!(suffix: suffix, **options)
+        rescue Elasticsearch::Transport::Transport::ServerError
+          { 'errors' => true }
+        end
+      end
+
+      include InstanceMethods
+    end
+  end
+end

data/lib/esse/backend/index/reset.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Esse
+  module Backend
+    class Index
+      module InstanceMethods
+        # Deletes, creates and imports data to the index. Performs zero-downtime index resetting.
+        #
+        # @option options [String, nil] :suffix The index suffix. Defaults to the index_version.
+        #   A uniq index name will be generated if one index already exist with the given alias.
+        # @option options [Time] :timeout Explicit operation timeout
+        # @raise [Elasticsearch::Transport::Transport::Errors::BadRequest, Elasticsearch::Transport::Transport::Errors::NotFound]
+        #   in case of failure
+        # @return [Hash] the elasticsearch response
+        #
+        # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html
+        def reset_index!(suffix: index_version, **options)
+          existing = []
+          suffix ||= Esse.timestamp
+          suffix = Esse.timestamp while exist?(suffix: suffix).tap { |exist| existing << suffix if exist }
+
+          create_index!(suffix: suffix, **options)
+          import!(suffix: suffix, **options)
+          update_aliases!(suffix: suffix)
+          existing.each { |_s| delete_index!(suffix: suffix, **options) }
+          true
+        end
+      end
+
+      include InstanceMethods
+    end
+  end
+end

data/lib/esse/backend/index/update.rb

@@ -27,9 +27,14 @@ module Esse
         #
         # @see http://www.elasticsearch.org/guide/reference/api/admin-indices-put-mapping/
         def update_mapping!(suffix: index_version, **options)
-          name = suffix ? real_index_name(suffix) : index_name
-
-          client.indices.put_mapping(options.merge(index: name, body: mappings_hash.fetch(Esse::MAPPING_ROOT_KEY)))
+          Esse::Events.instrument('elasticsearch.update_mapping') do |payload|
+            body = mappings_hash.fetch(Esse::MAPPING_ROOT_KEY)
+            if (type = options[:type])
+              body = body[type.to_s] || body[type.to_sym]
+            end
+            payload[:request] = opts = options.merge(index: index_name(suffix: suffix), body: body)
+            payload[:response] = client.indices.put_mapping(**opts)
+          end
         end
 
         # Create or update a mapping
@@ -49,16 +54,15 @@ module Esse
         #   with the same name across all types
         # @option options [Time] :timeout Explicit operation timeout
         # @option options [Boolean] :master_timeout Timeout for connection to master
-        # @return [Hash, false] the elasticsearch response, or false in case of failure
+        # @return [Hash] the elasticsearch response, or an hash with 'errors' as true in case of failure
         #
         # @see http://www.elasticsearch.org/guide/reference/api/admin-indices-put-mapping/
         def update_mapping(suffix: index_version, **options)
           update_mapping!(suffix: suffix, **options)
         rescue Elasticsearch::Transport::Transport::ServerError
-          false
+          { 'errors' => true }
         end
 
-
         # Closes the index for read/write operations, updates the index settings, and open it again
         #
         # @option options [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that
@@ -78,15 +82,33 @@ module Esse
         #
         # @see http://www.elasticsearch.org/guide/reference/api/admin-indices-update-settings/
         def update_settings!(suffix: index_version, **options)
-          name = suffix ? real_index_name(suffix) : index_name
           response = nil
 
-          close!(suffix: suffix)
-          begin
-            body = settings_hash(cluster_settings: false).fetch(Esse::SETTING_ROOT_KEY)
-            response = client.indices.put_settings(options.merge(index: name, body: body))
-          ensure
-            open!(suffix: suffix)
+          settings = settings_hash.fetch(Esse::SETTING_ROOT_KEY).transform_keys(&:to_s)
+          settings.delete('number_of_shards') # Can't change number of shards for an index
+          analysis = settings.delete('analysis')
+
+          if settings.any?
+            # When changing the number of replicas the index needs to be open. Changing the number of replicas on a
+            # closed index might prevent the index to be opened correctly again.
+            Esse::Events.instrument('elasticsearch.update_settings') do |payload|
+              payload[:request] = opts = options.merge(index: index_name(suffix: suffix), body: { index: settings })
+              payload[:response] = response = client.indices.put_settings(**opts)
+            end
+          end
+
+          if analysis
+            # It is also possible to define new analyzers for the index. But it is required to close the
+            # index first and open it after the changes are made.
+            close!(suffix: suffix)
+            begin
+              Esse::Events.instrument('elasticsearch.update_settings') do |payload|
+                payload[:request] = opts = options.merge(index: index_name(suffix: suffix), body: { analysis: analysis })
+                payload[:response] = response = client.indices.put_settings(**opts)
+              end
+            ensure
+              open!(suffix: suffix)
+            end
           end
 
           response
@@ -105,13 +127,13 @@ module Esse
         #   If set to `true` existing settings on an index remain unchanged, the default is `false`
         # @option options [Time] :master_timeout Specify timeout for connection to master
         # @option options [Boolean] :flat_settings Return settings in flat format (default: false)
-        # @return [Hash, false] the elasticsearch response, false in case of failure
+        # @return [Hash] the elasticsearch response, or an hash with 'errors' as true in case of failure
         #
         # @see http://www.elasticsearch.org/guide/reference/api/admin-indices-update-settings/
         def update_settings(suffix: index_version, **options)
           update_settings!(suffix: suffix, **options)
         rescue Elasticsearch::Transport::Transport::ServerError
-          false
+          { 'errors' => true }
         end
       end
 

data/lib/esse/backend/index.rb

@@ -10,13 +10,15 @@ module Esse
       require_relative 'index/delete'
       require_relative 'index/existance'
       require_relative 'index/update'
+      require_relative 'index/refresh'
+      require_relative 'index/reset'
       require_relative 'index/documents'
       require_relative 'index/open'
       require_relative 'index/close'
 
       extend Forwardable
 
-      NAMING = %i[index_name index_version].freeze
+      NAMING = %i[index_version].freeze
       DEFINITION = %i[settings_hash mappings_hash].freeze
 
       def_delegators :@index, :type_hash, *(NAMING + DEFINITION)
@@ -27,13 +29,25 @@ module Esse
 
       protected
 
-      def real_index_name(suffix = nil)
+      def index_name(suffix: nil)
+        suffix = Hstring.new(suffix).underscore.presence
+        return @index.index_name unless suffix
+
+        [@index.index_name, suffix].join('_')
+      end
+
+      def build_real_index_name(suffix = nil)
         suffix = Hstring.new(suffix).underscore.presence || index_version || Esse.timestamp
-        [index_name, suffix].compact.join('_')
+
+        index_name(suffix: suffix)
       end
 
       def client
-        @index.cluster.client
+        cluster.client
+      end
+
+      def cluster
+        @index.cluster
       end
     end
   end