esse 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a9a18ea9cc75f6e98edecd0c151fe3077dc362529dc562e380855ec651b898e
-  data.tar.gz: 3d1a96c3ba0a088e2a88b5e7aa3c1440662877d6ab2dc3e1edbd7276fe39611e
+  metadata.gz: 30ce86d71e0cf9b67506dd0caf1651b91ae836c9e7981e46a238446c8afde078
+  data.tar.gz: 9ec1069cbb8106743d5832f94fbce5d2c937037a62a746350e736385f3b78cdc
 SHA512:
-  metadata.gz: a70c285401ed1f1d3c43ebb73970339d9c48344d1872a8f5c8391ead77c75a3aeabbc8391acd88971e8bdfcc370abafdab726884deb86d713aabe75bf553c3da
-  data.tar.gz: 6ca5282a53d4e7b29bbe29e149fd5405c9a25a6bd561bc803969e173955f83625c33a6f5b1116280f5a327b54ad4ec5923b58844543da63c51ff029e805e64ec
+  metadata.gz: f2a5012e3d3dab0b3894f8e8fb1665be2a57c8538a3fc02bce2ac089510bd63b982f8f919554a9f16d6e8dafe5d93d9f5614d99ca4f170d82cc41e413fec0209
+  data.tar.gz: 034b15fd5cfc4f6ebdd97ba9d73ed284f50c209225b2c6a43b27216a57578ca1b91d9c92d4e020c4a1c7910038cf5978054bfa4b491bafb431bcfc1dcbc0a706

data/lib/esse/cli/event_listener.rb

@@ -84,17 +84,16 @@ module Esse
       end
 
       def elasticsearch_bulk(event)
-        print_message("[%<runtime>s] Bulk index %<name>s%<type>s%<wait_interval>s: ",
+        print_message('[%<runtime>s] Bulk index %<name>s%<type>s%<wait_interval>s: ',
           runtime: formatted_runtime(event[:runtime]),
           name: colorize(event[:request][:index], :bold),
           type: (event[:request][:type] ? " for type #{colorize(event[:request][:type], :bold)}" : ''),
           wait_interval: (event[:wait_interval].nonzero? ? " (wait interval #{event[:wait_interval]}s)" : ''),
-          newline: false,
-        )
-        stats = event[:request][:body_stats].select {|_, v| v.nonzero? }.map do |type, count|
+          newline: false,)
+        stats = event[:request][:body_stats].select { |_, v| v.nonzero? }.map do |type, count|
           "#{colorize(type, :bold)}: #{count} docs"
         end
-        print_message(stats.join(", ") + ".")
+        print_message(stats.join(', ') + '.')
       end
     end
   end

data/lib/esse/cli/generate.rb

@@ -16,19 +16,17 @@ module Esse
       desc 'index NAME *TYPES', 'Creates a new index'
       option :settings, type: :boolean, default: false, desc: 'Generate settings'
       option :mappings, type: :boolean, default: false, desc: 'Generate mappings'
-      option :serializers, type: :boolean, default: false, desc: 'Generate serializers'
+      option :documents, type: :boolean, default: false, desc: 'Generate documents'
       option :collections, type: :boolean, default: false, desc: 'Generate collections'
       option :active_record, type: :boolean, default: false, desc: 'Generate ActiveRecord models'
       option :cluster_id, type: :string, desc: 'Elasticsearch cluster ID'
-      def index(name, *types)
+      def index(name, *repos)
         ns_path = name.split(NAMESPACE_PATTERN_RE).tap(&:pop)
         @index_name = Hstring.new(name.to_s).modulize.sub(/Index$/, '') + 'Index'
         @index_name = Hstring.new(@index_name)
-        @types = types.map { |type| Hstring.new(type) }
+        @repos = repos.map { |repo| Hstring.new(repo) }
         @base_class = base_index_class(*ns_path)
-        if options[:cluster_id]
-          @base_class += format('(:%s)', options[:cluster_id])
-        end
+        @index_cluster_id = options[:cluster_id]
         @cli_options = options
 
         base_dir = Esse.config.indices_directory.join(*ns_path.map { |n| Hstring.new(n).underscore.to_s })
@@ -52,11 +50,11 @@ module Esse
           )
         end
 
-        if @types.empty?
-          if options[:serializers]
+        if @repos.empty?
+          if options[:documents]
             template(
-              'templates/serializer.rb.erb',
-              base_dir.join(index_name, 'serializers', 'serializer.rb'),
+              'templates/document.rb.erb',
+              base_dir.join(index_name, 'documents', 'document.rb'),
             )
           end
           if options[:collections] && !options[:active_record]
@@ -67,19 +65,19 @@ module Esse
           end
         end
 
-        @types.each do |type|
-          @type = Hstring.new(type).underscore
+        @repos.each do |type|
+          @repo = Hstring.new(type).underscore
 
-          if options[:serializers]
+          if options[:documents]
             template(
-              'templates/serializer.rb.erb',
-              base_dir.join(index_name, 'serializers', "#{@type}_serializer.rb"),
+              'templates/document.rb.erb',
+              base_dir.join(index_name, 'documents', "#{@repo}_document.rb"),
             )
           end
           if options[:collections] && !options[:active_record]
             template(
               'templates/collection.rb.erb',
-              base_dir.join(index_name, 'collections', "#{@type}_collection.rb"),
+              base_dir.join(index_name, 'collections', "#{@repo}_collection.rb"),
             )
           end
         end

data/lib/esse/cli/index/close.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.close!(**options)
+          index.close(**options)
         end
       end
 

data/lib/esse/cli/index/create.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.create_index!(**options)
+          index.create_index(**options)
         end
       end
 

data/lib/esse/cli/index/delete.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.delete_index!(**options)
+          index.delete_index(**options)
         end
       end
 

data/lib/esse/cli/index/import.rb

@@ -9,9 +9,9 @@ module Esse
         validate_options!
         indices.each do |index|
           if (repo = @options[:repo])
-            index.elasticsearch.import!(repo, **options)
+            index.import(repo, **options)
           else
-            index.elasticsearch.import!(**options)
+            index.import(**options)
           end
         end
       end

data/lib/esse/cli/index/open.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.open!(**options)
+          index.open(**options)
         end
       end
 

data/lib/esse/cli/index/reset.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.reset_index!(**options)
+          index.reset_index(**options)
         end
       end
 

data/lib/esse/cli/index/update_aliases.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.update_aliases!(**options)
+          index.update_aliases(**options)
         end
       end
 
@@ -21,7 +21,7 @@ module Esse
       def validate_options!
         validate_indices_option!
 
-        if @options[:suffix].nil?
+        if @options[:suffix].nil? || @options[:suffix].empty?
           raise InvalidOption.new(<<~END)
             You must specify a suffix to update the aliases.
           END

data/lib/esse/cli/index/update_mapping.rb

@@ -9,11 +9,16 @@ module Esse
         validate_options!
         indices.each do |index|
           if !index.mapping_single_type?
-            index.repo_hash.values.map(&:document_type).uniq.each do |doc_type|
-              index.elasticsearch.update_mapping!(type: doc_type, **options)
+            # Elasticsearch 6.x and older have multiple types per index.
+            # This gem supports multiple types per index for backward compatibility, but we recommend to update
+            # your elasticsearch to a at least 7.x version and use a single type per index.
+            #
+            # Note that the repository name will be used as the document type.
+            index.repo_hash.keys.each do |doc_type|
+              index.update_mapping(type: doc_type, **options)
             end
           else
-            index.elasticsearch.update_mapping!(**options)
+            index.update_mapping(**options)
           end
         end
       end

data/lib/esse/cli/index/update_settings.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.update_settings!(**options)
+          index.update_settings(**options)
         end
       end
 

data/lib/esse/cli/index.rb

@@ -21,12 +21,14 @@ module Esse
         Reset.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
       end
 
+      # @TODO Add reindex task to create a new index and import documents from the old index using _reindex API
+
       desc 'create *INDEX_CLASSES', 'Creates indices for the given classes'
       long_desc <<-DESC
         Creates index and applies mapping and settings for the given classes.
 
         Indices are created with the following naming convention:
-        <cluster.index_prefix>_<index_class.index_name>_<index_class.index_version>.
+        <cluster.index_prefix>_<index_class.index_name>_<index_class.index_suffix>.
       DESC
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :alias, type: :boolean, default: false, aliases: '-a', desc: 'Update alias after create index'
@@ -42,11 +44,14 @@ module Esse
         Delete.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
       end
 
-      desc 'update_aliases *INDEX_CLASS', 'Replaces all existing aliases by the given suffix'
-      option :suffix, type: :string, aliases: '-s', desc: 'Suffix to append to index name'
+      desc 'update_aliases *INDEX_CLASS', 'Replaces all existing aliases by the given suffix/suffixes'
+      option :suffix, type: :string, aliases: '-s', repeatable: true, desc: 'Suffix to append to index name'
       def update_aliases(*index_classes)
         require_relative 'index/update_aliases'
-        UpdateAliases.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+
+        kwargs = options.to_h.transform_keys(&:to_sym)
+        kwargs[:suffix] = (kwargs[:suffix] || []).flat_map { |s| s.split(',') }.uniq
+        UpdateAliases.new(indices: index_classes, **kwargs).run
       end
 
       desc 'update_settings *INDEX_CLASS', 'Closes the index for read/write operations, updates the index settings, and open it again'

data/lib/esse/cli/templates/collection.rb.erb

@@ -2,10 +2,10 @@
 
 class <%= @index_name %> < <%= @base_class %>
   module Collections
-    class <%= @type.camelize if @type %>Collection < Esse::Collection
-      <%- if @type -%>
-      # @yield [Array<<%= @type.camelize %>>]
-      # @see <%= @index_name %>::<%= @type.camelize %>#collection
+    class <%= @repo.camelize if @repo %>Collection < Esse::Collection
+      <%- if @repo -%>
+      # @yield [Array<<%= @repo.camelize %>>]
+      # @see <%= @index_name %>::<%= @repo.camelize %>#collection
       <%- end -%>
       def each
         offset = 0
@@ -13,7 +13,7 @@ class <%= @index_name %> < <%= @base_class %>
           break if rows.none?
           offset += 1
           yield(rows)
-          # You may also preload associations here or add metadata useful for the serializer
+          # You may also preload associations here or add metadata useful for the document
           # yield(rows, **preload_associations(rows))
         end
       end
@@ -22,7 +22,7 @@ class <%= @index_name %> < <%= @base_class %>
 
       # @param offset [Number] Offset to start from
       def find_all(offset)
-        # @TODO load data from persistent store
+        # @TODO load data from persistent storage
       end
     end
   end

data/lib/esse/cli/templates/{serializer.rb.erb → document.rb.erb}

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 class <%= @index_name %> < <%= @base_class %>
-  module Serializers
-    class <%= @type.camelize if @type %>Serializer < Esse::Serializer
+  module Documents
+    class <%= @repo.camelize if @repo %>Document < Esse::Document
       <%- if @cli_options[:active_record] -%>
       delegate :id, to: :object
       <%- else -%>
@@ -14,8 +14,8 @@ class <%= @index_name %> < <%= @base_class %>
       def source
         {
           name: object.name,
-          <%- if @type -%>
-          type: "<%= @type %>",
+          <%- if @repo && defined?(Elasticsearch::VERSION) && Elasticsearch::VERSION > '5' -%>
+          type: "<%= @repo %>",
           <%- end -%>
           <%- if @cli_options[:active_record] -%>
           created_at: object.created_at,
@@ -23,10 +23,10 @@ class <%= @index_name %> < <%= @base_class %>
           <%- end -%>
         }
       end
-      <%- if @type && defined?(Elasticsearch::VERSION) && Elasticsearch::VERSION <= '5' -%>
+      <%- if @repo && defined?(Elasticsearch::VERSION) && Elasticsearch::VERSION <= '5' -%>
 
       def type
-        '<%= @type %>'
+        '<%= @repo %>'
       end
       <%- end -%>
     end

data/lib/esse/cli/templates/index.rb.erb

@@ -1,43 +1,49 @@
 # frozen_string_literal: true
 
-<%- @types.each do |type| -%>
+<%- @repos.each do |type| -%>
 require_relative '<%= @index_name.demodulize.underscore.to_s %>/collections/<%= type.underscore %>_collection'
 <%- end if @cli_options[:collections] && !@cli_options[:active_record] -%>
-<%- @types.each do |type| -%>
-require_relative '<%= @index_name.demodulize.underscore.to_s %>/serializers/<%= type.underscore %>_serializer'
-<%- end if @cli_options[:serializers] -%>
+<%- @repos.each do |type| -%>
+require_relative '<%= @index_name.demodulize.underscore.to_s %>/documents/<%= type.underscore %>_document'
+<%- end if @cli_options[:documents] -%>
 class <%= @index_name %> < <%= @base_class %>
+  <%-if @index_cluster_id -%>
+  self.cluster_id = :<%= @index_cluster_id %>
+  <%- end -%>
   <%- if @cli_options[:active_record] -%>
   plugin :active_record
 
   <%- end -%>
-  <%- if @types.empty? -%>
-  <%- if @cli_options[:active_record] -%>
-  collection ::<%= @index_name.camelize %>.all
-  <%- elsif @cli_options[:collections] -%>
-  collection Collections::Collection
-  <%- else -%>
-  collection do |**context, &block|
-    query = <%= @index_name.camelize.sub(/Index$/, '') %>.all
-    query = query.where(id: context[:id]) if context[:id]
-    query.find_in_batches(batch_size: 1_000) do |batch|
-      block.call(batch)
+  <%- if @repos.empty? -%>
+  repository :default do
+    <%- if @cli_options[:active_record] -%>
+    collection ::<%= @index_name.camelize %>.all
+    <%- elsif @cli_options[:collections] -%>
+    collection Collections::Collection
+    <%- else -%>
+    collection do |**context, &block|
+      query = <%= @index_name.camelize.sub(/Index$/, '') %>.all
+      query = query.where(id: context[:id]) if context[:id]
+      query.find_in_batches(batch_size: 1_000) do |batch|
+        block.call(batch, **context)
+      end
     end
+    <%- end -%>
+
+    <%- if @cli_options[:documents] -%>
+    document Documents::Document
+    <%- else -%>
+    document do |object, **_context|
+      {
+        id: object.id,
+        name: object.name,
+      }
+    end
+    <%- end # if @cli_options[:documents] -%>
   end
-  <%- end -%>
-  <%- if @cli_options[:serializers] -%>
-  serializer Serializers::Serializer
-  <%- else -%>
-  serializer do |object, **_context|
-    {
-      id: object.id,
-      name: object.name,
-    }
-  end
-  <%- end # if @cli_options[:serializers] -%>
-  <%- end # /@types.empty?-%>
+  <%- end # /@repos.empty?-%>
+  <%- @repos.each do |type| -%>
 
-  <%- @types.each do |type| -%>
   repository :<%= type.underscore %> do
     <%- if @cli_options[:active_record] -%>
     collection ::<%= type.camelize %>.all
@@ -48,14 +54,14 @@ class <%= @index_name %> < <%= @base_class %>
       query = <%= type.camelize %>.all
       query = query.where(id: context[:id]) if context[:id]
       query.find_in_batches(batch_size: 1_000) do |batch|
-        block.call(batch)
+        block.call(batch, **context)
       end
     end
     <%- end -%>
-    <%- if @cli_options[:serializers] -%>
-    serializer Serializers::<%= type.camelize %>Serializer
+    <%- if @cli_options[:documents] -%>
+    document Documents::<%= type.camelize %>Document
     <%- else -%>
-    serializer do |<%= type.underscore %>, **_context|
+    document do |<%= type.underscore %>, **_context|
       {
         id: <%= type.underscore %>.id,
         name: <%= @index_name.underscore %>.name,
@@ -63,6 +69,5 @@ class <%= @index_name %> < <%= @base_class %>
     end
     <%- end -%>
   end
-
-  <%- end #@types.each do |type| -%>
+  <%- end #@repos.each do |type| -%>
 end

data/lib/esse/cli.rb

@@ -32,6 +32,7 @@ module Esse
       def initialize(*)
         super
 
+        after_initialize
         load_app_config(options[:require])
         setup_listeners if !options[:silent] && Esse.config.cli_event_listeners?
       end
@@ -71,6 +72,10 @@ module Esse
 
       private
 
+      def after_initialize
+        # esse plugins may override this method
+      end
+
       def setup_listeners
         Esse::Events.__bus__.events.keys.grep(/^elasticsearch/).each do |event_name|
           Esse::Events.subscribe(event_name) do |event|

data/lib/esse/cluster.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 require_relative 'cluster_engine'
-require_relative 'client_proxy'
+require_relative 'transport'
 
 module Esse
   class Cluster
-    ATTRIBUTES = %i[index_prefix settings mappings client wait_for_status].freeze
+    ATTRIBUTES = %i[index_prefix settings mappings client wait_for_status readonly].freeze
     WAIT_FOR_STATUSES = %w[green yellow red].freeze
 
     # The index prefix. For example an index named UsersIndex.
@@ -28,12 +28,17 @@ module Esse
     #   wait_for_status: green
     attr_accessor :wait_for_status
 
+    # Disable all writes from the application to the underlying Elasticsearch instance while keeping the
+    # application running and handling search requests.
+    attr_writer :readonly
+
     attr_reader :id
 
     def initialize(id:, **options)
       @id = id.to_sym
       @settings = {}
       @mappings = {}
+      @readonly = false
       assign(options)
     end
 
@@ -61,6 +66,17 @@ module Esse
       end
     end
 
+    # @return [Boolean] Return true if the cluster is readonly
+    def readonly?
+      !!@readonly
+    end
+
+    # @raise [Esse::Transport::ReadonlyClusterError] if the cluster is readonly
+    # @return [void]
+    def throw_error_when_readonly!
+      raise Esse::Transport::ReadonlyClusterError if readonly?
+    end
+
     # Define the elasticsearch client connection
     # @param es_client [Elasticsearch::Client, OpenSearch::Client, Hash] an instance of elasticsearch/api client or an hash
     #   with the settings that will be used to initialize the Client
@@ -77,24 +93,33 @@ module Esse
     end
 
     def inspect
-      attrs = ([:id] + ATTRIBUTES - [:client]).map do |method|
+      attrs = ([:id] + ATTRIBUTES - [:client, :readonly]).map do |method|
         value = public_send(method)
         format('%<k>s=%<v>p', k: method, v: value) if value
       end.compact
+      attrs << 'readonly=true' if readonly?
       attrs << format('client=%p', @client)
       format('#<Esse::Cluster %<attrs>s>', attrs: attrs.join(' '))
     end
 
-    def wait_for_status!(status: wait_for_status)
+    # Wait until cluster is in a specific state
+    #
+    # @option [String] :status Wait until cluster is in a specific state (options: green, yellow, red)
+    # @option [String] :index Limit the information returned to a specific index
+    def wait_for_status!(status: nil, **kwargs)
+      status ||= wait_for_status
       return unless WAIT_FOR_STATUSES.include?(status.to_s)
 
-      client.cluster.health(wait_for_status: status.to_s)
+      api.health(**kwargs, wait_for_status: status.to_s)
     end
 
-    # @idea Change this to use the response from `GET /`
-    def document_type?
-      defined?(OpenSearch::VERSION) || \
-        (defined?(Elasticsearch::VERSION) && Elasticsearch::VERSION < '7')
+    # @return [void]
+    def may_update_type!(hash, key: :type)
+      if (single_type = engine.mapping_default_type)
+        hash[key] = single_type
+        return
+      end
+      hash.delete(key) if engine.mapping_single_type?
     end
 
     def info
@@ -108,8 +133,9 @@ module Esse
     end
 
     def engine
-      ClusterEngine.new(**info)
+      @engine ||=ClusterEngine.new(**info)
     end
+    alias_method :warm_up!, :engine
 
     # Build a search query for the given indices
     #
@@ -121,9 +147,9 @@ module Esse
 
     # Return the proxy object used to perform low level actions on the elasticsearch cluster through the official api client
     #
-    # @return [Esse::ClientProxy] The cluster api instance
+    # @return [Esse::Transport] The cluster api instance
     def api
-      Esse::ClientProxy.new(self)
+      Esse::Transport.new(self)
     end
   end
 end

data/lib/esse/core.rb

@@ -5,7 +5,7 @@ module Esse
   require_relative 'cluster'
   require_relative 'primitives'
   require_relative 'collection'
-  require_relative 'serializer'
+  require_relative 'document'
   require_relative 'hash_document'
   require_relative 'null_document'
   require_relative 'repository'
@@ -15,8 +15,6 @@ module Esse
   require_relative 'template_loader'
   require_relative 'import/request_body'
   require_relative 'import/bulk'
-  require_relative 'backend/index'
-  require_relative 'backend/repository_backend'
   require_relative 'version'
   require_relative 'logging'
   require_relative 'events'
@@ -85,4 +83,10 @@ module Esse
     end
     true
   end
+
+  def self.document?(object)
+    return false unless object
+
+    !!(object.is_a?(Esse::Document) && object.id)
+  end
 end

data/lib/esse/deprecations/cluster.rb

@@ -2,26 +2,26 @@
 
 module Esse
   class Cluster
-    extend Gem::Deprecate
+    extend Esse::Deprecations::Deprecate
 
     def index_settings
       settings
     end
-    deprecate :index_settings, :settings, 2022, 10
+    deprecate :index_settings, :settings, 2023, 12
 
     def index_settings=(value)
       self.settings = value
     end
-    deprecate :index_settings=, :settings=, 2022, 10
+    deprecate :index_settings=, :settings=, 2023, 12
 
     def index_mappings
       mappings
     end
-    deprecate :index_mappings, :mappings, 2022, 10
+    deprecate :index_mappings, :mappings, 2023, 12
 
     def index_mappings=(value)
       self.mappings = value
     end
-    deprecate :index_mappings=, :mappings=, 2022, 10
+    deprecate :index_mappings=, :mappings=, 2023, 12
   end
 end

data/lib/esse/deprecations/deprecate.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Esse
+  module Deprecations
+    module Deprecate
+      def self.extended(base)
+        base.extend Gem::Deprecate
+        base.include InstanceMethods
+      end
+
+      module InstanceMethods
+        def warning(method, repl, year, month)
+          msg = ["NOTE: #{method} is deprecated"]
+          msg << if repl == :none
+            ' with no replacement'
+          elsif repl.respond_to?(:call)
+            "; use #{repl.call} instead"
+          else
+            "; use #{repl} instead"
+          end
+          msg << '. It will be removed on or after %4d-%02d-01.' % [year, month]
+          msg << "\n#{method} called from #{Gem.location_of_caller(2).join(':')}"
+
+          warn "#{msg.join}." unless Gem::Deprecate.skip
+        end
+      end
+    end
+  end
+end