esse 0.2.0 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a31d2a1c181dccd9c2acd7256d958010e7c636016b1d5a267597f7aec9ed3334
-  data.tar.gz: cdb390a10b0492833a9360d4fd39df0c18df114a44b37554cba1dd39d229f75b
+  metadata.gz: 30ce86d71e0cf9b67506dd0caf1651b91ae836c9e7981e46a238446c8afde078
+  data.tar.gz: 9ec1069cbb8106743d5832f94fbce5d2c937037a62a746350e736385f3b78cdc
 SHA512:
-  metadata.gz: 59ae5dcafd8998c69e5705910b119f5f9d1333899a4320b8f7afa4c5782aa435d820dde222726de111cee00c6d3b4bbed9d6fdd18dac6a6f740cf64138b70d53
-  data.tar.gz: 84b876eaec5b090718e2d265dd5d9456c5dfaeedafdae6e9361ac790de91eab0ba46b9ceabee3c5e8a891c5027ff3ac8443217554219f8dce5f52c853912989c
+  metadata.gz: f2a5012e3d3dab0b3894f8e8fb1665be2a57c8538a3fc02bce2ac089510bd63b982f8f919554a9f16d6e8dafe5d93d9f5614d99ca4f170d82cc41e413fec0209
+  data.tar.gz: 034b15fd5cfc4f6ebdd97ba9d73ed284f50c209225b2c6a43b27216a57578ca1b91d9c92d4e020c4a1c7910038cf5978054bfa4b491bafb431bcfc1dcbc0a706

data/lib/esse/cli/event_listener.rb

@@ -82,6 +82,19 @@ module Esse
             padding: runtime_padding(event[:runtime])
         end
       end
+
+      def elasticsearch_bulk(event)
+        print_message('[%<runtime>s] Bulk index %<name>s%<type>s%<wait_interval>s: ',
+          runtime: formatted_runtime(event[:runtime]),
+          name: colorize(event[:request][:index], :bold),
+          type: (event[:request][:type] ? " for type #{colorize(event[:request][:type], :bold)}" : ''),
+          wait_interval: (event[:wait_interval].nonzero? ? " (wait interval #{event[:wait_interval]}s)" : ''),
+          newline: false,)
+        stats = event[:request][:body_stats].select { |_, v| v.nonzero? }.map do |type, count|
+          "#{colorize(type, :bold)}: #{count} docs"
+        end
+        print_message(stats.join(', ') + '.')
+      end
     end
   end
 end

data/lib/esse/cli/generate.rb

@@ -14,34 +14,73 @@ module Esse
       end
 
       desc 'index NAME *TYPES', 'Creates a new index'
-      def index(name, *types)
+      option :settings, type: :boolean, default: false, desc: 'Generate settings'
+      option :mappings, type: :boolean, default: false, desc: 'Generate mappings'
+      option :documents, type: :boolean, default: false, desc: 'Generate documents'
+      option :collections, type: :boolean, default: false, desc: 'Generate collections'
+      option :active_record, type: :boolean, default: false, desc: 'Generate ActiveRecord models'
+      option :cluster_id, type: :string, desc: 'Elasticsearch cluster ID'
+      def index(name, *repos)
         ns_path = name.split(NAMESPACE_PATTERN_RE).tap(&:pop)
         @index_name = Hstring.new(name.to_s).modulize.sub(/Index$/, '') + 'Index'
         @index_name = Hstring.new(@index_name)
-        @types = types.map { |type| Hstring.new(type) }
+        @repos = repos.map { |repo| Hstring.new(repo) }
         @base_class = base_index_class(*ns_path)
+        @index_cluster_id = options[:cluster_id]
+        @cli_options = options
 
-        base_dir = Esse.config.indices_directory.join(*ns_path)
+        base_dir = Esse.config.indices_directory.join(*ns_path.map { |n| Hstring.new(n).underscore.to_s })
         index_name = @index_name.demodulize.underscore.to_s
         template(
           'templates/index.rb.erb',
           base_dir.join("#{index_name}.rb"),
         )
-        @types.each do |type|
-          @type = Hstring.new(type).underscore
+
+        if options[:settings]
           copy_file(
-            'templates/type_mappings.json',
-            base_dir.join(index_name, 'templates', "#{@type}_mapping.json"),
-          )
-          template(
-            'templates/type_serializer.rb.erb',
-            base_dir.join(index_name, 'serializers', "#{@type}_serializer.rb"),
+            'templates/settings.json',
+            base_dir.join(index_name, 'templates', 'settings.json'),
           )
-          template(
-            'templates/type_collection.rb.erb',
-            base_dir.join(index_name, 'collections', "#{@type}_collection.rb"),
+        end
+
+        if options[:mappings]
+          copy_file(
+            'templates/mappings.json',
+            base_dir.join(index_name, 'templates', 'mappings.json'),
           )
         end
+
+        if @repos.empty?
+          if options[:documents]
+            template(
+              'templates/document.rb.erb',
+              base_dir.join(index_name, 'documents', 'document.rb'),
+            )
+          end
+          if options[:collections] && !options[:active_record]
+            template(
+              'templates/collection.rb.erb',
+              base_dir.join(index_name, 'collections', 'collection.rb'),
+            )
+          end
+        end
+
+        @repos.each do |type|
+          @repo = Hstring.new(type).underscore
+
+          if options[:documents]
+            template(
+              'templates/document.rb.erb',
+              base_dir.join(index_name, 'documents', "#{@repo}_document.rb"),
+            )
+          end
+          if options[:collections] && !options[:active_record]
+            template(
+              'templates/collection.rb.erb',
+              base_dir.join(index_name, 'collections', "#{@repo}_collection.rb"),
+            )
+          end
+        end
       end
 
       protected

data/lib/esse/cli/index/base_operation.rb

@@ -31,7 +31,10 @@ module Esse
       end
 
       def indices
-        eager_load_indices!
+        Esse.eager_load_indices!
+        if @indices == ['all']
+          return Esse::Index.descendants.reject(&:abstract_class?)
+        end
         @indices.map do |class_name|
           const_exist = begin
             Kernel.const_defined?(class_name)
@@ -58,18 +61,7 @@ module Esse
           end
 
           klass
-        end
-      end
-
-      def eager_load_indices!
-        return false unless Esse.config.indices_directory.exist?
-
-        Esse.config.indices_directory.each_child do |path|
-          next unless path.extname == '.rb'
-
-          require(path.expand_path.to_s)
-        end
-        true
+        end.reject(&:abstract_class?)
       end
     end
   end

data/lib/esse/cli/index/close.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.close!(**options)
+          index.close(**options)
         end
       end
 

data/lib/esse/cli/index/create.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.create_index!(**options)
+          index.create_index(**options)
         end
       end
 

data/lib/esse/cli/index/delete.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.delete_index!(**options)
+          index.delete_index(**options)
         end
       end
 

data/lib/esse/cli/index/import.rb

@@ -8,14 +8,18 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.import!(**options)
+          if (repo = @options[:repo])
+            index.import(repo, **options)
+          else
+            index.import(**options)
+          end
         end
       end
 
       private
 
       def options
-        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS - [:repo])
       end
 
       def validate_options!

data/lib/esse/cli/index/open.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.open!(**options)
+          index.open(**options)
         end
       end
 

data/lib/esse/cli/index/reset.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.reset_index!(**options)
+          index.reset_index(**options)
         end
       end
 

data/lib/esse/cli/index/update_aliases.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.update_aliases!(**options)
+          index.update_aliases(**options)
         end
       end
 
@@ -21,7 +21,7 @@ module Esse
       def validate_options!
         validate_indices_option!
 
-        if @options[:suffix].nil?
+        if @options[:suffix].nil? || @options[:suffix].empty?
           raise InvalidOption.new(<<~END)
             You must specify a suffix to update the aliases.
           END

data/lib/esse/cli/index/update_mapping.rb

@@ -8,13 +8,17 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          if index.type_hash.any?
-            index.type_hash.each_value do |type|
-              # @idea Add update_mapping! to IndexType and use it here
-              index.elasticsearch.update_mapping!(type: type.type_name, **options)
+          if !index.mapping_single_type?
+            # Elasticsearch 6.x and older have multiple types per index.
+            # This gem supports multiple types per index for backward compatibility, but we recommend to update
+            # your elasticsearch to a at least 7.x version and use a single type per index.
+            #
+            # Note that the repository name will be used as the document type.
+            index.repo_hash.keys.each do |doc_type|
+              index.update_mapping(type: doc_type, **options)
             end
           else
-            index.elasticsearch.update_mapping!(**options)
+            index.update_mapping(**options)
           end
         end
       end

data/lib/esse/cli/index/update_settings.rb

@@ -8,7 +8,7 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.update_settings!(**options)
+          index.update_settings(**options)
         end
       end
 

data/lib/esse/cli/index.rb

@@ -15,17 +15,20 @@ module Esse
         * Delete the old index.
       DESC
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      option :import, type: :boolean, default: true, desc: 'Import documents before point alias to the new index'
       def reset(*index_classes)
         require_relative 'index/reset'
         Reset.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
       end
 
+      # @TODO Add reindex task to create a new index and import documents from the old index using _reindex API
+
       desc 'create *INDEX_CLASSES', 'Creates indices for the given classes'
       long_desc <<-DESC
         Creates index and applies mapping and settings for the given classes.
 
         Indices are created with the following naming convention:
-        <cluster.index_prefix>_<index_class.index_name>_<index_class.index_version>.
+        <cluster.index_prefix>_<index_class.index_name>_<index_class.index_suffix>.
       DESC
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :alias, type: :boolean, default: false, aliases: '-a', desc: 'Update alias after create index'
@@ -41,11 +44,14 @@ module Esse
         Delete.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
       end
 
-      desc 'update_aliases *INDEX_CLASS', 'Replaces all existing aliases by the given suffix'
-      option :suffix, type: :string, aliases: '-s', desc: 'Suffix to append to index name'
+      desc 'update_aliases *INDEX_CLASS', 'Replaces all existing aliases by the given suffix/suffixes'
+      option :suffix, type: :string, aliases: '-s', repeatable: true, desc: 'Suffix to append to index name'
       def update_aliases(*index_classes)
         require_relative 'index/update_aliases'
-        UpdateAliases.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+
+        kwargs = options.to_h.transform_keys(&:to_sym)
+        kwargs[:suffix] = (kwargs[:suffix] || []).flat_map { |s| s.split(',') }.uniq
+        UpdateAliases.new(indices: index_classes, **kwargs).run
       end
 
       desc 'update_settings *INDEX_CLASS', 'Closes the index for read/write operations, updates the index settings, and open it again'
@@ -81,6 +87,7 @@ module Esse
       desc 'import *INDEX_CLASSES', 'Import documents from the given classes'
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :context, type: :hash, default: {}, required: true, desc: 'List of options to pass to the index class'
+      option :repo, type: :string, default: nil, alias: '-r', desc: 'Repository to use for import'
       def import(*index_classes)
         require_relative 'index/import'
         Import.new(indices: index_classes, **HashUtils.deep_transform_keys(options.to_h, &:to_sym)).run

data/lib/esse/cli/templates/collection.rb.erb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class <%= @index_name %> < <%= @base_class %>
+  module Collections
+    class <%= @repo.camelize if @repo %>Collection < Esse::Collection
+      <%- if @repo -%>
+      # @yield [Array<<%= @repo.camelize %>>]
+      # @see <%= @index_name %>::<%= @repo.camelize %>#collection
+      <%- end -%>
+      def each
+        offset = 0
+        while (rows = find_all(offset))
+          break if rows.none?
+          offset += 1
+          yield(rows)
+          # You may also preload associations here or add metadata useful for the document
+          # yield(rows, **preload_associations(rows))
+        end
+      end
+
+      protected
+
+      # @param offset [Number] Offset to start from
+      def find_all(offset)
+        # @TODO load data from persistent storage
+      end
+    end
+  end
+end

data/lib/esse/cli/templates/config.rb.erb

@@ -8,13 +8,23 @@ Esse.configure do |config|
     # you also can add environment to the index prefix if available:
     #   cluster.index_prefix = "esse_#{ENV['RACK_ENV']}"
     #
-    # Initialize the ElastiSearch client to be used by the default cluster
-    cluster.client = Elasticsearch::Client.new
+    # Initialize the ElastiSearch/OpenSearch client to be used by the default cluster
+    # cluster.client = Elasticsearch::Client.new or OpenSearch::Client.new
 
     # Global index settings
-    #   cluster.index_settings = {
+    #   cluster.settings = {
     #     number_of_shards: 5,
     #     number_of_replicas: 0,
     #   }
+
+    # Global index mappings
+    #   cluster.mappings = {
+    #     dynamic_templates: [
+    #       # ... Dynamic templates ...
+    #     ],
+    #     properties: {
+    #       # ... Explicit mapping
+    #     },
+    #   }
   end
 end

data/lib/esse/cli/templates/document.rb.erb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class <%= @index_name %> < <%= @base_class %>
+  module Documents
+    class <%= @repo.camelize if @repo %>Document < Esse::Document
+      <%- if @cli_options[:active_record] -%>
+      delegate :id, to: :object
+      <%- else -%>
+      def id
+        object.id
+      end
+      <%- end -%>
+
+      def source
+        {
+          name: object.name,
+          <%- if @repo && defined?(Elasticsearch::VERSION) && Elasticsearch::VERSION > '5' -%>
+          type: "<%= @repo %>",
+          <%- end -%>
+          <%- if @cli_options[:active_record] -%>
+          created_at: object.created_at,
+          updated_at: object.updated_at,
+          <%- end -%>
+        }
+      end
+      <%- if @repo && defined?(Elasticsearch::VERSION) && Elasticsearch::VERSION <= '5' -%>
+
+      def type
+        '<%= @repo %>'
+      end
+      <%- end -%>
+    end
+  end
+end

data/lib/esse/cli/templates/index.rb.erb

@@ -1,124 +1,73 @@
 # frozen_string_literal: true
 
-<%- @types.each do |type| -%>
+<%- @repos.each do |type| -%>
 require_relative '<%= @index_name.demodulize.underscore.to_s %>/collections/<%= type.underscore %>_collection'
-<%- end -%>
-<%- @types.each do |type| -%>
-require_relative '<%= @index_name.demodulize.underscore.to_s %>/serializers/<%= type.underscore %>_serializer'
-<%- end -%>
-
+<%- end if @cli_options[:collections] && !@cli_options[:active_record] -%>
+<%- @repos.each do |type| -%>
+require_relative '<%= @index_name.demodulize.underscore.to_s %>/documents/<%= type.underscore %>_document'
+<%- end if @cli_options[:documents] -%>
 class <%= @index_name %> < <%= @base_class %>
-  # plugin :active_record
-  # plugin :sequel
+  <%-if @index_cluster_id -%>
+  self.cluster_id = :<%= @index_cluster_id %>
+  <%- end -%>
+  <%- if @cli_options[:active_record] -%>
+  plugin :active_record
 
-  <%- if @types.empty? -%>
-  # Collection
-  # ==========
-  #
-  # Collection is the source of data for the index. We hightly recommend using a
-  # another class that implements the Enumerable interface to better split the
-  # responsabilities and easy to test.
-  #
-  #   collection Collections::DocumentCollection
-  #
-  # but you can also use block definition style:
-  #   collection do |**context, &block|
-  #     block.call [{ title: 'foo' }, { title: 'bar' }], extra: 'info'
-  #   end
-  #
-  # Serializer block or class yielder should be called with an array of objects.
-  # Each these objects should be serialized using the serializer in described in the next section.
-  # The number of objects will be indexed to elasticsearch using the bulk api. So adjust the
-  # number of objects to be indexed accordingly.
-  #
-  # Here is a good place to eager loading data from database or any other repository.
-  # The bellow example is a rails like application that could preload using activerecord
-  #
-  #   collection do |**context, &block|
-  #     query = <%= @index_name.camelize %>.all
-  #     query = query.where(**context[:conditions]) if context[:conditions]
-  #     query = query.includes(:user) if context[:include_user]
-  #     query.find_in_batches(batch_size: 5000) do |batch|
-  #       block.call batch, context
-  #     end
-  #   end
+  <%- end -%>
+  <%- if @repos.empty? -%>
+  repository :default do
+    <%- if @cli_options[:active_record] -%>
+    collection ::<%= @index_name.camelize %>.all
+    <%- elsif @cli_options[:collections] -%>
+    collection Collections::Collection
+    <%- else -%>
+    collection do |**context, &block|
+      query = <%= @index_name.camelize.sub(/Index$/, '') %>.all
+      query = query.where(id: context[:id]) if context[:id]
+      query.find_in_batches(batch_size: 1_000) do |batch|
+        block.call(batch, **context)
+      end
+    end
+    <%- end -%>
 
+    <%- if @cli_options[:documents] -%>
+    document Documents::Document
+    <%- else -%>
+    document do |object, **_context|
+      {
+        id: object.id,
+        name: object.name,
+      }
+    end
+    <%- end # if @cli_options[:documents] -%>
+  end
+  <%- end # /@repos.empty?-%>
+  <%- @repos.each do |type| -%>
 
-  # Serializer
-  # ==========
-  #
-  # Serializer is the class responsible for serializing indexing documents.
-  # Each object yielded by the collection will be serialized on this step.
-  # We recommend using a another class to handle the serialization. The only requirement is
-  # that the class implements the `#to_h` method.
-  # The serializer class may also be initialized with context if collection block is called with that extra parameter.
-  # Ability to call serializer with context is useful for preloading data from database or any other repository.
-  #
-  #    serializer Serializers::DocumentSerializer
-  #
-  # You can also serialize the collection entry using a block:
-  #
-  #   serializer do |model, context|
-  #     hash = {
-  #       name: <%= @index_name.underscore %>.name,
-  #     }
-  #     # Context is just an example here. But it's useful for eager loading data.
-  #     # I'll think a better example when implement this idea.
-  #     hash[:some_attribute] = <%= @index_name.underscore %>.some_attribute if context[:include_some_attribute]
-  #     hash
-  #   end
-  <%- end -%>
-  <%- @types.each do |type| -%>
-  define_type :<%= type.underscore %> do
-    # Collection
-    # ==========
-    #
-    # Collection wraps the data into an array of items that should be serialized. The first argument that is
-    # yielded must extends Enumerable.
-    # Useful for eager loading data from database or any other repository. Below is an example of a rails like
-    # application could load using activerecord.
-    #
-    #   collection do |**context, &block|
-    #     <%= type.camelize %>.where(context[:conditions]).find_in_batches(batch_size: 5000) do |batch|
-    #       block.call batch, context
-    #     end
-    #   end
+  repository :<%= type.underscore %> do
+    <%- if @cli_options[:active_record] -%>
+    collection ::<%= type.camelize %>.all
+    <%- elsif @cli_options[:collections] -%>
     collection Collections::<%= type.camelize %>Collection
-    #
-    #
-    # Serializer
-    # ==========
-    #
-    # The serializer can be any class that respond with the `to_h` class method.
-    # And the result of its to_h is a Hash.
-    #
-    # Here is an example of a simple serializer:
-    # app/serializers/<%= type %>_serializer.rb
-    #   class <%= type.camelize %>Serializer
-    #     def initialize(<%= type %>, _context)
-    #       @<%= type %> = <%= type %>
-    #     end
-    #
-    #     def to_h
-    #       { '_id' => @<%= type %>.id, 'name' => @<%= type %>.name }
-    #     end
-    #   end
-    #
-    # And here you specify your serializer classe.
-    #    serializer Serializers::<%= type.camelize %>Serializer
-    #
-    # You can also serialize the collection entry using a block:
-    #
-    #   serializer do |model, **context|
-    #     hash = {
-    #       name: <%= type %>.name,
-    #     }
-    #     # Context is just an example here. But it's useful for eager loading data.
-    #     # I'll think a better example when implement this idea.
-    #     hash[:some_attribute] = <%= type %>.some_attribute if context[:include_some_attribute]
-    #     hash
-    #   end
-    serializer Serializers::<%= type.camelize %>Serializer
+    <%- else -%>
+    collection do |**context, &block|
+      query = <%= type.camelize %>.all
+      query = query.where(id: context[:id]) if context[:id]
+      query.find_in_batches(batch_size: 1_000) do |batch|
+        block.call(batch, **context)
+      end
+    end
+    <%- end -%>
+    <%- if @cli_options[:documents] -%>
+    document Documents::<%= type.camelize %>Document
+    <%- else -%>
+    document do |<%= type.underscore %>, **_context|
+      {
+        id: <%= type.underscore %>.id,
+        name: <%= @index_name.underscore %>.name,
+      }
+    end
+    <%- end -%>
   end
-  <%- end -%>
+  <%- end #@repos.each do |type| -%>
 end

data/lib/esse/cli/templates/mappings.json

@@ -0,0 +1,27 @@
+{
+  "dynamic_templates": [
+    {
+      "string_template": {
+        "match": "*",
+        "match_mapping_type": "string",
+        "mapping": {
+          "fields": {
+            "analyzed": {
+              "analyzer": "esse_index",
+              "index": true,
+              "type": "text"
+            }
+          },
+          "ignore_above": 30000,
+          "type": "keyword"
+        }
+      }
+    }
+  ],
+  "properties": {
+    "slug": {
+      "type": "keyword",
+      "ignore_above": 255
+    }
+  }
+}