esse 0.2.0 → 0.2.2

data/lib/esse/backend/repository_backend.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Esse
+  module Backend
+    class RepositoryBackend
+      extend Forwardable
+
+      # Type delegators
+      def_delegators :@repo, :document_type, :index, :serialize
+
+      def initialize(repo)
+        @repo = repo
+      end
+
+      def import(**kwargs)
+        elasticsearch.import(document_type, **kwargs)
+      end
+
+      def import!(**kwargs)
+        elasticsearch.import!(document_type, **kwargs)
+      end
+
+      def bulk!(**kwargs)
+        elasticsearch.bulk!(**kwargs, type: document_type)
+      end
+
+      def bulk(**kwargs)
+        elasticsearch.bulk(**kwargs, type: document_type)
+      end
+
+      def index!(**kwargs)
+        elasticsearch.index!(**kwargs, type: document_type)
+      end
+
+      def index(**kwargs)
+        elasticsearch.index(**kwargs, type: document_type)
+      end
+
+      # @param [Esse::Serializer] document A document instance
+      def index_document(document, **kwargs)
+        return unless document.is_a?(Esse::Serializer)
+        return if document.ignore_on_index?
+        return unless document.id
+
+        kwargs[:id] = document.id
+        kwargs[:routing] = document.routing if document.routing
+        kwargs[:type] = document.type || document_type
+        kwargs[:body] = document.source&.to_h
+        elasticsearch.index(**kwargs)
+      end
+
+      def update!(**kwargs)
+        elasticsearch.update!(**kwargs, type: document_type)
+      end
+
+      def update(**kwargs)
+        elasticsearch.update(**kwargs, type: document_type)
+      end
+
+      def delete!(**kwargs)
+        elasticsearch.delete!(**kwargs, type: document_type)
+      end
+
+      def delete(**kwargs)
+        elasticsearch.delete(**kwargs, type: document_type)
+      end
+
+      # @param [Esse::Serializer] document A document instance
+      def delete_document(document, **kwargs)
+        return unless document.is_a?(Esse::Serializer)
+        return if document.ignore_on_delete?
+        return unless document.id
+
+        kwargs[:id] = document.id
+        kwargs[:routing] = document.routing if document.routing
+        kwargs[:type] = document.type || document_type
+        elasticsearch.delete(**kwargs)
+      end
+
+      def exist?(**kwargs)
+        elasticsearch.exist?(**kwargs, type: document_type)
+      end
+
+      def count(**kwargs)
+        elasticsearch.count(**kwargs, type: document_type)
+      end
+
+      def find!(**kwargs)
+        elasticsearch.find!(**kwargs, type: document_type)
+      end
+
+      def find(**kwargs)
+        elasticsearch.find(**kwargs, type: document_type)
+      end
+
+      protected
+
+      def elasticsearch
+        @repo.index.elasticsearch
+      end
+    end
+  end
+end

data/lib/esse/cli/event_listener.rb

@@ -82,6 +82,20 @@ module Esse
             padding: runtime_padding(event[:runtime])
         end
       end
+
+      def elasticsearch_bulk(event)
+        print_message("[%<runtime>s] Bulk index %<name>s%<type>s%<wait_interval>s: ",
+          runtime: formatted_runtime(event[:runtime]),
+          name: colorize(event[:request][:index], :bold),
+          type: (event[:request][:type] ? " for type #{colorize(event[:request][:type], :bold)}" : ''),
+          wait_interval: (event[:wait_interval].nonzero? ? " (wait interval #{event[:wait_interval]}s)" : ''),
+          newline: false,
+        )
+        stats = event[:request][:body_stats].select {|_, v| v.nonzero? }.map do |type, count|
+          "#{colorize(type, :bold)}: #{count} docs"
+        end
+        print_message(stats.join(", ") + ".")
+      end
     end
   end
 end

data/lib/esse/cli/generate.rb

@@ -14,34 +14,75 @@ module Esse
       end
 
       desc 'index NAME *TYPES', 'Creates a new index'
+      option :settings, type: :boolean, default: false, desc: 'Generate settings'
+      option :mappings, type: :boolean, default: false, desc: 'Generate mappings'
+      option :serializers, type: :boolean, default: false, desc: 'Generate serializers'
+      option :collections, type: :boolean, default: false, desc: 'Generate collections'
+      option :active_record, type: :boolean, default: false, desc: 'Generate ActiveRecord models'
+      option :cluster_id, type: :string, desc: 'Elasticsearch cluster ID'
       def index(name, *types)
         ns_path = name.split(NAMESPACE_PATTERN_RE).tap(&:pop)
         @index_name = Hstring.new(name.to_s).modulize.sub(/Index$/, '') + 'Index'
         @index_name = Hstring.new(@index_name)
         @types = types.map { |type| Hstring.new(type) }
         @base_class = base_index_class(*ns_path)
+        if options[:cluster_id]
+          @base_class += format('(:%s)', options[:cluster_id])
+        end
+        @cli_options = options
 
-        base_dir = Esse.config.indices_directory.join(*ns_path)
+        base_dir = Esse.config.indices_directory.join(*ns_path.map { |n| Hstring.new(n).underscore.to_s })
         index_name = @index_name.demodulize.underscore.to_s
         template(
           'templates/index.rb.erb',
           base_dir.join("#{index_name}.rb"),
         )
-        @types.each do |type|
-          @type = Hstring.new(type).underscore
+
+        if options[:settings]
           copy_file(
-            'templates/type_mappings.json',
-            base_dir.join(index_name, 'templates', "#{@type}_mapping.json"),
+            'templates/settings.json',
+            base_dir.join(index_name, 'templates', 'settings.json'),
           )
-          template(
-            'templates/type_serializer.rb.erb',
-            base_dir.join(index_name, 'serializers', "#{@type}_serializer.rb"),
-          )
-          template(
-            'templates/type_collection.rb.erb',
-            base_dir.join(index_name, 'collections', "#{@type}_collection.rb"),
+        end
+
+        if options[:mappings]
+          copy_file(
+            'templates/mappings.json',
+            base_dir.join(index_name, 'templates', 'mappings.json'),
           )
         end
+
+        if @types.empty?
+          if options[:serializers]
+            template(
+              'templates/serializer.rb.erb',
+              base_dir.join(index_name, 'serializers', 'serializer.rb'),
+            )
+          end
+          if options[:collections] && !options[:active_record]
+            template(
+              'templates/collection.rb.erb',
+              base_dir.join(index_name, 'collections', 'collection.rb'),
+            )
+          end
+        end
+
+        @types.each do |type|
+          @type = Hstring.new(type).underscore
+
+          if options[:serializers]
+            template(
+              'templates/serializer.rb.erb',
+              base_dir.join(index_name, 'serializers', "#{@type}_serializer.rb"),
+            )
+          end
+          if options[:collections] && !options[:active_record]
+            template(
+              'templates/collection.rb.erb',
+              base_dir.join(index_name, 'collections', "#{@type}_collection.rb"),
+            )
+          end
+        end
       end
 
       protected

data/lib/esse/cli/index/base_operation.rb

@@ -31,7 +31,10 @@ module Esse
       end
 
       def indices
-        eager_load_indices!
+        Esse.eager_load_indices!
+        if @indices == ['all']
+          return Esse::Index.descendants.reject(&:abstract_class?)
+        end
         @indices.map do |class_name|
           const_exist = begin
             Kernel.const_defined?(class_name)
@@ -58,18 +61,7 @@ module Esse
           end
 
           klass
-        end
-      end
-
-      def eager_load_indices!
-        return false unless Esse.config.indices_directory.exist?
-
-        Esse.config.indices_directory.each_child do |path|
-          next unless path.extname == '.rb'
-
-          require(path.expand_path.to_s)
-        end
-        true
+        end.reject(&:abstract_class?)
       end
     end
   end

data/lib/esse/cli/index/import.rb

@@ -8,14 +8,18 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          index.elasticsearch.import!(**options)
+          if (repo = @options[:repo])
+            index.elasticsearch.import!(repo, **options)
+          else
+            index.elasticsearch.import!(**options)
+          end
         end
       end
 
       private
 
       def options
-        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS - [:repo])
       end
 
       def validate_options!

data/lib/esse/cli/index/update_mapping.rb

@@ -8,10 +8,9 @@ module Esse
       def run
         validate_options!
         indices.each do |index|
-          if index.type_hash.any?
-            index.type_hash.each_value do |type|
-              # @idea Add update_mapping! to IndexType and use it here
-              index.elasticsearch.update_mapping!(type: type.type_name, **options)
+          if !index.mapping_single_type?
+            index.repo_hash.values.map(&:document_type).uniq.each do |doc_type|
+              index.elasticsearch.update_mapping!(type: doc_type, **options)
             end
           else
             index.elasticsearch.update_mapping!(**options)

data/lib/esse/cli/index.rb

@@ -15,6 +15,7 @@ module Esse
         * Delete the old index.
       DESC
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      option :import, type: :boolean, default: true, desc: 'Import documents before point alias to the new index'
       def reset(*index_classes)
         require_relative 'index/reset'
         Reset.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
@@ -81,6 +82,7 @@ module Esse
       desc 'import *INDEX_CLASSES', 'Import documents from the given classes'
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :context, type: :hash, default: {}, required: true, desc: 'List of options to pass to the index class'
+      option :repo, type: :string, default: nil, alias: '-r', desc: 'Repository to use for import'
       def import(*index_classes)
         require_relative 'index/import'
         Import.new(indices: index_classes, **HashUtils.deep_transform_keys(options.to_h, &:to_sym)).run

data/lib/esse/cli/templates/{type_collection.rb.erb → collection.rb.erb}

@@ -2,36 +2,24 @@
 
 class <%= @index_name %> < <%= @base_class %>
   module Collections
-    class <%= @type.camelize %>Collection
-      include Enumerable
-
-      # @param params [Hash] List of parameters
-      def initialize(**params)
-        @params = params
-      end
-
-      # Find all <%= @type %> in batches
-      #
+    class <%= @type.camelize if @type %>Collection < Esse::Collection
+      <%- if @type -%>
       # @yield [Array<<%= @type.camelize %>>]
       # @see <%= @index_name %>::<%= @type.camelize %>#collection
+      <%- end -%>
       def each
         offset = 0
         while (rows = find_all(offset))
           break if rows.none?
-
-          # You may preload associations before serialize them
-          # associations = preload_associations!(rows)
-          # yield(row, associations)
           offset += 1
-
-          yield(rows, **params)
+          yield(rows)
+          # You may also preload associations here or add metadata useful for the serializer
+          # yield(rows, **preload_associations(rows))
         end
       end
 
       protected
 
-      attr_reader :params
-
       # @param offset [Number] Offset to start from
       def find_all(offset)
         # @TODO load data from persistent store

data/lib/esse/cli/templates/config.rb.erb

@@ -8,13 +8,23 @@ Esse.configure do |config|
     # you also can add environment to the index prefix if available:
     #   cluster.index_prefix = "esse_#{ENV['RACK_ENV']}"
     #
-    # Initialize the ElastiSearch client to be used by the default cluster
-    cluster.client = Elasticsearch::Client.new
+    # Initialize the ElastiSearch/OpenSearch client to be used by the default cluster
+    # cluster.client = Elasticsearch::Client.new or OpenSearch::Client.new
 
     # Global index settings
-    #   cluster.index_settings = {
+    #   cluster.settings = {
     #     number_of_shards: 5,
     #     number_of_replicas: 0,
     #   }
+
+    # Global index mappings
+    #   cluster.mappings = {
+    #     dynamic_templates: [
+    #       # ... Dynamic templates ...
+    #     ],
+    #     properties: {
+    #       # ... Explicit mapping
+    #     },
+    #   }
   end
 end

data/lib/esse/cli/templates/index.rb.erb

@@ -2,123 +2,67 @@
 
 <%- @types.each do |type| -%>
 require_relative '<%= @index_name.demodulize.underscore.to_s %>/collections/<%= type.underscore %>_collection'
-<%- end -%>
+<%- end if @cli_options[:collections] && !@cli_options[:active_record] -%>
 <%- @types.each do |type| -%>
 require_relative '<%= @index_name.demodulize.underscore.to_s %>/serializers/<%= type.underscore %>_serializer'
-<%- end -%>
-
+<%- end if @cli_options[:serializers] -%>
 class <%= @index_name %> < <%= @base_class %>
-  # plugin :active_record
-  # plugin :sequel
+  <%- if @cli_options[:active_record] -%>
+  plugin :active_record
 
+  <%- end -%>
   <%- if @types.empty? -%>
-  # Collection
-  # ==========
-  #
-  # Collection is the source of data for the index. We hightly recommend using a
-  # another class that implements the Enumerable interface to better split the
-  # responsabilities and easy to test.
-  #
-  #   collection Collections::DocumentCollection
-  #
-  # but you can also use block definition style:
-  #   collection do |**context, &block|
-  #     block.call [{ title: 'foo' }, { title: 'bar' }], extra: 'info'
-  #   end
-  #
-  # Serializer block or class yielder should be called with an array of objects.
-  # Each these objects should be serialized using the serializer in described in the next section.
-  # The number of objects will be indexed to elasticsearch using the bulk api. So adjust the
-  # number of objects to be indexed accordingly.
-  #
-  # Here is a good place to eager loading data from database or any other repository.
-  # The bellow example is a rails like application that could preload using activerecord
-  #
-  #   collection do |**context, &block|
-  #     query = <%= @index_name.camelize %>.all
-  #     query = query.where(**context[:conditions]) if context[:conditions]
-  #     query = query.includes(:user) if context[:include_user]
-  #     query.find_in_batches(batch_size: 5000) do |batch|
-  #       block.call batch, context
-  #     end
-  #   end
-
-
-  # Serializer
-  # ==========
-  #
-  # Serializer is the class responsible for serializing indexing documents.
-  # Each object yielded by the collection will be serialized on this step.
-  # We recommend using a another class to handle the serialization. The only requirement is
-  # that the class implements the `#to_h` method.
-  # The serializer class may also be initialized with context if collection block is called with that extra parameter.
-  # Ability to call serializer with context is useful for preloading data from database or any other repository.
-  #
-  #    serializer Serializers::DocumentSerializer
-  #
-  # You can also serialize the collection entry using a block:
-  #
-  #   serializer do |model, context|
-  #     hash = {
-  #       name: <%= @index_name.underscore %>.name,
-  #     }
-  #     # Context is just an example here. But it's useful for eager loading data.
-  #     # I'll think a better example when implement this idea.
-  #     hash[:some_attribute] = <%= @index_name.underscore %>.some_attribute if context[:include_some_attribute]
-  #     hash
-  #   end
+  <%- if @cli_options[:active_record] -%>
+  collection ::<%= @index_name.camelize %>.all
+  <%- elsif @cli_options[:collections] -%>
+  collection Collections::Collection
+  <%- else -%>
+  collection do |**context, &block|
+    query = <%= @index_name.camelize.sub(/Index$/, '') %>.all
+    query = query.where(id: context[:id]) if context[:id]
+    query.find_in_batches(batch_size: 1_000) do |batch|
+      block.call(batch)
+    end
+  end
   <%- end -%>
+  <%- if @cli_options[:serializers] -%>
+  serializer Serializers::Serializer
+  <%- else -%>
+  serializer do |object, **_context|
+    {
+      id: object.id,
+      name: object.name,
+    }
+  end
+  <%- end # if @cli_options[:serializers] -%>
+  <%- end # /@types.empty?-%>
+
   <%- @types.each do |type| -%>
-  define_type :<%= type.underscore %> do
-    # Collection
-    # ==========
-    #
-    # Collection wraps the data into an array of items that should be serialized. The first argument that is
-    # yielded must extends Enumerable.
-    # Useful for eager loading data from database or any other repository. Below is an example of a rails like
-    # application could load using activerecord.
-    #
-    #   collection do |**context, &block|
-    #     <%= type.camelize %>.where(context[:conditions]).find_in_batches(batch_size: 5000) do |batch|
-    #       block.call batch, context
-    #     end
-    #   end
+  repository :<%= type.underscore %> do
+    <%- if @cli_options[:active_record] -%>
+    collection ::<%= type.camelize %>.all
+    <%- elsif @cli_options[:collections] -%>
     collection Collections::<%= type.camelize %>Collection
-    #
-    #
-    # Serializer
-    # ==========
-    #
-    # The serializer can be any class that respond with the `to_h` class method.
-    # And the result of its to_h is a Hash.
-    #
-    # Here is an example of a simple serializer:
-    # app/serializers/<%= type %>_serializer.rb
-    #   class <%= type.camelize %>Serializer
-    #     def initialize(<%= type %>, _context)
-    #       @<%= type %> = <%= type %>
-    #     end
-    #
-    #     def to_h
-    #       { '_id' => @<%= type %>.id, 'name' => @<%= type %>.name }
-    #     end
-    #   end
-    #
-    # And here you specify your serializer classe.
-    #    serializer Serializers::<%= type.camelize %>Serializer
-    #
-    # You can also serialize the collection entry using a block:
-    #
-    #   serializer do |model, **context|
-    #     hash = {
-    #       name: <%= type %>.name,
-    #     }
-    #     # Context is just an example here. But it's useful for eager loading data.
-    #     # I'll think a better example when implement this idea.
-    #     hash[:some_attribute] = <%= type %>.some_attribute if context[:include_some_attribute]
-    #     hash
-    #   end
+    <%- else -%>
+    collection do |**context, &block|
+      query = <%= type.camelize %>.all
+      query = query.where(id: context[:id]) if context[:id]
+      query.find_in_batches(batch_size: 1_000) do |batch|
+        block.call(batch)
+      end
+    end
+    <%- end -%>
+    <%- if @cli_options[:serializers] -%>
     serializer Serializers::<%= type.camelize %>Serializer
+    <%- else -%>
+    serializer do |<%= type.underscore %>, **_context|
+      {
+        id: <%= type.underscore %>.id,
+        name: <%= @index_name.underscore %>.name,
+      }
+    end
+    <%- end -%>
   end
-  <%- end -%>
+
+  <%- end #@types.each do |type| -%>
 end

data/lib/esse/cli/templates/mappings.json

@@ -0,0 +1,27 @@
+{
+  "dynamic_templates": [
+    {
+      "string_template": {
+        "match": "*",
+        "match_mapping_type": "string",
+        "mapping": {
+          "fields": {
+            "analyzed": {
+              "analyzer": "esse_index",
+              "index": true,
+              "type": "text"
+            }
+          },
+          "ignore_above": 30000,
+          "type": "keyword"
+        }
+      }
+    }
+  ],
+  "properties": {
+    "slug": {
+      "type": "keyword",
+      "ignore_above": 255
+    }
+  }
+}

data/lib/esse/cli/templates/serializer.rb.erb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class <%= @index_name %> < <%= @base_class %>
+  module Serializers
+    class <%= @type.camelize if @type %>Serializer < Esse::Serializer
+      <%- if @cli_options[:active_record] -%>
+      delegate :id, to: :object
+      <%- else -%>
+      def id
+        object.id
+      end
+      <%- end -%>
+
+      def source
+        {
+          name: object.name,
+          <%- if @type -%>
+          type: "<%= @type %>",
+          <%- end -%>
+          <%- if @cli_options[:active_record] -%>
+          created_at: object.created_at,
+          updated_at: object.updated_at,
+          <%- end -%>
+        }
+      end
+      <%- if @type && defined?(Elasticsearch::VERSION) && Elasticsearch::VERSION <= '5' -%>
+
+      def type
+        '<%= @type %>'
+      end
+      <%- end -%>
+    end
+  end
+end

data/lib/esse/cli/templates/settings.json

@@ -0,0 +1,62 @@
+{
+  "index": {
+    "max_ngram_diff": 5,
+    "max_shingle_diff": 3,
+    "analysis": {
+      "filter": {
+        "esse_search_shingle": {
+          "token_separator": "",
+          "output_unigrams_if_no_shingles": "true",
+          "output_unigrams": "false",
+          "type": "shingle"
+        },
+        "esse_index_shingle": {
+          "token_separator": "",
+          "type": "shingle"
+        },
+        "esse_stemmer": {
+          "type": "stemmer",
+          "language": "English"
+        }
+      },
+      "analyzer": {
+        "esse_index": {
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "english_stop",
+            "esse_index_shingle",
+            "esse_stemmer"
+          ],
+          "char_filter": [
+            "ampersand"
+          ],
+          "type": "custom",
+          "tokenizer": "standard"
+        },
+        "esse_search": {
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "english_stop",
+            "esse_search_shingle",
+            "esse_stemmer"
+          ],
+          "char_filter": [
+            "ampersand"
+          ],
+          "type": "custom",
+          "tokenizer": "standard"
+        }
+      },
+      "char_filter": {
+        "ampersand": {
+          "type": "mapping",
+          "mappings": [
+            "&=> and "
+          ]
+        }
+      }
+    }
+  }
+}