esse 0.0.5 → 0.1.1

data/lib/esse/cli/index/create.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative 'base_operation'
+
+module Esse
+  module CLI
+    class Index::Create < Index::BaseOperation
+      def run
+        validate_options!
+        indices.each do |index|
+          index.elasticsearch.create_index!(**options)
+        end
+      end
+
+      private
+
+      def options
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+      end
+
+      def validate_options!
+        validate_indices_option!
+      end
+    end
+  end
+end

data/lib/esse/cli/index/delete.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative 'base_operation'
+
+module Esse
+  module CLI
+    class Index::Delete < Index::BaseOperation
+      def run
+        validate_options!
+        indices.each do |index|
+          index.elasticsearch.delete_index!(**options)
+        end
+      end
+
+      private
+
+      def options
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+      end
+
+      def validate_options!
+        validate_indices_option!
+      end
+    end
+  end
+end

data/lib/esse/cli/index/open.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative 'base_operation'
+
+module Esse
+  module CLI
+    class Index::Open < Index::BaseOperation
+      def run
+        validate_options!
+        indices.each do |index|
+          index.elasticsearch.open!(**options)
+        end
+      end
+
+      private
+
+      def options
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+      end
+
+      def validate_options!
+        validate_indices_option!
+      end
+    end
+  end
+end

data/lib/esse/cli/index/reset.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative 'base_operation'
+
+module Esse
+  module CLI
+    class Index::Reset < Index::BaseOperation
+      def run
+        validate_options!
+        indices.each do |index|
+          index.elasticsearch.reset_index!(**options)
+        end
+      end
+
+      private
+
+      def options
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+      end
+
+      def validate_options!
+        validate_indices_option!
+      end
+    end
+  end
+end

data/lib/esse/cli/index/update_aliases.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'base_operation'
+
+module Esse
+  module CLI
+    class Index::UpdateAliases < Index::BaseOperation
+      def run
+        validate_options!
+        indices.each do |index|
+          index.elasticsearch.update_aliases!(**options)
+        end
+      end
+
+      private
+
+      def options
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+      end
+
+      def validate_options!
+        validate_indices_option!
+
+        if @options[:suffix].nil?
+          raise InvalidOption.new(<<~END)
+            You must specify a suffix to update the aliases.
+          END
+        end
+      end
+    end
+  end
+end

data/lib/esse/cli/index/update_mapping.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative 'base_operation'
+
+module Esse
+  module CLI
+    class Index::UpdateMapping < Index::BaseOperation
+      def run
+        validate_options!
+        indices.each do |index|
+          if index.type_hash.any?
+            index.type_hash.each_value do |type|
+              # @idea Add update_mapping! to IndexType and use it here
+              index.elasticsearch.update_mapping!(type: type.type_name, **options)
+            end
+          else
+            index.elasticsearch.update_mapping!(**options)
+          end
+        end
+      end
+
+      private
+
+      def options
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+      end
+
+      def validate_options!
+        validate_indices_option!
+      end
+    end
+  end
+end

data/lib/esse/cli/index/update_settings.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative 'base_operation'
+
+module Esse
+  module CLI
+    class Index::UpdateSettings < Index::BaseOperation
+      def run
+        validate_options!
+        indices.each do |index|
+          index.elasticsearch.update_settings!(**options)
+        end
+      end
+
+      private
+
+      def options
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+      end
+
+      def validate_options!
+        validate_indices_option!
+      end
+    end
+  end
+end

data/lib/esse/cli/index.rb

@@ -2,12 +2,80 @@
 
 require 'thor'
 require_relative 'base'
+
 module Esse
   module CLI
     class Index < Base
-      desc 'create *INDEX_CLASSES', 'Creates a new index'
+      desc 'reset *INDEX_CLASSES', 'Performs zero-downtime index resetting.'
+      long_desc <<-DESC
+        This task is used to rebuild index with zero-downtime. The task will:
+        * Creates a new index using the suffix defined on index class or from CLI.
+        * Import documents using the Index collection.
+        * Update alias to point to the new index.
+        * Delete the old index.
+      DESC
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      def reset(*index_classes)
+        require_relative 'index/reset'
+        Reset.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'create *INDEX_CLASSES', 'Creates indices for the given classes'
+      long_desc <<-DESC
+        Creates index and applies mapping and settings for the given classes.
+
+        Indices are created with the following naming convention:
+        <cluster.index_prefix>_<index_class.index_name>_<index_class.index_version>.
+      DESC
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      option :alias, type: :boolean, default: false, aliases: '-a', desc: 'Update alias after create index'
       def create(*index_classes)
-        # Add action here
+        require_relative 'index/create'
+        Create.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'delete *INDEX_CLASSES', 'Deletes indices for the given classes'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      def delete(*index_classes)
+        require_relative 'index/delete'
+        Delete.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'update_aliases *INDEX_CLASS', 'Replaces all existing aliases by the given suffix'
+      option :suffix, type: :string, aliases: '-s', desc: 'Suffix to append to index name'
+      def update_aliases(*index_classes)
+        require_relative 'index/update_aliases'
+        UpdateAliases.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'update_settings *INDEX_CLASS', 'Closes the index for read/write operations, updates the index settings, and open it again'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      option :type, type: :string, default: nil, aliases: '-t', desc: 'Document Type to update mapping for'
+      def update_settings(*index_classes)
+        require_relative 'index/update_settings'
+        UpdateSettings.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'update_mapping *INDEX_CLASS', 'Create or update a mapping'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      option :type, type: :string, default: nil, aliases: '-t', desc: 'Document Type to update mapping for'
+      def update_mapping(*index_classes)
+        require_relative 'index/update_mapping'
+        UpdateMapping.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'close *INDEX_CLASS', 'Close an index (keep the data on disk, but deny operations with the index).'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      def close(*index_classes)
+        require_relative 'index/close'
+        Close.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'open *INDEX_CLASS', 'Open a previously closed index.'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      def open(*index_classes)
+        require_relative 'index/open'
+        Open.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
       end
     end
   end

data/lib/esse/cli/templates/config.rb.erb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+Esse.configure do |config|
+  config.indices_directory = File.expand_path('../../app/indices', __FILE__)
+  config.cluster(:default) do |cluster|
+    # Default index prefix
+    cluster.index_prefix = "esse"
+    # you also can add environment to the index prefix if available:
+    #   cluster.index_prefix = "esse_#{ENV['RACK_ENV']}"
+    #
+    # Initialize the ElastiSearch client to be used by the default cluster
+    cluster.client = Elasticsearch::Client.new
+
+    # Global index settings
+    #   cluster.index_settings = {
+    #     number_of_shards: 5,
+    #     number_of_replicas: 0,
+    #   }
+  end
+end

data/lib/esse/cli/templates/index.rb.erb

@@ -1,11 +1,75 @@
 # frozen_string_literal: true
 
+<%- @types.each do |type| -%>
+require_relative '<%= @index_name.demodulize.underscore.to_s %>/collections/<%= type.underscore %>_collection'
+<%- end -%>
+<%- @types.each do |type| -%>
+require_relative '<%= @index_name.demodulize.underscore.to_s %>/serializers/<%= type.underscore %>_serializer'
+<%- end -%>
+
 class <%= @index_name %> < <%= @base_class %>
   # plugin :active_record
   # plugin :sequel
 
+  <%- if @types.empty? -%>
+  # Collection
+  # ==========
+  #
+  # Collection is the source of data for the index. We hightly recommend using a
+  # another class that implements the Enumerable interface to better split the
+  # responsabilities and easy to test.
+  #
+  #   collection Collections::DocumentCollection
+  #
+  # but you can also use block definition style:
+  #   collection do |_context, &block|
+  #     block.call [{ title: 'foo' }, { title: 'bar' }], context
+  #   end
+  #
+  # Serializer block or class yielder should be called with an array of objects.
+  # Each these objects should be serialized using the serializer in described in the next section.
+  # The number of objects will be indexed to elasticsearch using the bulk api. So adjust the
+  # number of objects to be indexed accordingly.
+  #
+  # Here is a good place to eager loading data from database or any other repository.
+  # The bellow example is a rails like application that could preload using activerecord
+  #
+  #   collection do |**context, &block|
+  #     query = <%= @index_name.camelize %>.all
+  #     query = query.where(**context[:conditions]) if context[:conditions]
+  #     query = query.includes(:user) if context[:include_user]
+  #     query.find_in_batches(batch_size: 5000) do |batch|
+  #       block.call batch, context
+  #     end
+  #   end
+
+
+  # Serializer
+  # ==========
+  #
+  # Serializer is the class responsible for serializing indexing documents.
+  # Each object yielded by the collection will be serialized on this step.
+  # We recommend using a another class to handle the serialization. The only requirement is
+  # that the class implements the `#to_h` method.
+  # The serializer class may also be initialized with context if collection block is called with that extra parameter.
+  # Ability to call serializer with context is useful for preloading data from database or any other repository.
+  #
+  #    serializer Serializers::DocumentSerializer
+  #
+  # You can also serialize the collection entry using a block:
+  #
+  #   serializer do |model, context|
+  #     hash = {
+  #       name: <%= @index_name.underscore %>.name,
+  #     }
+  #     # Context is just an example here. But it's useful for eager loading data.
+  #     # I'll think a better example when implement this idea.
+  #     hash[:some_attribute] = <%= @index_name.underscore %>.some_attribute if context[:include_some_attribute]
+  #     hash
+  #   end
+  <%- end -%>
   <%- @types.each do |type| -%>
-  define_type :<%= type %> do
+  define_type :<%= type.underscore %> do
     # Collection
     # ==========
     #
@@ -14,19 +78,19 @@ class <%= @index_name %> < <%= @base_class %>
     # Useful for eager loading data from database or any other repository. Below is an example of a rails like
     # application could load using activerecord.
     #
-    #   collection do |conditions, &block|
-    #     context = {}
-    #     <%= type.camelize %>.where(conditions).find_in_batches(batch_size: 5000) do |batch|
-    #       block.call batch, context, ...
+    #   collection do |context, &block|
+    #     <%= type.camelize %>.where(context[:conditions]).find_in_batches(batch_size: 5000) do |batch|
+    #       block.call batch, context
     #     end
     #   end
+    collection Collections::<%= type.camelize %>Collection
     #
     #
     # Serializer
     # ==========
     #
-    # The serializer can be any class that respond with the `as_json` class method.
-    # And the result of its as_json is a Hash.
+    # The serializer can be any class that respond with the `to_h` class method.
+    # And the result of its to_h is a Hash.
     #
     # Here is an example of a simple serializer:
     # app/serializers/<%= type %>_serializer.rb
@@ -35,7 +99,7 @@ class <%= @index_name %> < <%= @base_class %>
     #       @<%= type %> = <%= type %>
     #     end
     #
-    #     def as_json
+    #     def to_h
     #       { '_id' => @<%= type %>.id, 'name' => @<%= type %>.name }
     #     end
     #   end
@@ -45,7 +109,7 @@ class <%= @index_name %> < <%= @base_class %>
     #
     # You can also serialize the collection entry using a block:
     #
-    #   serializer do |model, context = {}|
+    #   serializer do |model, **context|
     #     hash = {
     #       name: <%= type %>.name,
     #     }
@@ -54,6 +118,7 @@ class <%= @index_name %> < <%= @base_class %>
     #     hash[:some_attribute] = <%= type %>.some_attribute if context[:include_some_attribute]
     #     hash
     #   end
+    serializer Serializers::<%= type.camelize %>Serializer
   end
   <%- end -%>
 end

data/lib/esse/cli/templates/type_collection.rb.erb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class <%= @index_name %> < <%= @base_class %>
+  module Collections
+    class <%= @type.camelize %>Collection
+      include Enumerable
+
+      # @param params [Hash] List of parameters
+      def initialize(**params)
+        @params = params
+      end
+
+      # Find all <%= @type %> in batches
+      #
+      # @yield [Array<<%= @type.camelize %>>]
+      # @see <%= @index_name %>::<%= @type.camelize %>#collection
+      def each
+        offset = 0
+        while (rows = find_all(offset))
+          break if rows.none?
+
+          # You may preload associations before serialize them
+          # associations = preload_associations!(rows)
+          # yield(row, associations)
+          offset += 1
+
+          yield(rows, **params)
+        end
+
+        protected
+
+        attr_reader :params
+
+        # @param offset [Number] Offset to start from
+        def find_all(offset)
+          # @TODO load data from persistent store
+        end
+      end
+    end
+  end
+end

data/lib/esse/cli/templates/{serializer.rb.erb → type_serializer.rb.erb}

@@ -3,16 +3,21 @@
 class <%= @index_name %> < <%= @base_class %>
   module Serializers
     class <%= @type.camelize %>Serializer
-      def initialize(<%= @type %>, *_other)
+      def initialize(<%= @type %>, **params)
         @entity = <%= @type %>
+        @params = params
       end
 
-      def as_json
+      def to_h
         {
-          id: @entity.id, # This field is required
-          name: @entity.name,
+          id: entity.id, # This field is required
+          name: entity.name,
         }
       end
+
+      protected
+
+      attr_reader :entity, :params
     end
   end
 end

data/lib/esse/cli.rb

@@ -2,16 +2,48 @@
 
 require 'thor'
 
+require_relative 'primitives/output'
 require_relative 'cli/index'
 require_relative 'cli/generate'
+require_relative 'cli/event_listener'
 
 module Esse
   module CLI
-    def self.start(*args)
-      Root.start(*args)
+    class << self
+      def start(*args)
+        Root.start(*args)
+      end
+
+      def with_friendly_errors
+        yield
+      rescue CLI::Error => e
+        Output.print_error(e)
+        exit(1)
+      end
     end
 
     class Root < Thor
+      include Thor::Actions
+
+      class_option :require, type: :string, aliases: '-r', required: false,
+        default: nil, desc: 'Require config file where the application is defined'
+      class_option :silent, type: :boolean, aliases: '-s', required: false, default: false, desc: 'Silent mode'
+
+      def initialize(*)
+        super
+
+        load_app_config(options[:require])
+        setup_listeners if !options[:silent] && Esse.config.cli_event_listeners?
+      end
+
+      def self.source_root
+        File.expand_path('../cli', __FILE__)
+      end
+
+      def self.exit_on_failure?
+        true
+      end
+
       map %w[--version -v] => :version
 
       desc 'index SUBCOMMAND ...ARGS', 'Manage indices'
@@ -22,7 +54,47 @@ module Esse
 
       desc '--version, -v', 'Show package version'
       def version
-        puts format('Esse version: %<version>s', version: Esse::VERSION)
+        Output.print_message('Esse version: %<version>s', version: Esse::VERSION)
+      end
+
+      desc 'install', 'Generate boilerplate configuration files'
+      option :path, type: :string, aliases: '-p', required: false, default: './'
+      def install
+        path = Pathname.new(File.expand_path(options[:path], Dir.pwd))
+        path = path.dirname unless path.directory?
+        @app_dir = path.basename
+        template(
+          'templates/config.rb.erb',
+          path.join('config/esse.rb'),
+        )
+      end
+
+      private
+
+      def setup_listeners
+        Esse::Events.__bus__.events.keys.grep(/^elasticsearch/).each do |event_name|
+          Esse::Events.subscribe(event_name) do |event|
+            EventListener[event_name]&.call(event)
+          end
+        end
+      end
+
+      def load_app_config(path)
+        if path.nil?
+          CLI_CONFIG_PATHS.each do |config_path|
+            next unless File.exist?(config_path)
+            path = config_path
+            break
+          end
+        end
+        return unless path
+
+        begin
+          Output.print_message('Loading configuration file: %<path>s', path: path)
+          load path
+        rescue LoadError => e
+          raise InvalidOption, e.message
+        end
       end
     end
   end

data/lib/esse/cluster.rb

@@ -2,7 +2,8 @@
 
 module Esse
   class Cluster
-    ATTRIBUTES = %i[index_prefix index_settings client].freeze
+    ATTRIBUTES = %i[index_prefix index_settings client wait_for_status].freeze
+    WAIT_FOR_STATUSES = %w[green yellow red].freeze
 
     # The index prefix. For example an index named UsersIndex.
     # With `index_prefix = 'app1'`. Final index/alias is: 'app1_users'
@@ -11,6 +12,15 @@ module Esse
     # This settings will be passed through all indices during the mapping
     attr_accessor :index_settings
 
+    # if this option set, actions such as creating or deleting index,
+    # importing data will wait for the status specified. Extremely useful
+    # for tests under heavy indices manipulations.
+    # Value can be set to `red`, `yellow` or `green`.
+    #
+    # Example:
+    #   wait_for_status: green
+    attr_accessor :wait_for_status
+
     attr_reader :id
 
     def initialize(id:, **options)
@@ -39,7 +49,7 @@ module Esse
     #   with the settings that will be used to initialize Elasticsearch::Client
     def client=(es_client)
       @client = if es_client.is_a?(Hash)
-        settings = es_client.each_with_object({}) { |(k,v), r| r[k.to_sym] = v }
+        settings = es_client.each_with_object({}) { |(k, v), r| r[k.to_sym] = v }
         Elasticsearch::Client.new(settings)
       else
         es_client
@@ -54,5 +64,11 @@ module Esse
       attrs << format('client=%p', @client)
       format('#<Esse::Cluster %<attrs>s>', attrs: attrs.join(' '))
     end
+
+    def wait_for_status!(status: wait_for_status)
+      return unless WAIT_FOR_STATUSES.include?(status.to_s)
+
+      client.cluster.health(wait_for_status: status.to_s)
+    end
   end
 end