esse 0.0.3 → 0.1.2

data/lib/esse/cli/index/update_settings.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative 'base_operation'
+
+module Esse
+  module CLI
+    class Index::UpdateSettings < Index::BaseOperation
+      def run
+        validate_options!
+        indices.each do |index|
+          index.elasticsearch.update_settings!(**options)
+        end
+      end
+
+      private
+
+      def options
+        @options.slice(*@options.keys - CLI_IGNORE_OPTS)
+      end
+
+      def validate_options!
+        validate_indices_option!
+      end
+    end
+  end
+end

data/lib/esse/cli/index.rb

@@ -2,12 +2,88 @@
 
 require 'thor'
 require_relative 'base'
+
 module Esse
   module CLI
     class Index < Base
-      desc 'create *INDEX_CLASSES', 'Creates a new index'
+      desc 'reset *INDEX_CLASSES', 'Performs zero-downtime index resetting.'
+      long_desc <<-DESC
+        This task is used to rebuild index with zero-downtime. The task will:
+        * Creates a new index using the suffix defined on index class or from CLI.
+        * Import documents using the Index collection.
+        * Update alias to point to the new index.
+        * Delete the old index.
+      DESC
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      def reset(*index_classes)
+        require_relative 'index/reset'
+        Reset.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'create *INDEX_CLASSES', 'Creates indices for the given classes'
+      long_desc <<-DESC
+        Creates index and applies mapping and settings for the given classes.
+
+        Indices are created with the following naming convention:
+        <cluster.index_prefix>_<index_class.index_name>_<index_class.index_version>.
+      DESC
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      option :alias, type: :boolean, default: false, aliases: '-a', desc: 'Update alias after create index'
       def create(*index_classes)
-        # Add action here
+        require_relative 'index/create'
+        Create.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'delete *INDEX_CLASSES', 'Deletes indices for the given classes'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      def delete(*index_classes)
+        require_relative 'index/delete'
+        Delete.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'update_aliases *INDEX_CLASS', 'Replaces all existing aliases by the given suffix'
+      option :suffix, type: :string, aliases: '-s', desc: 'Suffix to append to index name'
+      def update_aliases(*index_classes)
+        require_relative 'index/update_aliases'
+        UpdateAliases.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'update_settings *INDEX_CLASS', 'Closes the index for read/write operations, updates the index settings, and open it again'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      option :type, type: :string, default: nil, aliases: '-t', desc: 'Document Type to update mapping for'
+      def update_settings(*index_classes)
+        require_relative 'index/update_settings'
+        UpdateSettings.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'update_mapping *INDEX_CLASS', 'Create or update a mapping'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      option :type, type: :string, default: nil, aliases: '-t', desc: 'Document Type to update mapping for'
+      def update_mapping(*index_classes)
+        require_relative 'index/update_mapping'
+        UpdateMapping.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'close *INDEX_CLASS', 'Close an index (keep the data on disk, but deny operations with the index).'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      def close(*index_classes)
+        require_relative 'index/close'
+        Close.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'open *INDEX_CLASS', 'Open a previously closed index.'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      def open(*index_classes)
+        require_relative 'index/open'
+        Open.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+      end
+
+      desc 'import *INDEX_CLASSES', 'Import documents from the given classes'
+      option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
+      option :context, type: :hash, default: {}, required: true, desc: 'List of options to pass to the index class'
+      def import(*index_classes)
+        require_relative 'index/import'
+        Import.new(indices: index_classes, **HashUtils.deep_transform_keys(options.to_h, &:to_sym)).run
       end
     end
   end

data/lib/esse/cli/templates/config.rb.erb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+Esse.configure do |config|
+  config.indices_directory = File.expand_path('../../app/indices', __FILE__)
+  config.cluster(:default) do |cluster|
+    # Default index prefix
+    cluster.index_prefix = "esse"
+    # you also can add environment to the index prefix if available:
+    #   cluster.index_prefix = "esse_#{ENV['RACK_ENV']}"
+    #
+    # Initialize the ElastiSearch client to be used by the default cluster
+    cluster.client = Elasticsearch::Client.new
+
+    # Global index settings
+    #   cluster.index_settings = {
+    #     number_of_shards: 5,
+    #     number_of_replicas: 0,
+    #   }
+  end
+end

data/lib/esse/cli/templates/index.rb.erb

@@ -1,11 +1,75 @@
 # frozen_string_literal: true
 
+<%- @types.each do |type| -%>
+require_relative '<%= @index_name.demodulize.underscore.to_s %>/collections/<%= type.underscore %>_collection'
+<%- end -%>
+<%- @types.each do |type| -%>
+require_relative '<%= @index_name.demodulize.underscore.to_s %>/serializers/<%= type.underscore %>_serializer'
+<%- end -%>
+
 class <%= @index_name %> < <%= @base_class %>
   # plugin :active_record
   # plugin :sequel
-  <%- @types.each do |type| -%>
 
-  define_type :<%= type %> do
+  <%- if @types.empty? -%>
+  # Collection
+  # ==========
+  #
+  # Collection is the source of data for the index. We hightly recommend using a
+  # another class that implements the Enumerable interface to better split the
+  # responsabilities and easy to test.
+  #
+  #   collection Collections::DocumentCollection
+  #
+  # but you can also use block definition style:
+  #   collection do |_context, &block|
+  #     block.call [{ title: 'foo' }, { title: 'bar' }], context
+  #   end
+  #
+  # Serializer block or class yielder should be called with an array of objects.
+  # Each these objects should be serialized using the serializer in described in the next section.
+  # The number of objects will be indexed to elasticsearch using the bulk api. So adjust the
+  # number of objects to be indexed accordingly.
+  #
+  # Here is a good place to eager loading data from database or any other repository.
+  # The bellow example is a rails like application that could preload using activerecord
+  #
+  #   collection do |**context, &block|
+  #     query = <%= @index_name.camelize %>.all
+  #     query = query.where(**context[:conditions]) if context[:conditions]
+  #     query = query.includes(:user) if context[:include_user]
+  #     query.find_in_batches(batch_size: 5000) do |batch|
+  #       block.call batch, context
+  #     end
+  #   end
+
+
+  # Serializer
+  # ==========
+  #
+  # Serializer is the class responsible for serializing indexing documents.
+  # Each object yielded by the collection will be serialized on this step.
+  # We recommend using a another class to handle the serialization. The only requirement is
+  # that the class implements the `#to_h` method.
+  # The serializer class may also be initialized with context if collection block is called with that extra parameter.
+  # Ability to call serializer with context is useful for preloading data from database or any other repository.
+  #
+  #    serializer Serializers::DocumentSerializer
+  #
+  # You can also serialize the collection entry using a block:
+  #
+  #   serializer do |model, context|
+  #     hash = {
+  #       name: <%= @index_name.underscore %>.name,
+  #     }
+  #     # Context is just an example here. But it's useful for eager loading data.
+  #     # I'll think a better example when implement this idea.
+  #     hash[:some_attribute] = <%= @index_name.underscore %>.some_attribute if context[:include_some_attribute]
+  #     hash
+  #   end
+  <%- end -%>
+  <%- @types.each do |type| -%>
+  define_type :<%= type.underscore %> do
     # Collection
     # ==========
     #
@@ -14,19 +78,19 @@ class <%= @index_name %> < <%= @base_class %>
     # Useful for eager loading data from database or any other repository. Below is an example of a rails like
     # application could load using activerecord.
     #
-    #   collection do |conditions|
-    #     context = {}
-    #     <%= type.camelize %>.where(conditions).find_in_batches(batch_size: 5000) do |batch|
-    #       yield batch, context, ...
+    #   collection do |context, &block|
+    #     <%= type.camelize %>.where(context[:conditions]).find_in_batches(batch_size: 5000) do |batch|
+    #       block.call batch, context
     #     end
     #   end
+    collection Collections::<%= type.camelize %>Collection
     #
     #
     # Serializer
     # ==========
     #
-    # The serializer can be any class that respond with the `as_json` class method.
-    # And the result of its as_json is a Hash.
+    # The serializer can be any class that respond with the `to_h` class method.
+    # And the result of its to_h is a Hash.
     #
     # Here is an example of a simple serializer:
     # app/serializers/<%= type %>_serializer.rb
@@ -35,17 +99,17 @@ class <%= @index_name %> < <%= @base_class %>
     #       @<%= type %> = <%= type %>
     #     end
     #
-    #     def as_json
+    #     def to_h
     #       { '_id' => @<%= type %>.id, 'name' => @<%= type %>.name }
     #     end
     #   end
     #
     # And here you specify your serializer classe.
-    #    serializer <%= @index_name %>::Serializers::<%= type.camelize %>Serializer
+    #    serializer Serializers::<%= type.camelize %>Serializer
     #
     # You can also serialize the collection entry using a block:
     #
-    #   serializer(<%= type %>, context = {}) do
+    #   serializer do |model, **context|
     #     hash = {
     #       name: <%= type %>.name,
     #     }
@@ -54,6 +118,7 @@ class <%= @index_name %> < <%= @base_class %>
     #     hash[:some_attribute] = <%= type %>.some_attribute if context[:include_some_attribute]
     #     hash
     #   end
+    serializer Serializers::<%= type.camelize %>Serializer
   end
   <%- end -%>
 end

data/lib/esse/cli/templates/type_collection.rb.erb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class <%= @index_name %> < <%= @base_class %>
+  module Collections
+    class <%= @type.camelize %>Collection
+      include Enumerable
+
+      # @param params [Hash] List of parameters
+      def initialize(**params)
+        @params = params
+      end
+
+      # Find all <%= @type %> in batches
+      #
+      # @yield [Array<<%= @type.camelize %>>]
+      # @see <%= @index_name %>::<%= @type.camelize %>#collection
+      def each
+        offset = 0
+        while (rows = find_all(offset))
+          break if rows.none?
+
+          # You may preload associations before serialize them
+          # associations = preload_associations!(rows)
+          # yield(row, associations)
+          offset += 1
+
+          yield(rows, **params)
+        end
+      end
+
+      protected
+
+      attr_reader :params
+
+      # @param offset [Number] Offset to start from
+      def find_all(offset)
+        # @TODO load data from persistent store
+      end
+    end
+  end
+end

data/lib/esse/cli/templates/type_serializer.rb.erb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class <%= @index_name %> < <%= @base_class %>
+  module Serializers
+    class <%= @type.camelize %>Serializer
+      def initialize(<%= @type %>, **params)
+        @entity = <%= @type %>
+        @params = params
+      end
+
+      def to_h
+        {
+          id: entity.id, # This field is required
+          name: entity.name,
+        }
+      end
+
+      protected
+
+      attr_reader :entity, :params
+    end
+  end
+end

data/lib/esse/cli.rb

@@ -2,16 +2,48 @@
 
 require 'thor'
 
+require_relative 'primitives/output'
 require_relative 'cli/index'
 require_relative 'cli/generate'
+require_relative 'cli/event_listener'
 
 module Esse
   module CLI
-    def self.start(*args)
-      Root.start(*args)
+    class << self
+      def start(*args)
+        Root.start(*args)
+      end
+
+      def with_friendly_errors
+        yield
+      rescue CLI::Error => e
+        Output.print_error(e)
+        exit(1)
+      end
     end
 
     class Root < Thor
+      include Thor::Actions
+
+      class_option :require, type: :string, aliases: '-r', required: false,
+        default: nil, desc: 'Require config file where the application is defined'
+      class_option :silent, type: :boolean, aliases: '-s', required: false, default: false, desc: 'Silent mode'
+
+      def initialize(*)
+        super
+
+        load_app_config(options[:require])
+        setup_listeners if !options[:silent] && Esse.config.cli_event_listeners?
+      end
+
+      def self.source_root
+        File.expand_path('../cli', __FILE__)
+      end
+
+      def self.exit_on_failure?
+        true
+      end
+
       map %w[--version -v] => :version
 
       desc 'index SUBCOMMAND ...ARGS', 'Manage indices'
@@ -22,7 +54,47 @@ module Esse
 
       desc '--version, -v', 'Show package version'
       def version
-        puts format('Esse version: %<version>s', version: Esse::VERSION)
+        Output.print_message('Esse version: %<version>s', version: Esse::VERSION)
+      end
+
+      desc 'install', 'Generate boilerplate configuration files'
+      option :path, type: :string, aliases: '-p', required: false, default: './'
+      def install
+        path = Pathname.new(File.expand_path(options[:path], Dir.pwd))
+        path = path.dirname unless path.directory?
+        @app_dir = path.basename
+        template(
+          'templates/config.rb.erb',
+          path.join('config/esse.rb'),
+        )
+      end
+
+      private
+
+      def setup_listeners
+        Esse::Events.__bus__.events.keys.grep(/^elasticsearch/).each do |event_name|
+          Esse::Events.subscribe(event_name) do |event|
+            EventListener[event_name]&.call(event)
+          end
+        end
+      end
+
+      def load_app_config(path)
+        if path.nil?
+          CLI_CONFIG_PATHS.each do |config_path|
+            next unless File.exist?(config_path)
+            path = config_path
+            break
+          end
+        end
+        return unless path
+
+        begin
+          Output.print_message('Loading configuration file: %<path>s', path: path)
+          load path
+        rescue LoadError => e
+          raise InvalidOption, e.message
+        end
       end
     end
   end

data/lib/esse/cluster.rb

@@ -2,7 +2,8 @@
 
 module Esse
   class Cluster
-    ATTRIBUTES = %i[index_prefix index_settings client].freeze
+    ATTRIBUTES = %i[index_prefix index_settings client wait_for_status].freeze
+    WAIT_FOR_STATUSES = %w[green yellow red].freeze
 
     # The index prefix. For example an index named UsersIndex.
     # With `index_prefix = 'app1'`. Final index/alias is: 'app1_users'
@@ -11,6 +12,15 @@ module Esse
     # This settings will be passed through all indices during the mapping
     attr_accessor :index_settings
 
+    # if this option set, actions such as creating or deleting index,
+    # importing data will wait for the status specified. Extremely useful
+    # for tests under heavy indices manipulations.
+    # Value can be set to `red`, `yellow` or `green`.
+    #
+    # Example:
+    #   wait_for_status: green
+    attr_accessor :wait_for_status
+
     attr_reader :id
 
     def initialize(id:, **options)
@@ -35,14 +45,14 @@ module Esse
     end
 
     # Define the elasticsearch client connectio
-    # @param client [Elasticsearch::Client, Hash] an instance of elasticsearch/api client or an hash
+    # @param es_client [Elasticsearch::Client, Hash] an instance of elasticsearch/api client or an hash
     #   with the settings that will be used to initialize Elasticsearch::Client
-    def client=(val)
-      @client = if val.is_a?(Hash)
-        settings = val.each_with_object({}) { |(k,v), r| r[k.to_sym] = v }
+    def client=(es_client)
+      @client = if es_client.is_a?(Hash)
+        settings = es_client.each_with_object({}) { |(k, v), r| r[k.to_sym] = v }
         Elasticsearch::Client.new(settings)
       else
-        val
+        es_client
       end
     end
 
@@ -54,5 +64,11 @@ module Esse
       attrs << format('client=%p', @client)
       format('#<Esse::Cluster %<attrs>s>', attrs: attrs.join(' '))
     end
+
+    def wait_for_status!(status: wait_for_status)
+      return unless WAIT_FOR_STATUSES.include?(status.to_s)
+
+      client.cluster.health(wait_for_status: status.to_s)
+    end
   end
 end

data/lib/esse/config.rb

@@ -3,6 +3,31 @@
 require 'pathname'
 
 module Esse
+  # Block configurations
+  #   Esse.configure do |conf|
+  #     conf.indices_directory = 'app/indices/directory'
+  #     conf.cluster(:v1) do |cluster|
+  #       cluster.index_prefix = 'backend'
+  #       cluster.client = Elasticsearch::Client.new
+  #       cluster.index_settings = {
+  #         number_of_shards: 2,
+  #         number_of_replicas: 0
+  #       }
+  #     end
+  #   end
+  #
+  # Inline configurations
+  #   Esse.config.indices_directory = 'app/indices/directory'
+  #   Esse.config.cluster(:v1).client = Elasticsearch::Client.new
+  class << self
+    def config
+      @config ||= Config.new
+      yield(@config) if block_given?
+      @config
+    end
+    alias_method :configure, :config
+  end
+
   # Provides all configurations
   #
   # Example
@@ -19,14 +44,14 @@ module Esse
     def initialize
       self.indices_directory = 'app/indices'
       @clusters = {}
-      clusters(DEFAULT_CLUSTER_ID) # initialize the :default client
+      cluster(DEFAULT_CLUSTER_ID) # initialize the :default client
     end
 
     def cluster_ids
       @clusters.keys
     end
 
-    def clusters(key = DEFAULT_CLUSTER_ID, **options)
+    def cluster(key = DEFAULT_CLUSTER_ID, **options)
       return unless key
 
       id = key.to_sym
@@ -35,6 +60,7 @@ module Esse
         yield c if block_given?
       end
     end
+    alias_method :clusters, :cluster
 
     def indices_directory=(value)
       @indices_directory = value.is_a?(Pathname) ? value : Pathname.new(value)
@@ -45,14 +71,22 @@ module Esse
       when Hash
         assign(arg)
       when File, Pathname
-        # @TODO Load JSON or YAML
+        assign(YAML.load_file(arg))
       when String
-        # @TODO Load JSON or YAML if File.exist?(arg)
+        return load(Pathname.new(arg)) if File.exist?(arg)
+
+        assign(YAML.safe_load(arg))
       else
         raise ArgumentError, printf('could not load configuration using: %p', val)
       end
     end
 
+    # :nodoc:
+    # This is only used by rspec to disable the CLI print out.
+    def cli_event_listeners?
+      true
+    end
+
     private
 
     def assign(hash)
@@ -64,7 +98,7 @@ module Esse
       end
       if (connections = hash['clusters'] || hash[:clusters]).is_a?(Hash)
         connections.each do |key, value|
-          clusters(key).assign(value) if value.is_a?(Hash)
+          cluster(key).assign(value) if value.is_a?(Hash)
         end
       end
       true

data/lib/esse/core.rb

@@ -1,35 +1,28 @@
 # frozen_string_literal: true
 
+
 require 'multi_json'
 require 'elasticsearch'
 
 module Esse
+  require_relative 'config'
+  require_relative 'cluster'
+  require_relative 'primitives'
+  require_relative 'index_type'
+  require_relative 'index_setting'
+  require_relative 'index_mapping'
+  require_relative 'template_loader'
+  require_relative 'backend/index'
+  require_relative 'backend/index_type'
+  require_relative 'version'
+  require_relative 'logging'
+  require_relative 'events'
+  include Logging
+
   @single_threaded = false
   # Mutex used to protect mutable data structures
   @data_mutex = Mutex.new
 
-  # Block configurations
-  #   Esse.config do |conf|
-  #     conf.indices_directory = 'app/indices/directory'
-  #     conf.clusters(:v1) do |cluster|
-  #       cluster.index_prefix = 'backend'
-  #       cluster.client = Elasticsearch::Client.new
-  #       cluster.index_settings = {
-  #         number_of_shards: 2,
-  #         number_of_replicas: 0
-  #       }
-  #     end
-  #   end
-  #
-  # Inline configurations
-  #   Esse.config.indices_directory = 'app/indices/directory'
-  #   Esse.config.clusters(:v1).client = Elasticsearch::Client.new
-  def self.config
-    @config ||= Config.new
-    yield(@config) if block_given?
-    @config
-  end
-
   # Unless in single threaded mode, protects access to any mutable
   # global data structure in Esse.
   # Uses a non-reentrant mutex, so calling code should be careful.
@@ -48,9 +41,9 @@ module Esse
 
   # Simple helper used to fetch Hash value using Symbol and String keys.
   #
-  # @param [Hash] the JSON document
-  # @option [Array] :delete Removes the hash key and return its value
-  # @option [Array] :keep Fetch the hash key and return its value
+  # @param hash [Hash] the JSON document
+  # @param delete [Array] Removes the hash key and return its value
+  # @param keep [Array] Fetch the hash key and return its value
   # @return [Array([Integer, String, nil], Hash)] return the key value and the modified hash
   def self.doc_id!(hash, delete: %w[_id], keep: %w[id])
     return unless hash.is_a?(Hash)
@@ -75,15 +68,4 @@ module Esse
     end
     [id, modified]
   end
-
-  require_relative 'config'
-  require_relative 'cluster'
-  require_relative 'primitives'
-  require_relative 'index_type'
-  require_relative 'index_setting'
-  require_relative 'index_mapping'
-  require_relative 'template_loader'
-  require_relative 'backend/index'
-  require_relative 'backend/index_type'
-  require_relative 'version'
 end