elastics 0.1.1 → 0.2.0

data/lib/elastics/active_record/search_result.rb

@@ -0,0 +1,49 @@
+module Elastics
+  module ActiveRecord
+    class SearchResult
+      attr_reader :result
+
+      def initialize(model, result, options = {})
+        @model = model
+        @result = result
+        @options = options
+      end
+
+      def hits
+        @hits ||= @result['hits'.freeze]
+      end
+
+      def ids
+        @ids ||= hits['hits'.freeze].map { |x| x['_id'.freeze].to_i }
+      end
+
+      def ids_to_find
+        @ids_to_find ||= begin
+          limit = @options[:limit]
+          limit ? ids[0...limit] : ids
+        end
+      end
+
+      def rest_ids
+        limit = @options[:limit]
+        limit ? ids[limit..-1] : []
+      end
+
+      def collection
+        @collection ||= @model.find_all_ordered ids_to_find
+      end
+
+      def relation
+        @model.where id: ids_to_find
+      end
+
+      def aggregations
+        @aggregations ||= @result['aggregations'.freeze]
+      end
+
+      def total
+        hits['total'.freeze]
+      end
+    end
+  end
+end

data/lib/elastics/active_record/tasks_config.rb

@@ -0,0 +1,25 @@
+module Elastics
+  module ActiveRecord
+    module TasksConfig
+      def base_paths
+        @base_paths ||= if defined?(Rails)
+          [File.join(Rails.root, 'db', 'elastics')]
+        else
+          super
+        end
+      end
+
+      def client
+        @client ||= ::ActiveRecord::Base.elastics
+      end
+
+      def version_manager
+        @version_manager ||= ::ActiveRecord::Base.elastics_version_manager
+      end
+
+      def config
+        @config ||= ::ActiveRecord::Base.elastics_config
+      end
+    end
+  end
+end

data/lib/elastics/capistrano.rb

@@ -0,0 +1,14 @@
+Capistrano::Configuration.instance.load do
+  namespace :elastics do
+    %w(create drop migrate reindex).each do |method|
+      desc "rake elastics:#{method}"
+      task method, roles: :elastics, only: {primary: true} do
+        bundle_cmd = fetch(:bundle_cmd, 'bundle')
+        env = fetch(:rack_env, fetch(:rails_env, 'production'))
+        run "cd #{current_path} && " \
+          "#{bundle_cmd} exec rake elastics:#{method}[#{ENV['INDICES']}] #{ENV['ES_OPTIONS']} " \
+          "RAILS_ENV=#{env}"
+      end
+    end
+  end
+end

data/lib/elastics/client.rb

@@ -4,12 +4,18 @@ module Elastics
   class Client
     HEADERS = {'Content-Type' => 'application/json'}
 
+    autoload :Cluster, 'elastics/client/cluster'
+
     attr_writer :index, :type
     attr_reader :client
 
     def initialize(defaults = {})
-      @host = defaults[:host] || '127.0.0.1'
-      @port = defaults[:port] || 9200
+      if defaults[:host].is_a?(Array)
+        extend Cluster
+        initialize_cluster(defaults)
+      else
+        @host = defaults[:host] || '127.0.0.1:9200'
+      end
       @index  = defaults[:index]
       @type   = defaults[:type]
       @client = HTTPClient.new
@@ -28,45 +34,60 @@ module Elastics
       @type  = type  || nil
     end
 
-    def uri(params)
-      str = "http://#{@host}:#{@port}"
-      if index = params[:index]  || @index
-        str += "/#{index}"
-        type  = params[:type]   || @type
-        str += "/#{type}" if type
+    def request_path(params)
+      str = ""
+      if index = params[:index] || @index
+        str << "/#{index}"
+        type = params[:type] || @type
+        str << "/#{type}" if type
       end
       path = params[:id]
-      str += "/#{path}" if path
+      str << "/#{path}" if path
       str
     end
 
     def request(params)
-      http_method = params[:method] || :get
+      method = params[:method] || :get
       body = params[:data].try!(:to_json)
-      res = @client.request(http_method, uri(params), params[:query], body, HEADERS)
+      res = http_request(method, request_path(params), params[:query], body, params)
       status = res.status
       return JSON.parse(res.body) if 300 > status
-      if 404 == status
-        message = JSON.parse(res.body)['error'] rescue 'Not found'
-        raise NotFound, message
-      end
-      raise Error.new(res.reason)
+      result = JSON.parse(res.body) rescue nil
+      err_msg = "#{res.reason}: #{result && result['error'] || '-'}"
+      # NotFound is raised only for valid responses from ElasticSearch
+      raise NotFound, err_msg if 404 == status && result
+      raise Error, err_msg
     end
 
     # shortcuts
-    [:put, :post, :delete].each do |method|
+    [:put, :post].each do |method|
       define_method(method) do |params|
         params[:method] = method
         request params
       end
     end
 
-    def get(params)
+    def delete!(params)
+      params[:method] = :delete
+      request params
+    end
+
+    def delete(params)
+      delete!(params)
+    rescue NotFound
+    end
+
+    def get!(params)
       params = {id: params} unless params.is_a?(Hash)
       params[:method] = :get
       request(params)
     end
 
+    def get(params)
+      get!(params)
+    rescue NotFound
+    end
+
     def set(id, data)
       request(id: id, data: data, method: :put)
     end
@@ -88,10 +109,16 @@ module Elastics
     end
 
     def index_exists?(index)
-      get(index: index, type: nil, id: :_mapping)
-      true
-    rescue NotFound
-      false
+      !!get(index: index, type: nil, id: :_mapping)
     end
+
+    private
+      # Endpoint for low-level request. For easy host highjacking & instrumentation.
+      # Params are not used directly but kept for instrumentation purpose.
+      # You probably don't want to use this method directly.
+      def http_request(method, path, query, body, params = nil, host = @host)
+        uri = "http://#{host}#{path}"
+        @client.request(method, uri, query, body, HEADERS)
+      end
   end
 end

data/lib/elastics/client/cluster.rb

@@ -0,0 +1,70 @@
+require 'timeout'
+require 'thread'
+require 'thread_safe'
+
+module Elastics
+  class Client
+    module Cluster
+      class NoAliveHosts < Error; end
+
+      private
+        def initialize_cluster(defaults)
+          @hosts = ThreadSafe::Array.new defaults[:host]
+          @dead_hosts = ThreadSafe::Hash.new
+          @connect_timeout = defaults[:connect_timeout] || 10
+          @resurrect_timeout = defaults[:resurrect_timeout] || 10
+          @current_host_n = 0
+          @cluster_mutex = Mutex.new
+        end
+
+        def http_request(method, path, query, body, params = nil)
+          host = next_cluster_host
+          Timeout.timeout(@connect_timeout) do
+            super(method, path, query, body, params, host)
+          end
+        rescue Timeout::Error, HTTPClient::ConnectTimeoutError
+          add_dead_host(host)
+          retry
+        end
+
+      # TODO: check Enumerable#cycle for thread-safety and use it if possible
+      def next_cluster_host
+        if @resurrect_at
+          time = Time.now.to_i
+          resurrect_cluster(time) if @resurrect_at <= time
+        end
+        host_n = @current_host_n
+        loop do
+          host = @hosts[host_n]
+          if !host
+            raise NoAliveHosts if host_n == 0
+            host_n = 0
+          else
+            @current_host_n = host_n + 1
+            return host
+          end
+        end
+      end
+
+      def resurrect_cluster(time = Time.now.to_i)
+        @cluster_mutex.synchronize do
+          @dead_hosts.delete_if do |host, resurrect_at|
+            # skip the rest because values are sorted
+            if time < resurrect_at
+              @resurrect_at = resurrect_at
+              break
+            end
+            @hosts << host
+          end
+        end
+      end
+
+      def add_dead_host(host, resurrect_at = nil)
+        resurrect_at ||= Time.now.to_i + @resurrect_timeout
+        @hosts.delete(host)
+        @dead_hosts[host] = resurrect_at
+        @resurrect_at ||= resurrect_at
+      end
+    end
+  end
+end

data/lib/elastics/railtie.rb

@@ -3,14 +3,15 @@ require 'elastics/active_record'
 module Elastics
   class Railtie < Rails::Railtie
     initializer 'elastics.configure_rails_initialization' do
-      ::ActiveRecord::Base.extend Elastics::ActiveRecord
-      unless ::ActiveRecord::LogSubscriber < ActiveRecord::LogSubscriber
-        ::ActiveRecord::LogSubscriber.send :include, ActiveRecord::LogSubscriber
-      end
+      ActiveRecord.install
     end
 
     rake_tasks do
       load 'tasks/elastics.rake'
     end
+
+    config.to_prepare do
+      Elastics.reset_models
+    end
   end
 end

data/lib/elastics/tasks.rb

@@ -1,35 +1,36 @@
+require 'active_support'
+require 'active_support/core_ext'
+
 module Elastics
   module Tasks
+    require 'elastics/tasks/config'
+    include Config
+
     require 'elastics/tasks/indices'
     include Indices
 
     require 'elastics/tasks/mappings'
     include Mappings
 
-    extend self
-
-    attr_writer :base_paths, :client, :config
-
-    def base_paths
-      @base_paths ||= [File.join(Rails.root, 'db', 'elastics')]
-    end
+    require 'elastics/tasks/migrations'
+    include Migrations
 
-    def migrate(options = {})
-      delete_indices if options[:flush]
-      create_indices
-      put_mappings
+    if defined?(::ActiveRecord) && defined?(::Elastics::ActiveRecord)
+      require 'elastics/active_record/tasks_config'
+      include ActiveRecord::TasksConfig
     end
 
-    def client
-      @client ||= ::ActiveRecord::Base.elastics
-    end
-
-    def config
-      @config ||= ::ActiveRecord::Base.elastics_config
-    end
+    extend self
 
     def log(*args)
-      Rails.logger.info(*args)
+      puts(*args)
     end
+
+    private
+      def each_filtered(collection, filter, &block)
+        filter = filter && filter.map(&:to_s)
+        collection = collection.select { |x| filter.include?(x) } if filter
+        collection.each &block
+      end
   end
 end

data/lib/elastics/tasks/config.rb

@@ -0,0 +1,38 @@
+module Elastics
+  module Tasks
+    # Module contains basic configuration methods.
+    # You should setup Elastics::Task yourself unless you you use ActiveRecord.
+    module Config
+      attr_writer :base_paths
+
+      def base_paths
+        @base_paths ||= Dir.pwd
+      end
+
+      def client
+        @client ||= Client.new config.slice(:host, :port)
+      end
+
+      def client=(val)
+        @version_manager = nil
+        @client = val
+      end
+
+      def version_manager
+        @version_manager ||= VersionManager.new(client, config.slice(
+          :service_index,
+          :index_prefix,
+        ))
+      end
+
+      def config
+        @config ||= {}
+      end
+
+      def config=(val)
+        @version_manager = nil
+        @config = val
+      end
+    end
+  end
+end

data/lib/elastics/tasks/indices.rb

@@ -1,5 +1,9 @@
 module Elastics
   module Tasks
+    # Most of methods accepts `options` hash with:
+    # - `:indices` - array of indices to perform action on
+    # - `:version` - mapping version to use in method
+    #
     module Indices
       attr_writer :indices_path
 
@@ -13,7 +17,6 @@ module Elastics
           each_with_object({}) do |file, hash|
             name = File.basename file, '.yml'
             data = YAML.load_file(file)
-            name = "#{config[:index_prefix]}#{name}"
             hash[name] = data[Rails.env] || data
           end
       end
@@ -22,20 +25,79 @@ module Elastics
         @indices ||= config[:index] ? [config[:index]] : indices_settings.keys
       end
 
-      def delete_indices
-        indices.each do |index|
-          log "Delete index #{index}"
-          client.delete index: index rescue NotFound
+      def versioned_index_name(*args)
+        version_manager.index_name *args
+      end
+
+      def purge(keep_data = false)
+        unless keep_data
+          drop_indices
+          drop_indices version: :next
+        end
+        index = version_manager.service_index
+        log "Deleting index #{index}"
+        version_manager.reset
+        client.delete index: index
+      end
+
+      def drop_indices(options = {})
+        version = options.fetch :version, :current
+        each_filtered(indices, options[:indices]) do |index|
+          versioned_index = versioned_index_name(index, version)
+          log "Deleting index #{index} (#{versioned_index})"
+          client.delete index: versioned_index
         end
       end
 
-      def create_indices
-        indices.each do |index|
-          log "Create index #{index}"
-          unless client.index_exists?(index)
-            client.put(index: index, data: indices_settings[index])
+      def create_indices(options = {})
+        version = options.fetch :version, :current
+        each_filtered(indices, options[:indices]) do |index|
+          versioned_index = versioned_index_name(index, version)
+          exists = client.index_exists?(versioned_index)
+          log_msg = "Creating index #{index} (#{versioned_index})"
+          log_msg << ' - Skipping: exists' if exists
+          log log_msg
+          unless exists
+            client.put(index: versioned_index, data: indices_settings[index])
           end
         end
+        manage_aliases :add, options if version.to_s == 'current'
+      end
+
+      # Action can be :add or :remove.
+      def manage_aliases(action, options = {})
+        version = options.fetch :version, :current
+        post_aliases(options) do |index|
+          alias_action(action, index, version)
+        end
+      end
+
+      def forward_aliases(options = {})
+        new_versions = {}
+        post_aliases options do |index|
+          new_versions[index] = version_manager.next_version index
+          [
+            alias_action(:remove, index, :current),
+            alias_action(:add, index, :next),
+          ]
+        end
+        drop_indices(options.merge version: :current) if options.fetch(:drop, true)
+        new_versions.each do |index, version|
+          version_manager.set index, current: version
+        end
+      end
+
+      def alias_action(action, index, version)
+        {action => {
+          index: versioned_index_name(index, version),
+          alias: versioned_index_name(index, :alias),
+        }}
+      end
+
+      def post_aliases(options = {}, &block)
+        actions = each_filtered(indices, options[:indices]).map(&block).flatten
+        log "Posting aliases: #{actions.inspect}"
+        client.post id: :_aliases, data: {actions: actions} if actions.any?
       end
     end
   end