elastics 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f7b4e6384b29ab3105fb8619706a05afdce54a0e
-  data.tar.gz: d5eb3b6133e0682e5470943e5ec4d9ffcc077c7c
+  metadata.gz: 39da83ca0918ecd5e5bced988a20a0f43c4a0651
+  data.tar.gz: 1b71e46b7873678e169586cf4f16204c3f01d01c
 SHA512:
-  metadata.gz: a2bf37ecf37ed6734e2f3ba4e95a70db13c8c5fd9fd50722a9e81842a9ed241d9d6dbce51a628763770358eb80dc132ecd39424ff2a017e3bf38def5976d5ec0
-  data.tar.gz: b3c316df5e6ce8113a0e41b5879049775adcb127593b8a970487e479198577c281e8611bbe57a13820c1bec795d366e3ea14af6b41db7ddec7d30f5f48355e54
+  metadata.gz: fc66215c7c93d9209828494f50aa2ed7a39ec4477924728955641ee6d8ed4cfa0fe09ed7388e7d6c599923c3eef28b729c1c3fd92f0c370a9f02b88bf16d2240
+  data.tar.gz: b03818b919cb671ec17c259f30527d18d994a5dc3c05377b1fbacd87bcbb73b68f4de5bdff16af9f43500f465f950d795be53a82562b7ec0eec86f03b7defe6c

data/.rspec

@@ -0,0 +1 @@
+--color

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.4
+notifications:
+  email: false

data/README.md

@@ -1,13 +1,21 @@
 # elastics
+[![Gem Version](https://badge.fury.io/rb/elastics.svg)](http://badge.fury.io/rb/elastics)
+[![Code Climate](https://codeclimate.com/github/printercu/elastics-rb/badges/gpa.svg)](https://codeclimate.com/github/printercu/elastics-rb)
+[![Build Status](https://travis-ci.org/printercu/elastics-rb.svg)](https://travis-ci.org/printercu/elastics-rb)
 
 Simple ElasticSearch client.
+- basic API only
+- transparent aliases management & zero-downtime migrations
+- capistrano integration
 
-Fast and thread-safe [httpclient](https://github.com/nahi/httpclient) under the hood.
+Fast and thread-safe [httpclient](https://github.com/nahi/httpclient) is under the hood.
 
 ## Install
 
 ```ruby
 # Gemfile
+gem 'elastics', '~> 0.2' # use version from the badge above
+# or
 gem 'elastics', github: 'printercu/elastics-rb'
 ```
 
@@ -19,10 +27,11 @@ gem 'elastics', github: 'printercu/elastics-rb'
 # initialize client with
 client = Elastics::Client.new(options)
 # options is hash with
-#   :host
-#   :port
+#   :host   - hostname with port or array with hosts (default 127.0.0.1:9200)
 #   :index  - (default index)
 #   :type   - (default type)
+#   :connect_timeout    - timeout to mark the host as dead in cluster-mode (default 10)
+#   :resurrect_timeout  - timeout to mark dead host as alive in cluster-mode (default 10)
 
 # basic request
 client.request(options)
@@ -49,6 +58,8 @@ client.index(params) # PUT if :id is set, otherwise POST
 client.index_exists?(name)
 ```
 
+When using cluster-mode you should also install `gem 'thread_safe'`.
+
 ### ActiveRecord
 
 ```ruby
@@ -63,6 +74,7 @@ class User < ActiveRecord::Base
 end
 
 User.search_elastics(data)
+# Returns Elastics::ActiveRecord::SearchResult object with some useful methods
 ```
 
 #### Configure
@@ -78,26 +90,85 @@ development:
 
 production:
   elastics:
-    host: 10.0.0.1
-    port: 1234
+    host: 10.0.0.1:1234
+    # or
+    host:
+      - 10.0.0.1:1234
+      - 10.0.0.2:1234
 
     index: app
     # or
     index_prefix: app_
 ```
 
+#### Create mappings & import data
+```
+$ rake elastics:migrate elastics:reindex
+```
+
 #### Mappings & index settings
 Mappings & index settings `.yml` files are placed in
 `db/elastics/mappings` & `db/elastics/indices`.
 For now this files are not related to models and only used by rake tasks.
 
-- `rake elastics:create` (or `Elastics::Tasks.create_indices`)
+### Index management
+When index is created elastics transparently manages aliases for it.
+Instead of creating `index1` it creates `index1-v0` and create `index1` alias for it.
+When you perform normal migration, mappings are applied to the current version.
+Later when you perform full migration `index1-v1` is created, after reindexing
+aliases are changed and `index-v0` is droped.
+
+Versions of indices are stored in ElasticSearch in `.elastics` index.
+
+### Rake tasks
+All rake tasks except `purge` accepts list of indices to process
+(`rake elastics:create[index1,index2]`).
+Also you can specify index version like this `rake elastics:migrate version=next`.
+Version can be set to `next` or `current` (default).
+
+Rake tasks are just frontend for `Elastics::Tasks`'s methods.
+For complex migrations, when you need partially reindex data,
+you may want to write custom scripts using this methods.
+
+- `rake elastics:create` (`.create_indices`)
 creates index with settings for each file from `indices` folder.
-For single index it only processes file with index name.
-For multiple indices each index name is `#{index_prefix}#{file.basename}`
 
-- `rake elastics:migrate` (or `Elastics::Tasks.migrate`)
+- `rake elastics:migrate` (`.migrate`)
 puts mappings from `mappings` folder.
 
+- `rake elastics:migrate full=true` (`.migrate!`)
+performs full migration.
+
+- `rake elastics:reindex` (`.reindex`)
+reindexes data.
+
+#### Using without Rails
+You need to setup `Elastics::Tasks` yourself. This can be done in `environment` or
+`db:load_config` rake tasks.
+
+```ruby
+task :environment do
+  Elastics::Tasks.base_paths = '/path/to/your/elastics/folder'
+  Elastics::Tasks.config = your_configuration
+end
+```
+
+Also you need to install `active_support` & require
+`active_support/core_ext/object` to be able to run tasks.
+
+### Use with capistrano
+Add following lines to your `deploy.rb` and all rake tasks will be available in cap.
+
+```ruby
+role :elastics, '%HOSTNAME%', primary: true
+
+require 'elastics/capistrano'
+```
+
+Indices & rake options can be passed like this:
+```
+cap --dry-run elastics:migrate INDICES=index1,index2 ES_OPTIONS='full=true no_drop=true'
+```
+
 ## License
 MIT

data/Rakefile

@@ -1,11 +1,6 @@
 require 'bundler/gem_tasks'
-require 'rake/testtask'
+require 'rspec/core/rake_task'
 
-Rake::TestTask.new do |t|
-  t.libs << 'lib/elastics'
-  t.libs << 'test'
-  t.test_files = FileList['test/lib/elastics/*_test.rb']
-  t.verbose = true
-end
+RSpec::Core::RakeTask.new(:spec)
 
-task default: :test
+task default: :spec

data/elastics.gemspec

@@ -21,5 +21,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'httpclient', '~> 2.4.0'
 
   spec.add_development_dependency 'bundler', '~> 1.5'
-  spec.add_development_dependency 'rake', '~> 10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.1.0'
+  spec.add_development_dependency 'thread_safe', '~> 0.3.4'
+  spec.add_development_dependency 'activesupport', '~> 4.1.6'
 end

data/lib/elastics.rb

@@ -3,11 +3,22 @@ module Elastics
   class NotFound < Error; end
 
   require 'elastics/client'
+  require 'elastics/version_manager'
   require 'elastics/query_helper'
 
   autoload :Tasks, 'elastics/tasks'
 
   extend QueryHelper
+
+  class << self
+    attr_reader :models
+
+    def reset_models
+      @models = []
+    end
+  end
+
+  reset_models
 end
 
 require 'elastics/railtie' if defined?(Rails)

data/lib/elastics/active_record.rb

@@ -2,10 +2,18 @@ module Elastics
   module ActiveRecord
     extend ActiveSupport::Autoload
 
+    autoload :SearchResult
     autoload :ModelSchema
     autoload :HelperMethods
     autoload :Instrumentation
-    autoload :LogSubscriber
+    autoload :LogSubscriber, 'elastics/active_record/instrumentation'
+
+    class << self
+      def install
+        ::ActiveRecord::Base.extend self
+        Instrumentation.install
+      end
+    end
 
     def elastics_config
       @elastics_config ||= connection_config[:elastics].try!(:with_indifferent_access) ||
@@ -16,16 +24,23 @@ module Elastics
       @elastics ||= Client.new elastics_config.slice(:host, :port)
     end
 
+    # Don't memoize to GC it after initialization
+    def elastics_version_manager
+      VersionManager.new(elastics, elastics_config.slice(
+        :service_index,
+        :index_prefix,
+      ))
+    end
+
     def indexed_with_elastics(options = {})
       options = {
         hooks: [:update, :destroy],
-      }.merge(options)
+      }.merge!(options)
 
       extend ModelSchema
       include HelperMethods
-      extend Instrumentation
 
-      self.elastics_index_name  = options[:index] if options[:index]
+      self.elastics_index_base  = options[:index] if options[:index]
       self.elastics_type_name   = options[:type]  if options[:type]
 
       hooks = options[:hooks]

data/lib/elastics/active_record/helper_methods.rb

@@ -3,42 +3,41 @@ module Elastics
     module HelperMethods
       extend ActiveSupport::Concern
 
+      included do
+        alias_method :to_elastics, :as_json unless instance_methods.include?(:to_elastics)
+      end
+
       module ClassMethods
-        def search(data = {}, routing = nil)
-          es_results = search_elastics(data, routing)
-          ids = es_results['hits'.freeze]['hits'.freeze].map { |x| x['_id'.freeze].to_i }
-          relation = where(id: ids)
-          items_by_id = relation.index_by(&:id)
-          collection = ids.map { |i| items_by_id[i] }
-          {
-            collection: collection,
-            relation: relation,
-            search: es_results,
-          }
+        def request_elastics(params)
+          request = {
+            index:  elastics_index_name,
+            type:   elastics_type_name,
+            model:  self,
+          }.merge!(params)
+          elastics.request(request)
         end
 
-        def search_elastics(data = {}, routing = nil)
+        def search_elastics(data = {}, options = {})
           request = {
-            id: :_search,
+            id:   :_search,
             data: data,
           }
-          request[:query] = {routing: routing} if routing
-          request_elastics(request)
+          if routing = options[:routing]
+            request[:query] = {routing: routing}
+          end
+          SearchResult.new self, request_elastics(request), options
         end
 
-        def request_elastics(params)
-          request = {
-            index:  elastics_index_name,
-            type:   elastics_type_name,
-          }.merge!(params)
-          elastics.request(request)
+        def find_all_ordered(ids)
+          items_by_id = where(id: ids).index_by(&:id)
+          ids.map { |i| items_by_id[i] }
         end
 
         def elastics_mapping
           request_elastics(method: :get, id: :_mapping)
         end
 
-        def reindex(*args)
+        def reindex_elastics(*args)
           find_each(*args, &:index_elastics)
         end
       end
@@ -47,12 +46,14 @@ module Elastics
         self.class.request_elastics(method: :post, id: id, data: to_elastics)
       end
 
-      def delete_elastics
-        self.class.request_elastics(method: :delete, id: id)
+      def update_elastics(fields)
+        self.class.request_elastics(method: :post, id: "#{id}/_update", data: {
+          doc: fields
+        })
       end
 
-      def to_elastics
-        as_json
+      def delete_elastics
+        self.class.request_elastics(method: :delete, id: id)
       end
     end
   end

data/lib/elastics/active_record/instrumentation.rb

@@ -1,15 +1,71 @@
 module Elastics
   module ActiveRecord
+    # To be included in `Elastics::Client`
     module Instrumentation
-      def request_elastics(params = {})
-        data = {
-          name:     name,
-          request:  params,
-        }
-        ActiveSupport::Notifications.instrument 'request_elastics.active_record', data do
-          super(params)
+      class << self
+        def install
+          if Client.respond_to?(:prepend)
+            Client.prepend self
+          else
+            Client.include Fallback
+          end
+          unless ::ActiveRecord::LogSubscriber < LogSubscriber
+            ::ActiveRecord::LogSubscriber.send :include, LogSubscriber
+          end
         end
       end
+
+      def http_request(*args)
+        ActiveSupport::Notifications.instrument 'request_elastics.active_record', args: args do
+          super
+        end
+      end
+
+      # old rubies support
+      module Fallback
+        extend ActiveSupport::Concern
+
+        included do
+          alias_method_chain :http_request, :instrumentation
+        end
+
+        def http_request_with_instrumentation(*args)
+          ActiveSupport::Notifications.instrument 'request_elastics.active_record', args: args do
+            http_request_without_instrumentation(*args)
+          end
+        end
+      end
+    end
+
+    module LogSubscriber
+      def self.included(base)
+        instance_methods.each { |method| base.method_added(method) }
+      end
+
+      def request_elastics(event)
+        return unless logger.debug?
+
+        payload = event.payload[:args]
+        method, path, query, body, params = payload
+        path = '/' if path.blank?
+        path << "?#{query.to_param}" if query.present?
+        model = params[:model]
+
+        name = ""
+        name << "#{model.name} " if model
+        name << "elastics (#{event.duration.round(1)}ms)"
+        request = "#{method.to_s.upcase} #{path}"
+        request << " #{body}" if body.present?
+
+        if odd?
+          name    = color(name, ActiveSupport::LogSubscriber::CYAN, true)
+          request = color(request, nil, true)
+        else
+          name = color(name, ActiveSupport::LogSubscriber::MAGENTA, true)
+        end
+
+        debug "  #{name}  #{request}"
+      end
     end
   end
 end

data/lib/elastics/active_record/model_schema.rb

@@ -1,7 +1,17 @@
 module Elastics
   module ActiveRecord
     module ModelSchema
-      attr_writer :elastics_index_name, :elastics_type_name
+      class << self
+        def track_model(model)
+          Elastics.models << model unless model.abstract_class?
+        end
+
+        def extended(base)
+          track_model(base)
+        end
+      end
+
+      attr_writer :elastics_index_base, :elastics_type_name
 
       def elastics_index_name
         reset_elastics_index_name unless defined?(@elastics_index_name)
@@ -9,30 +19,25 @@ module Elastics
       end
 
       def elastics_type_name
-        reset_elastics_index_name unless defined?(@elastics_type_name)
-        @elastics_type_name
+        @elastics_type_name ||= model_name.to_s.demodulize.underscore.singularize
       end
 
       def reset_elastics_index_name
-        superclass_responds = superclass.respond_to?(:elastics_index_name)
-        index = if abstract_class? && superclass_responds
-          superclass == ::ActiveRecord::Base ? nil : superclass.elastics_index_name
-        elsif superclass.abstract_class? && superclass_responds
-          superclass.elastics_index_name || compute_elastics_index_name
-        else
-          compute_elastics_index_name
+        @elastics_index_name = if self != ::ActiveRecord::Base && !abstract_class?
+          superclass.try(:elastics_index_name) || compute_elastics_index_name
         end
-        @elastics_index_name = index
-        @elastics_type_name = compute_elastics_type_name
       end
 
-      def compute_elastics_index_name(name = nil)
-        elastics_config[:index] ||
-          "#{elastics_config[:index_prefix]}#{name || table_name.singularize}"
+      def compute_elastics_index_name
+        elastics_version_manager.index_name(elastics_index_base)
+      end
+
+      def elastics_index_base
+        @elastics_index_base || elastics_config[:index] || elastics_type_name
       end
 
-      def compute_elastics_type_name
-        model_name.to_s.demodulize.underscore.singularize
+      def inherited(base)
+        super.tap { ::Elastics::ActiveRecord::ModelSchema.track_model(base) }
       end
     end
   end