elastic-rails 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f9ab5fe73898a276646cbf7c3f9a564d3f7419e4
-  data.tar.gz: a90c322ea5f8226affff52f5646974ac42aebe5a
+  metadata.gz: 7d00b5b5c6f5a1bc7701c5d635198e88094e65ea
+  data.tar.gz: 09550318bf8064a884b097576ff2aa15d6702daa
 SHA512:
-  metadata.gz: 54b6f86610924e06e58690fa99e45d74dd2c18f5fd902985efe5ff6ab827117a44ceb6be50a8637f3dee085591f677514d706556907af15e2c2b2925ebc45980
-  data.tar.gz: 346536b41ffcaef70416b0afd92f2c54ee4b4658077a500cb71cda8c24fd572636d619eae0f4456873b8115c84dbb3db6a1ad12110755780c3663ba5709db417
+  metadata.gz: 5be452213033d0d2fb6a868d65709ed68260cfffb847566dd3ac2094a571c69aba9b6b75a06455f85091d5a35a12269cab78b68435efd355af40199cb1e8f4c1
+  data.tar.gz: 03b952d5219a6c47606db579f1b06c73c8a9c326949ce12da855f995c9053adfdd458b5e1570ae71296446a8412b4a4485c9615d3dc4d8800ca0aedc9a95ced8

data/README.md

@@ -60,7 +60,7 @@ BikeIndex.must(brand: 'Trek', size: 'M').should(year: { gte: 2015 }).avg(:price)
 BikeIndex.must(origin: 'China').segment(:brand).each { |brand, bikes|   }
 ```
 
-migrate: remaps if necessary 
+migrate: remaps if necessary
 
 reindex: attempts to rotate if index already exist
 
@@ -70,8 +70,6 @@ TODO: Write usage instructions here
 
 These are some features that will be added in the future:
 
-* Support for record deletion
-
 
 ## Development
 

data/lib/elastic/core/connector.rb

@@ -1,23 +1,16 @@
 module Elastic::Core
   class Connector
-    def initialize(_name, _types, _mapping)
+    def initialize(_name, _types, _mapping, settling_time: 10.seconds)
       @name = _name
       @types = _types
       @mapping = _mapping
+      @settling_time = settling_time
     end
 
     def index_name
       @index_name ||= "#{Elastic.config.index}_#{@name}"
     end
 
-    def read_index_name
-      index_name
-    end
-
-    def write_index_name
-      Thread.current[write_index_thread_override] || write_index_alias
-    end
-
     def status
       actual_name = resolve_actual_index_name
       return :not_available if actual_name.nil?
@@ -47,8 +40,8 @@ module Elastic::Core
 
     def migrate(batch_size: nil)
       unless remap
-        rollover do
-          copy_documents(read_index_name, write_index_name, batch_size || default_batch_size)
+        rollover do |new_index|
+          copy_to new_index, batch_size: batch_size
         end
       end
 
@@ -57,84 +50,130 @@ module Elastic::Core
 
     def index(_document)
       # TODO: validate document type
+      operations = write_indices.map do |write_index|
+        { 'index' => _document.merge('_index' => write_index) }
+      end
 
-      api.index(
-        index: write_index_name,
-        id: _document['_id'],
-        type: _document['_type'],
-        body: _document['data']
-      )
+      api.bulk(body: operations)
     end
 
     def bulk_index(_documents)
       # TODO: validate documents type
-
       body = _documents.map { |doc| { 'index' => doc } }
 
-      retry_on_temporary_error('bulk indexing') do
-        api.bulk(index: write_index_name, body: body)
+      write_indices.each do |write_index|
+        retry_on_temporary_error('bulk indexing') do
+          api.bulk(index: write_index, body: body)
+        end
       end
     end
 
-    def refresh
-      api.indices.refresh index: read_index_name
+    def delete(_document)
+      raise ArgumentError, 'document must provide an id' unless _document['_id']
+      raise ArgumentError, 'document must provide a type' unless _document['_type']
+
+      write_index, rolling_index = write_indices
+
+      operations = [{ 'delete' => _document.merge('_index' => write_index) }]
+
+      if rolling_index
+        operations << {
+          'index' => _document.merge(
+            '_index' => rolling_index,
+            'data' => { '_mark_for_deletion' => true }
+          )
+        }
+      end
+
+      api.bulk(body: operations)
     end
 
-    def find(_type, _id)
-      api.get(index: write_index_name, type: _type, id: _id)
+    def refresh
+      api.indices.refresh index: index_name
     end
 
-    def delete(_type, _id)
-      api.delete(index: write_index_name, type: _type, id: _id)
+    def find(_type, _id)
+      api.get(index: index_name, type: _type, id: _id)
     end
 
     def count(query: nil, type: nil)
-      api.count(index: read_index_name, type: type, body: query)['count']
+      api.count(index: index_name, type: type, body: query)['count']
     end
 
     def query(query: nil, type: nil)
-      api.search(index: read_index_name, type: type, body: query)
+      api.search(index: index_name, type: type, body: query)
     end
 
-    def rollover(&_block) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def rollover(&_block) # rubocop:disable Metrics/MethodLength
+      actual_index, rolling_index = resolve_write_indices
+
+      unless rolling_index.nil?
+        raise Elastic::RolloverError, 'rollover process already in progress'
+      end
+
       new_index = create_index_w_mapping
-      tmp_index = create_index_w_mapping('tmp')
-      actual_index = resolve_actual_index_name
 
       begin
-        transfer_alias(write_index_alias, from: actual_index, to: tmp_index)
-
+        transfer_alias(write_index_alias, to: new_index)
+        wait_for_index_to_stabilize
         perform_optimized_write_on(new_index, &_block)
-
+        delete_marked_for_deletion new_index
         transfer_alias(index_name, from: actual_index, to: new_index)
-        transfer_alias(write_index_alias, from: tmp_index, to: new_index)
-        api.indices.delete index: actual_index if actual_index
+        transfer_alias(write_index_alias, from: actual_index)
+        wait_for_index_to_stabilize
+        api.indices.delete index: actual_index
       rescue
-        transfer_alias(write_index_alias, from: tmp_index, to: actual_index)
         api.indices.delete index: new_index
-      ensure
-        # rollback
-        # TODO: what would happen if the following fails? O.O
-        copy_documents(tmp_index, write_index_name, small_batch_size)
-        api.indices.delete index: tmp_index
-        api.indices.refresh index: index_name
+        raise
+      end
+    end
+
+    def copy_to(_to, batch_size: nil) # rubocop:disable Metrics/AbcSize
+      api.indices.refresh index: index_name
+
+      r = api.search(
+        index: index_name,
+        body: { sort: ['_doc'] },
+        scroll: '5m',
+        size: batch_size || default_batch_size
+      )
+
+      count = 0
+      while !r['hits']['hits'].empty?
+        count += r['hits']['hits'].count
+        Elastic.logger.info "Copied #{count} docs"
+
+        body = r['hits']['hits'].map { |h| transform_hit_to_create(h) }
+        api.bulk(index: _to, body: body)
+
+        r = api.scroll scroll: '5m', scroll_id: r['_scroll_id']
       end
     end
 
     private
 
+    def wait_for_index_to_stabilize
+      return if @settling_time == 0
+      Elastic.logger.info "Waiting #{@settling_time * 1.2}s for write indices to stabilize ..."
+      sleep(@settling_time * 1.2)
+    end
+
     def api
       Elastic.config.api_client
     end
 
+    def write_indices
+      Thread.current[write_index_thread_override] || resolve_write_indices
+    end
+
     def perform_optimized_write_on(_index)
-      old_index = Thread.current[write_index_thread_override]
-      Thread.current[write_index_thread_override] = _index
+      old_indices = Thread.current[write_index_thread_override]
+      Thread.current[write_index_thread_override] = [_index]
       configure_index(_index, refresh_interval: -1)
-      yield
+      yield _index
     ensure
       configure_index(_index, refresh_interval: '1s')
-      Thread.current[write_index_thread_override] = old_index
+      Thread.current[write_index_thread_override] = old_indices
     end
 
     def write_index_thread_override
@@ -142,7 +181,26 @@ module Elastic::Core
     end
 
     def write_index_alias
-      @write_index_alias = "#{index_name}.w"
+      @write_index_alias ||= "#{index_name}.w"
+    end
+
+    def resolve_write_indices
+      @write_indices = nil if write_indices_expired?
+      @write_indices ||= begin
+        result = api.indices.get_alias(name: write_index_alias)
+        @write_indices_expiration = @settling_time.from_now
+        result.keys.sort # lower timestamp first (actual)
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        raise Elastic::MissingIndexError, 'index does not exist, call migrate first'
+      end
+    end
+
+    def delete_marked_for_deletion(_index)
+      api.delete_by_query(index: _index, body: { query: { term: { _mark_for_deletion: true } } })
+    end
+
+    def write_indices_expired?
+      @write_indices_expiration && @write_indices_expiration < Time.current
     end
 
     def resolve_actual_index_name
@@ -152,8 +210,8 @@ module Elastic::Core
       nil
     end
 
-    def create_index_w_mapping(_role = 'main')
-      new_name = "#{index_name}:#{_role}:#{Time.now.to_i}"
+    def create_index_w_mapping
+      new_name = "#{index_name}:#{Time.now.to_i}"
       api.indices.create index: new_name
       api.cluster.health wait_for_status: 'yellow'
       setup_index_types new_name
@@ -201,44 +259,24 @@ module Elastic::Core
       api.indices.update_aliases body: { actions: actions }
     end
 
-    def copy_documents(_from, _to, _batch_size)
-      api.indices.refresh index: _from
-
-      r = api.search(
-        index: _from,
-        body: { sort: ['_doc'] },
-        scroll: '5m',
-        size: _batch_size
-      )
-
-      count = 0
-      while !r['hits']['hits'].empty?
-        count += r['hits']['hits'].count
-        Elastic.logger.info "Copied #{count} docs"
-
-        body = r['hits']['hits'].map { |h| { 'index' => transform_hit_to_doc(h) } }
-        api.bulk(index: _to, body: body)
-
-        r = api.scroll scroll: '5m', scroll_id: r['_scroll_id']
-      end
-    end
-
     def configure_index(_index, _settings)
       api.indices.put_settings index: _index, body: { index: _settings }
     end
 
-    def transform_hit_to_doc(_hit)
-      { '_id' => _hit['_id'], '_type' => _hit['_type'], 'data' => _hit['_source'] }
+    def transform_hit_to_create(_hit)
+      {
+        'create' => {
+          '_id' => _hit['_id'],
+          '_type' => _hit['_type'],
+          'data' => _hit['_source']
+        }
+      }
     end
 
     def default_batch_size
       1_000
     end
 
-    def small_batch_size
-      500
-    end
-
     def retry_on_temporary_error(_action, retries: 3)
       return yield
     rescue Elasticsearch::Transport::Transport::Errors::ServiceUnavailable,

data/lib/elastic/core/serializer.rb

@@ -16,34 +16,21 @@ module Elastic::Core
       @definition.fields
     end
 
-    def read_elastic_type
-      object.class.to_s
-    end
-
-    def read_elastic_id
-      if has_attribute_for_indexing?(:id)
-        read_attribute_for_indexing(:id)
-      end
+    def as_elastic_document(only_meta: false)
+      result = { '_type' => object.class.to_s }
+      result['_id'] = read_attribute_for_indexing(:id) if has_attribute_for_indexing?(:id)
+      result['data'] = as_elastic_source unless only_meta
+      result
     end
 
-    def as_elastic_document(only_data: false)
-      data = {}.tap do |hash|
+    def as_elastic_source
+      {}.tap do |hash|
         fields.each do |field|
           value = read_attribute_for_indexing(field.name)
           value = field.prepare_value_for_index(value)
           hash[field.name] = value
         end
       end
-
-      return data if only_data
-
-      result = {
-        '_type' => read_elastic_type,
-        'data' => data
-      }
-
-      read_elastic_id.tap { |id| result['_id'] = id unless id.nil? }
-      result
     end
 
     private

data/lib/elastic/errors.rb

@@ -1,4 +1,10 @@
 module Elastic
   class Error < StandardError
   end
+
+  class MissingIndexError < Error
+  end
+
+  class RolloverError < Error
+  end
 end

data/lib/elastic/fields/nested.rb

@@ -41,7 +41,7 @@ module Elastic::Fields
     end
 
     def prepare_value_for_index(_values)
-      _values.map { |v| @index.new(v).as_elastic_document(only_data: true) }
+      _values.map { |v| @index.new(v).as_elastic_source }
     end
 
     def prepare_value_for_result(_values)

data/lib/elastic/railtie.rb

@@ -4,7 +4,8 @@ require "elastic/railties/ar_middleware"
 require "elastic/railties/configuration_extensions"
 require "elastic/railties/type_extensions"
 require "elastic/railties/query_extensions"
-# disabled for now: require "elastic/railties/indexing_job"
+require "elastic/railties/jobs/indexing_job"
+require "elastic/railties/jobs/deleting_job"
 require "elastic/railties/indexable_record"
 
 module Elastic

data/lib/elastic/railties/ar_helpers.rb

@@ -8,7 +8,7 @@ module Elastic::Railties
         _collection = _collection.send(scope) if scope
         _collection.find_each(&_block)
       elsif _collection.respond_to? :each
-        ActiveRecord::Associations::Preloader.new.preload(_collection, *includes) if includes
+        ActiveRecord::Associations::Preloader.new.preload(_collection, includes) if includes
         _collection.each(&_block)
       else
         raise 'Elastic ActiveRecord importing is only supported for collection types'

data/lib/elastic/railties/indexable_record.rb

@@ -18,32 +18,28 @@ module Elastic::Railties
         @constantized_index_class ||= index_class.constantize
       end
 
-      def index(on: nil, unindex: true, delayed: false)
-        raise NotImplementedError, 'delayed indexing not implemented' if delayed
+      def index(on: nil, unindex: true, delayed: true)
+        index_m, unindex_m = delayed ? [:index_later, :unindex_later] : [:index_now, :unindex_now]
 
         if on == :create
-          index_on_create
+          after_create { public_send(index_m) }
         elsif on == :save
-          index_on_save
+          after_save { public_send(index_m) }
         else
           raise ArgumentError, 'must provide an indexing target when calling index \
 (ie: `index on: :save`)'
         end
 
-        unindex_on_destroy if unindex
-      end
-
-      def index_on_create(_options = {})
-        after_create(_options) { index_now }
+        before_destroy { public_send(unindex_m) } if unindex
       end
+    end
 
-      def index_on_save(_options = {})
-        after_save(_options) { index_now }
-      end
+    def index_later
+      self.class.constantized_index_class.index_later self
+    end
 
-      def unindex_on_destroy(_options = {})
-        before_destroy(_options) { unindex_now }
-      end
+    def unindex_later
+      self.class.constantized_index_class.delete_later self
     end
 
     def index_now

data/lib/elastic/railties/jobs/deleting_job.rb

@@ -0,0 +1,7 @@
+module Elastic::Railties::Jobs
+  class DeletingJob < ActiveJob::Base
+    def perform(_type, _document)
+      _type.constantize.connector.delete _document
+    end
+  end
+end

data/lib/elastic/railties/jobs/indexing_job.rb

@@ -0,0 +1,7 @@
+module Elastic::Railties::Jobs
+  class IndexingJob < ActiveJob::Base
+    def perform(_type, _document)
+      _type.constantize.connector.index _document
+    end
+  end
+end

data/lib/elastic/railties/type_extensions.rb

@@ -9,6 +9,22 @@ module Elastic::Railties
         # TODO: check target allows options
         pre_definition.middleware_options[:ar_collect_includes] = _includes
       end
+
+      def index_later(_object)
+        wrapped = new(_object)
+
+        Jobs::IndexingJob
+          .set(queue: Elastic.config.active_job_queue)
+          .perform_later(to_s, wrapped.as_elastic_document.as_json)
+      end
+
+      def delete_later(_object)
+        wrapped = new(_object)
+
+        Jobs::DeletingJob
+          .set(queue: Elastic.config.active_job_queue)
+          .perform_later(to_s, wrapped.as_elastic_document(only_meta: true).as_json)
+      end
     end
   end
 end

data/lib/elastic/type.rb

@@ -78,20 +78,13 @@ module Elastic
     end
 
     def self.index(_object)
-      new(_object).save
+      connector.index new(_object).as_elastic_document
+      self
     end
 
     def self.delete(_object)
-      wrapper = new(_object)
-      id = wrapper.read_elastic_id
-      raise ArgumentError, 'index does not provide an id' if id.nil?
-
-      connector.delete(
-        wrapper.read_elastic_type,
-        wrapper.read_elastic_id
-      )
-
-      nil
+      connector.delete new(_object).as_elastic_document(only_meta: true)
+      self
     end
 
     def self.query
@@ -107,11 +100,5 @@ module Elastic
       connector.refresh
       self
     end
-
-    def save
-      self.class.tap do |klass|
-        klass.connector.index as_elastic_document
-      end
-    end
   end
 end

data/lib/elastic/version.rb

@@ -1,3 +1,3 @@
 module Elastic
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elastic-rails
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-29 00:00:00.000000000 Z
+date: 2016-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -285,7 +285,8 @@ files:
 - lib/elastic/railties/ar_middleware.rb
 - lib/elastic/railties/configuration_extensions.rb
 - lib/elastic/railties/indexable_record.rb
-- lib/elastic/railties/indexing_job.rb
+- lib/elastic/railties/jobs/deleting_job.rb
+- lib/elastic/railties/jobs/indexing_job.rb
 - lib/elastic/railties/query_extensions.rb
 - lib/elastic/railties/rspec.rb
 - lib/elastic/railties/tasks/es.rake

data/lib/elastic/railties/indexing_job.rb

@@ -1,8 +0,0 @@
-module Elastic::Railties
-  class IndexingJob < ActiveJob::Base
-    def perform(*_indexables)
-      # TODO: use import for many indexables
-      _indexables.each &:index_now
-    end
-  end
-end