elastic_record 1.1.4 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 43d163448ac2df0ed13d06df549e2f986f812491
-  data.tar.gz: 99603b58b72f7d4c74d49cd99e0c9877ad9a32a9
+  metadata.gz: 0f00def1bf454a01cf7db164569a256fa3872f12
+  data.tar.gz: 3674835adb8a4d2b803f8b00b1f71e9239e3cf15
 SHA512:
-  metadata.gz: 65ff3c68ddab4c46310bef51356c2f7c164f2aac953103c92bd7471854c5ee8feede0824f5e472db227404f0af560fc58247b474ec39d0484680a96158af73f1
-  data.tar.gz: 97198a8a88d57eab1fc7b8ecc12ebfd368f172f5f3fd6548765e83d35bb8b53d4bc2ace3a2da001ac8392288421590c1cae5203f67c24e670260c34db87bb581
+  metadata.gz: cbe0b1f75dc593b47372e0b4d8da74398747dfd87c520e3286baf459e349f8665a6c9863111b3c3415da749c97f1a503a66356a58104cda647008dfd5351ccd2
+  data.tar.gz: a5d239eab2b96cab5aa461bd06671d6e46f7c22ff85ba688b7e0a5e22b4688c385446ba37af750e1b3476488f0028fd2d4da90b034afda842b35e8cfb053ebe4

data/Gemfile

@@ -2,5 +2,7 @@ source 'https://rubygems.org'
 
 gemspec
 
-gem 'rake'
+gem 'activerecord'
+gem 'mysql2'
 gem 'fakeweb'
+gem 'rake'

data/Rakefile

@@ -8,6 +8,6 @@ desc 'Test.'
 Rake::TestTask.new(:test) do |t|
   t.libs << 'lib'
   t.libs << 'test'
-  t.pattern = 'test/**/*_test.rb'
+  t.pattern = 'test/elastic_record/**/*_test.rb'
   t.verbose = true
-end
+end

data/elastic_record.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = 'elastic_record'
-  s.version = '1.1.4'
+  s.version = '1.1.5'
   s.summary = 'Use Elastic Search with your objects'
   s.description = 'Find your records with elastic search'
 
@@ -18,6 +18,6 @@ Gem::Specification.new do |s|
   s.files       = `git ls-files`.split("\n")
   s.test_files  = `git ls-files -- {test}/*`.split("\n")
 
-  s.add_dependency 'arelastic', '>= 0.3.0'
+  s.add_dependency 'arelastic', '>= 0.4.0'
   s.add_dependency 'activemodel'
 end

data/lib/elastic_record.rb

@@ -21,4 +21,5 @@ module ElasticRecord
   end
 end
 
+require 'elastic_record/errors'
 require 'elastic_record/railtie' if defined?(Rails)

data/lib/elastic_record/callbacks.rb

@@ -3,7 +3,7 @@ module ElasticRecord
     def self.included(base)
       base.class_eval do
         after_save if: :changed? do
-          self.class.elastic_index.index_document id, as_search
+          self.class.elastic_index.index_record self
         end
 
         after_destroy do
@@ -12,15 +12,23 @@ module ElasticRecord
       end
     end
 
+
     def as_search
       json = {}
 
-      elastic_index.mapping[:properties].each do |key, value|
-        next if !respond_to?(key) || value[:type] == 'object'
-        value = send(key)
+      elastic_index.mapping[:properties].each do |field, mapping|
+        next if !respond_to?(field)
+        value = send(field)
 
         if value.present? || value == false
-          json[key] = value
+          json[field] = case mapping[:type]
+            when :object
+              value.as_search
+            when :nested
+              value.map(&:as_search)
+            else
+              value
+            end
         end
       end
 

data/lib/elastic_record/connection.rb

@@ -1,13 +1,11 @@
 require 'net/http'
 
 module ElasticRecord
-  class ConnectionError < StandardError
-  end
-
   class Connection
     attr_accessor :servers, :options
     attr_accessor :request_count, :current_server
     attr_accessor :max_request_count
+    attr_accessor :bulk_stack
     def initialize(servers, options = {})
       if servers.is_a?(Array)
         self.servers = servers
@@ -15,10 +13,11 @@ module ElasticRecord
         self.servers = servers.split(',')
       end
 
-      self.current_server = next_server
-      self.request_count = 0
-      self.max_request_count = 100
-      self.options = options
+      self.current_server     = next_server
+      self.request_count      = 0
+      self.max_request_count  = 100
+      self.options            = options
+      self.bulk_stack        = []
     end
 
     def head(path)
@@ -78,7 +77,7 @@ module ElasticRecord
         if @shuffled_servers.nil?
           @shuffled_servers = servers.shuffle
         else
-          @shuffled_servers.push(@shuffled_servers.shift)
+          @shuffled_servers.rotate!
         end
 
         @shuffled_servers.first

data/lib/elastic_record/errors.rb

@@ -0,0 +1,10 @@
+module ElasticRecord
+  class Error < StandardError
+  end
+
+  class ConnectionError < Error
+  end
+
+  class BulkError < Error
+  end
+end

data/lib/elastic_record/index/deferred.rb

@@ -11,9 +11,11 @@ module ElasticRecord
         attr_accessor :index
         attr_accessor :deferred_actions
         attr_accessor :writes_made
+        attr_accessor :bulk_stack
 
         def initialize(index)
           self.index = index
+          self.bulk_stack = []
           reset!
         end
 
@@ -45,12 +47,16 @@ module ElasticRecord
 
             if READ_METHODS.include?(method)
               flush!
-              index.real_connection.json_post "/#{index.alias_name}/_refresh"
+              index.real_connection.json_post("/#{index.alias_name}/_refresh") if requires_refresh?(method, *args)
               index.real_connection.send(method, *args, &block)
             else
               deferred_actions << DeferredAction.new(method, args, block)
             end
           end
+
+          def requires_refresh?(method, *args)
+            method == :json_get && args.first =~ /_search/
+          end
       end
 
       def enable_deferring!

data/lib/elastic_record/index/documents.rb

@@ -3,14 +3,20 @@ require 'active_support/core_ext/object/to_query'
 module ElasticRecord
   class Index
     module Documents
+      def index_record(record, index_name = nil)
+        return if disabled
+
+        index_document(record.send(record.class.primary_key), record.as_search, index_name)
+      end
+
       def index_document(id, document, index_name = nil)
         return if disabled
 
         index_name ||= alias_name
 
-        if batch = current_batch
-          current_batch << { index: { _index: index_name, _type: type, _id: id } }
-          current_batch << document
+        if batch = current_bulk_batch
+          batch << { index: { _index: index_name, _type: type, _id: id } }
+          batch << document
         else
           connection.json_put "/#{index_name}/#{type}/#{id}", document
         end
@@ -19,7 +25,7 @@ module ElasticRecord
       def delete_document(id, index_name = nil)
         index_name ||= alias_name
 
-        if batch = current_batch
+        if batch = current_bulk_batch
           batch << { delete: { _index: index_name, _type: type, _id: id } }
         else
           connection.json_delete "/#{index_name}/#{type}/#{id}"
@@ -52,15 +58,18 @@ module ElasticRecord
         connection.json_get("/_search/scroll?#{options.to_query}")
       end
 
-      def bulk
-        @_batch = []
+      def bulk(options = {})
+        connection.bulk_stack.push []
+
         yield
-        if @_batch.any?
-          body = @_batch.map { |action| "#{ActiveSupport::JSON.encode(action)}\n" }.join
-          connection.json_post "/_bulk", body
+
+        if current_bulk_batch.any?
+          body = current_bulk_batch.map { |action| "#{ActiveSupport::JSON.encode(action)}\n" }.join
+          results = connection.json_post("/_bulk?#{options.to_query}", body)
+          verify_bulk_results(results)
         end
       ensure
-        @_batch = nil
+        connection.bulk_stack.pop
       end
 
       def bulk_add(batch, index_name = nil)
@@ -68,18 +77,26 @@ module ElasticRecord
 
         bulk do
           batch.each do |record|
-            index_document(record.id, record.as_search, index_name)
+            index_record(record, index_name)
           end
         end
       end
 
-      def current_batch
-        if @_batch
-          @_batch
-        elsif model.superclass.respond_to?(:elastic_index)
-          model.superclass.elastic_index.current_batch
-        end
+      def current_bulk_batch
+        connection.bulk_stack.last
       end
+
+      private
+
+        def verify_bulk_results(results)
+          return unless results.is_a?(Hash)
+
+          errors = results['items'].select do |item|
+            item.values.first['error']
+          end
+
+          raise ElasticRecord::BulkError.new(errors) unless errors.empty?
+        end
     end
   end
 end

data/lib/elastic_record/index/manage.rb

@@ -23,6 +23,14 @@ module ElasticRecord
         end
       end
 
+      def open(index_name)
+        connection.json_post("/#{index_name}/_open")
+      end
+
+      def close(index_name)
+        connection.json_post("/#{index_name}/_close")
+      end
+
       def exists?(index_name)
         connection.head("/#{index_name}") == '200'
       end

data/lib/elastic_record/index/percolator.rb

@@ -2,13 +2,6 @@ module ElasticRecord
   class Index
     module Percolator
       def create_percolator(name, elastic_query)
-        unless exists? percolator_index_name
-          create percolator_index_name
-        else
-          delete_mapping(percolator_index_name) if type_exists?(percolator_index_name)
-          update_mapping percolator_index_name
-        end
-
         connection.json_put "/_percolator/#{percolator_index_name}/#{name}", elastic_query
       end
 
@@ -17,20 +10,33 @@ module ElasticRecord
       end
 
       def percolator_exists?(name)
-        connection.json_get("/_percolator/#{percolator_index_name}/#{name}")['exists']
+        !get_percolator(name).nil?
+      end
+
+      def get_percolator(name)
+        json = connection.json_get("/_percolator/#{percolator_index_name}/#{name}")
+        json['_source'] if json['exists']
       end
 
       def percolate(document)
         connection.json_get("/#{percolator_index_name}/#{type}/_percolate", 'doc' => document)['matches']
       end
 
+      def all_percolators
+        if hits = connection.json_get("/_percolator/#{percolator_index_name}/_search?q=*&size=500")['hits']
+          hits['hits'].map { |hit| hit['_id'] }
+        end
+      end
+
       def reset_percolator
         delete(percolator_index_name) if exists?(percolator_index_name)
+        create(percolator_index_name)
       end
 
       def percolator_index_name
-        @percolator_index_name ||= "percolate_#{alias_name}"
+        alias_name
+        # @percolator_index_name ||= "percolate_#{alias_name}"
       end
     end
   end
-end
+end

data/lib/elastic_record/integration/active_record.rb

@@ -0,0 +1,7 @@
+module ActiveRecord
+  class Base
+    def self.load_elastic_record_hits(ids)
+      order("FIELD(#{connection.quote_column_name(primary_key)}, #{ids.join(',')})").find(ids)
+    end
+  end
+end

data/lib/elastic_record/integration/cassandra_object.rb

@@ -0,0 +1,7 @@
+module CassandraObject
+  class Base
+    def self.load_elastic_record_hits(ids)
+      find(ids)
+    end
+  end
+end

data/lib/elastic_record/model.rb

@@ -5,6 +5,9 @@ module ElasticRecord
         extend Searching
         extend ClassMethods
         include SearchesMany
+
+        class_attribute :elastic_connection
+        self.elastic_connection = ElasticRecord::Connection.new(ElasticRecord::Config.servers, ElasticRecord::Config.connection_options)
       end
     end
 
@@ -17,20 +20,12 @@ module ElasticRecord
         end
       end
 
-      def elastic_connection
-        @elastic_connection ||= ElasticRecord::Connection.new(ElasticRecord::Config.servers, ElasticRecord::Config.connection_options)
-      end
-
-      def elastic_connection=(connection)
-        @elastic_connection = connection
-      end
-
       def elastic_relation
-        ElasticRecord::Relation.new(self, arelastic)
+        ElasticRecord::Relation.new(self)
       end
 
       def arelastic
-        @arelastic ||= Arelastic::Builders::Search.new
+        Arelastic::Builders::Search
       end
 
       def elastic_index

data/lib/elastic_record/railtie.rb

@@ -15,5 +15,15 @@ module ElasticRecord
         include ElasticRecord::Railties::ControllerRuntime
       end
     end
+
+    initializer "elastic_record.orm" do |app|
+      ActiveSupport.on_load(:active_record) do
+        require 'elastic_record/integration/active_record'
+      end
+
+      ActiveSupport.on_load(:cassandra_object) do
+        require 'elastic_record/integration/cassandra_object'
+      end
+    end
   end
 end

data/lib/elastic_record/relation.rb

@@ -11,11 +11,10 @@ module ElasticRecord
   class Relation
     include Admin, Batches, Delegation, FinderMethods, Merging, SearchMethods
 
-    attr_reader :klass, :arelastic, :values
+    attr_reader :klass, :values
 
-    def initialize(klass, arelastic)
+    def initialize(klass)
       @klass = klass
-      @arelastic = arelastic
       @values = {}
     end
 
@@ -36,10 +35,16 @@ module ElasticRecord
       reset
     end
 
+    def eager_loading?
+      @should_eager_load ||= eager_load_values.any?
+    end
+
     def to_a
       @records ||= begin
         scope = select_values.any? ? klass.select(select_values) : klass
-        scope.find(to_ids)
+        records = scope.load_elastic_record_hits(to_ids)
+        eager_load_associations(records) if eager_loading?
+        records
       end
     end
 
@@ -65,6 +70,7 @@ module ElasticRecord
     private
       def reset
         @search_results = @records = nil
+        @should_eager_load = nil
       end
 
       def search_hits
@@ -74,5 +80,20 @@ module ElasticRecord
       def search_results
         @search_results ||= klass.elastic_index.search(as_elastic)
       end
+
+      def eager_load_associations(records)
+        records = records.reject(&:marked_for_destruction?)
+        ids = records.map(&:id)
+        eager_load_values.each do |to_load|
+          reflection = klass.searches_many_reflections[to_load] ||(raise "searches_many #{to_load} does not exist on #{klass}")
+          foreign_key = reflection.foreign_key.to_sym
+          grouped_children = reflection.klass.elastic_search.filter(foreign_key => ids).limit(1000000).group_by(&foreign_key)
+          records.each do |record|
+            children = grouped_children.fetch(record.id, [])
+            record.send(to_load).eager_loaded(children)
+          end
+        end
+        records
+      end
   end
-end
+end