elastic_searchable 1.4.1 → 1.5

data/.rvmrc

@@ -1 +1 @@
-rvm use ruby-1.9.2@elastic_searchable --create
+rvm use ruby-1.9.3-p125@elastic_searchable --create

data/elastic_searchable.gemspec

@@ -21,11 +21,11 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency(%q<activerecord>, [">= 3.0.5"])
   s.add_runtime_dependency(%q<httparty>, [">= 0.6.0"])
-  s.add_runtime_dependency(%q<backgrounded>, [">= 0.7.0"])
+  s.add_runtime_dependency(%q<backgrounded>, ["~> 0.7.0"])
   s.add_runtime_dependency(%q<multi_json>, [">= 1.0.0"])
   s.add_development_dependency(%q<rake>, ["0.9.2.2"])
-  s.add_development_dependency(%q<sqlite3>, ["1.3.4"])
+  s.add_development_dependency(%q<sqlite3>)
   s.add_development_dependency(%q<pry>, ["0.9.6.2"])
   s.add_development_dependency(%q<shoulda>, ["2.11.3"])
-  s.add_development_dependency(%q<mocha>, ["0.10.0"])
+  s.add_development_dependency(%q<mocha>)
 end

data/lib/elastic_searchable/index.rb

@@ -70,7 +70,7 @@ module ElasticSearchable
         scope = options.delete(:scope) || self
         page = options[:page]
         per_page = options[:per_page]
-        records = scope.limit(per_page).offset(per_page * (page -1)).all
+        records = scope.limit(per_page).offset(per_page * (page -1)).to_a
         while records.any? do
           ElasticSearchable.logger.debug "reindexing batch ##{page}..."
           actions = []
@@ -92,7 +92,7 @@ module ElasticSearchable
           end
 
           page += 1
-          records = scope.limit(per_page).offset(per_page* (page-1)).all
+          records = scope.limit(per_page).offset(per_page* (page-1)).to_a
         end
       end
 

data/lib/elastic_searchable/queries.rb

@@ -1,6 +1,7 @@
 module ElasticSearchable
   module Queries
     PER_PAGE_DEFAULT = 20
+    MAX_RETRIES = 5
 
     def per_page
       PER_PAGE_DEFAULT
@@ -14,8 +15,8 @@ module ElasticSearchable
     # http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/
     def search(query, options = {})
       page = (options.delete(:page) || 1).to_i
+      size = (options[:size] ||= per_page_for_search(options))
       options[:fields] ||= '_id'
-      options[:size] ||= per_page_for_search(options)
       options[:from] ||= options[:size] * (page - 1)
       if query.is_a?(Hash)
         options[:query] = query
@@ -35,19 +36,51 @@ module ElasticSearchable
         query[:sort] = sort
       end
 
-      response = ElasticSearchable.request :get, index_type_path('_search'), :query => query, :json_body => options
-      hits = response['hits']
-      ids = hits['hits'].collect {|h| h['_id'].to_i }
-      results = self.find(ids).sort_by {|result| ids.index(result.id) }
+      ids_to_delete = []
+      results = []
+      ids = []
+      hits_total = nil
+      retries = MAX_RETRIES
 
-      results.each do |result|
-        result.instance_variable_set '@hit', hits['hits'][ids.index(result.id)]
+      loop do
+        response = ElasticSearchable.request :get, index_type_path('_search'), :query => query, :json_body => options
+        hits = response['hits']
+        hits_total ||= hits['total'].to_i
+        new_ids = collect_hit_ids(hits)
+        new_results = collect_result_records(new_ids, hits)
+        ids += new_ids
+        results += new_results
+
+        break if results.size >= ids.size || retries <= 0
+
+        retries -= 1
+
+        options[:from] = options[:from] + options[:size]
+        options[:size] = ids.size - results.size
+
+        ids_to_delete += (new_ids - new_results.map(&:id))
+        ids -= ids_to_delete
+      end
+
+      ids_to_delete.each do |id|
+        delete_id_from_index_backgrounded id
       end
 
-      ElasticSearchable::Paginator.handler.new(results, page, options[:size], hits['total'])
+      ElasticSearchable::Paginator.handler.new(results, page, size, hits_total - ids_to_delete.size)
     end
 
     private
+
+    def collect_hit_ids(hits)
+      hits['hits'].collect {|h| h['_id'].to_i }
+    end
+
+    def collect_result_records(ids, hits)
+      self.where(:id => ids).to_a.sort_by{ |result| ids.index(result.id) }.each do |result|
+        result.instance_variable_set '@hit', hits['hits'][ids.index(result.id)]
+      end
+    end
+
     # determine the number of search results per page
     # supports will_paginate configuration by using:
     # Model.per_page

data/lib/elastic_searchable/version.rb

@@ -1,3 +1,3 @@
 module ElasticSearchable
-  VERSION = '1.4.1'
+  VERSION = '1.5'
 end

data/test/helper.rb

@@ -9,7 +9,7 @@ rescue Bundler::BundlerError => e
 end
 require 'test/unit'
 require 'shoulda'
-require 'mocha'
+require 'mocha/setup'
 require 'pry'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
@@ -17,6 +17,19 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'elastic_searchable'
 require 'setup_database'
 
+class DeprecationLogger < Logger
+  def format_message(severity, timestamp, progname, msg)
+    "#{severity} #{msg}\n"
+  end
+end
+
+DEPRECATION_LOGGER = DeprecationLogger.new(File.join(File.dirname(__FILE__), "/deprecations.log"))
+ActiveSupport::Deprecation.debug = false
+ActiveSupport::Deprecation::DEFAULT_BEHAVIORS[:deprecation_log] = lambda { |message, callstack|
+  DEPRECATION_LOGGER.warn(message)
+}
+ActiveSupport::Deprecation.behavior = :deprecation_log
+
 class Test::Unit::TestCase
   def delete_index
     ElasticSearchable.delete '/elastic_searchable' rescue nil

data/test/test_elastic_searchable.rb

@@ -160,7 +160,7 @@ class TestElasticSearchable < Test::Unit::TestCase
     end
     context 'Model.reindex' do
       setup do
-        Post.reindex :per_page => 1, :scope => Post.scoped(:order => 'body desc')
+        Post.reindex :per_page => 1, :scope => Post.order('body desc')
         Post.refresh_index
       end
       should 'have reindexed both records' do
@@ -288,6 +288,39 @@ class TestElasticSearchable < Test::Unit::TestCase
     end
   end
 
+  context 'deleting a record without updating the index' do
+
+    context 'backfilling partial result pages' do
+      setup do
+        @posts = (1..8).map do |i|
+          Post.create :title => 'foo', :body => "#{i} bar"
+        end
+        Post.refresh_index
+
+        @removed_posts = []
+        @posts.each_with_index do |post, i|
+          if i % 2 == 1
+            @removed_posts << post
+            post.delete
+            Post.expects(:delete_id_from_index_backgrounded).with(post.id)
+          end
+        end
+
+        assert_nothing_raised do
+          @results = Post.search 'foo', :size => 4, :sort => 'id:desc'
+        end
+      end
+      # should 'not raise an exception' do end
+      # should 'trigger deletion of bad index records' do end
+      should 'backfill the first page with results from other pages' do
+        assert_equal((@posts - @removed_posts).reverse, @results)
+      end
+      should 'adjust total hit count to exclude invalid results encountered from repaging' do
+        assert_equal 4, @results.total_entries
+      end
+    end
+
+  end
 
   class Blog < ActiveRecord::Base
     elastic_searchable :if => proc {|b| b.should_index? }, :index_options => SINGLE_NODE_CLUSTER_CONFIG
@@ -324,6 +357,7 @@ class TestElasticSearchable < Test::Unit::TestCase
       end
       should 'have used custom index_options' do
         @status = ElasticSearchable.request :get, '/elastic_searchable/_settings'
+        @status['elastic_searchable']['settings'].delete('index.version.created')
         expected = {
           "index.number_of_replicas" => "0",
           "index.number_of_shards" => "1",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastic_searchable
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: '1.5'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-22 00:00:00.000000000 Z
+date: 2013-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -48,7 +48,7 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.0
   type: :runtime
@@ -56,7 +56,7 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.0
 - !ruby/object:Gem::Dependency
@@ -96,17 +96,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -144,17 +144,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0'
 description: integrate the elastic search engine with rails
 email:
 - ryan@codecrate.com
@@ -196,18 +196,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 3119629998043065822
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 3119629998043065822
 requirements: []
 rubyforge_project: elastic_searchable
 rubygems_version: 1.8.24