elastic_searchable 1.6 → 2.0.0

data/.gitignore

@@ -18,8 +18,6 @@ pkg
 # test files
 test/*.log
 test/*.sqlite3
-spec/**/*.log
-spec/*.sqlite3
 
 # For vim:
 *.swp

data/.rvmrc

@@ -1 +1 @@
-rvm use default@elastic_searchable --create
+rvm use ruby-1.9.3@elastic_searchable --create

data/CONTRIBUTORS.txt

@@ -1,5 +1,4 @@
 Ryan Sonnek - Original Author
-Geoff Hichborn - Implemented search result autoscrubbing
 
 
 Complete list of contributors:

data/Rakefile

@@ -1,11 +1,13 @@
-require "bundler/gem_tasks"
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
 require 'rake'
 
-require 'rspec/core/rake_task'
-
-desc "Run specs"
-RSpec::Core::RakeTask.new do |t|
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
 end
-task :default => :spec
-task :test => :spec
+task :default => :test
+

data/elastic_searchable.gemspec

@@ -21,12 +21,11 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency(%q<activerecord>, [">= 3.0.5"])
   s.add_runtime_dependency(%q<httparty>, [">= 0.6.0"])
-  s.add_runtime_dependency(%q<backgrounded>, ["~> 0.7.0"])
+  s.add_runtime_dependency(%q<backgrounded>, [">= 0.7.0"])
   s.add_runtime_dependency(%q<multi_json>, [">= 1.0.0"])
-  s.add_development_dependency(%q<rake>)
-  s.add_development_dependency(%q<sqlite3>)
-  s.add_development_dependency(%q<pry>)
-  s.add_development_dependency(%q<rspec>)
-  s.add_development_dependency(%q<byebug>)
-  s.add_development_dependency(%q<prefactory>)
+  s.add_development_dependency(%q<rake>, ["0.9.2.2"])
+  s.add_development_dependency(%q<sqlite3>, ["1.3.4"])
+  s.add_development_dependency(%q<pry>, ["0.9.6.2"])
+  s.add_development_dependency(%q<shoulda>, ["2.11.3"])
+  s.add_development_dependency(%q<mocha>, ["0.10.0"])
 end

data/lib/elastic_searchable/active_record_extensions.rb

@@ -8,9 +8,7 @@ require 'elastic_searchable/paginator'
 module ElasticSearchable
   module ActiveRecordExtensions
     # Valid options:
-    # :index (optional) configure index to store data in.  default to ElasticSearchable.default_index
     # :type (optional) configue type to store data in.  default to model table name
-    # :index_options (optional) configure index properties (ex: tokenizer)
     # :mapping (optional) configure field properties for this model (ex: skip analyzer for field)
     # :if (optional) reference symbol/proc condition to only index when condition is true
     # :unless (optional) reference symbol/proc condition to skip indexing when condition is true
@@ -29,6 +27,13 @@ module ElasticSearchable
       cattr_accessor :elastic_options
       self.elastic_options = options.symbolize_keys.merge(:unless => Array.wrap(options[:unless]).push(:elasticsearch_offline?))
 
+      if self.elastic_options[:index_options]
+        ActiveSupport::Deprecation.warn ":index_options has been deprecated.  Use ElasticSearchable.index_settings instead.", caller
+      end
+      if self.elastic_options[:index]
+        ActiveSupport::Deprecation.warn ":index has been deprecated.  Use ElasticSearchable.index_name instead.", caller
+      end
+
       extend ElasticSearchable::Indexing::ClassMethods
       extend ElasticSearchable::Queries
 

data/lib/elastic_searchable/index.rb

@@ -3,57 +3,28 @@ module ElasticSearchable
     module ClassMethods
       # delete all documents of this type in the index
       # http://www.elasticsearch.com/docs/elasticsearch/rest_api/admin/indices/delete_mapping/
-      def clean_index
-        ElasticSearchable.request :delete, index_type_path
+      def delete_mapping
+        ElasticSearchable.request :delete, index_mapping_path
       end
 
       # configure the index for this type
       # http://www.elasticsearch.com/docs/elasticsearch/rest_api/admin/indices/put_mapping/
-      def update_index_mapping
-        if mapping = self.elastic_options[:mapping]
-          ElasticSearchable.request :put, index_type_path('_mapping'), :json_body => {index_type => mapping}
-        end
-      end
-
-      # create the index
-      # http://www.elasticsearch.org/guide/reference/api/admin-indices-create-index.html
-      def create_index
-        options = {}
-        options.merge! :settings => self.elastic_options[:index_options] if self.elastic_options[:index_options]
-        options.merge! :mappings => {index_type => self.elastic_options[:mapping]} if self.elastic_options[:mapping]
-        ElasticSearchable.request :put, index_path, :json_body => options
-      end
-
-      # explicitly refresh the index, making all operations performed since the last refresh
-      # available for search
-      #
-      # http://www.elasticsearch.com/docs/elasticsearch/rest_api/admin/indices/refresh/
-      def refresh_index
-        ElasticSearchable.request :post, index_path('_refresh')
-      end
-
-      # deletes the entire index
-      # http://www.elasticsearch.com/docs/elasticsearch/rest_api/admin/indices/delete_index/
-      def delete_index
-        ElasticSearchable.request :delete, index_path
+      def create_mapping
+        return unless self.elastic_options[:mapping]
+        ElasticSearchable.request :put, index_mapping_path('_mapping'), :json_body => {index_type => mapping}
       end
 
       # delete one record from the index
       # http://www.elasticsearch.com/docs/elasticsearch/rest_api/delete/
       def delete_id_from_index(id)
-        ElasticSearchable.request :delete, index_type_path(id)
+        ElasticSearchable.request :delete, index_mapping_path(id)
       rescue ElasticSearchable::ElasticError => e
         ElasticSearchable.logger.warn e
       end
 
       # helper method to generate elasticsearch url for this object type
-      def index_type_path(action = nil)
-        index_path [index_type, action].compact.join('/')
-      end
-
-      # helper method to generate elasticsearch url for this index
-      def index_path(action = nil)
-        ['', index_name, action].compact.join('/')
+      def index_mapping_path(action = nil)
+        ElasticSearchable.request_path [index_type, action].compact.join('/')
       end
 
       # reindex all records using bulk api
@@ -65,12 +36,12 @@ module ElasticSearchable
       #
       # TODO: move this to AREL relation to remove the options scope param
       def reindex(options = {})
-        self.update_index_mapping
+        self.create_mapping
         options.reverse_merge! :page => 1, :per_page => 1000
         scope = options.delete(:scope) || self
         page = options[:page]
         per_page = options[:per_page]
-        records = scope.limit(per_page).offset(per_page * (page -1)).to_a
+        records = scope.limit(per_page).offset(per_page * (page -1)).all
         while records.any? do
           ElasticSearchable.logger.debug "reindexing batch ##{page}..."
           actions = []
@@ -78,7 +49,7 @@ module ElasticSearchable
             next unless record.should_index?
             begin
               doc = ElasticSearchable.encode_json(record.as_json_for_index)
-              actions << ElasticSearchable.encode_json({:index => {'_index' => index_name, '_type' => index_type, '_id' => record.id}})
+              actions << ElasticSearchable.encode_json({:index => {'_index' => ElasticSearchable.index_name, '_type' => index_type, '_id' => record.id}})
               actions << doc
             rescue => e
               ElasticSearchable.logger.warn "Unable to bulk index record: #{record.inspect} [#{e.message}]"
@@ -92,14 +63,11 @@ module ElasticSearchable
           end
 
           page += 1
-          records = scope.limit(per_page).offset(per_page* (page-1)).to_a
+          records = scope.limit(per_page).offset(per_page* (page-1)).all
         end
       end
 
       private
-      def index_name
-        self.elastic_options[:index] || ElasticSearchable.default_index
-      end
       def index_type
         self.elastic_options[:type] || self.table_name
       end
@@ -111,15 +79,15 @@ module ElasticSearchable
       # see http://www.elasticsearch.org/guide/reference/api/index_.html
       def reindex(lifecycle = nil)
         query = {}
-        response = ElasticSearchable.request :put, self.class.index_type_path(id), :query => query, :json_body => as_json_for_index
+        query[:percolate] = "*" if _percolate_callbacks.any?
+        response = ElasticSearchable.request :put, self.class.index_mapping_path(self.id), :query => query, :json_body => self.as_json_for_index
 
         self.index_lifecycle = lifecycle ? lifecycle.to_sym : nil
-        run_callbacks :index
+        _run_index_callbacks
 
-        self.percolate if _percolate_callbacks.any?
-        run_callbacks :percolate if percolations.any?
+        self.percolations = response['matches'] || []
+        _run_percolate_callbacks if self.percolations.any?
       end
-
       # document to index in elasticsearch
       def as_json_for_index
         original_include_root_in_json = self.class.include_root_in_json
@@ -128,12 +96,10 @@ module ElasticSearchable
       ensure
         self.class.include_root_in_json = original_include_root_in_json
       end
-
       def should_index?
         [self.class.elastic_options[:if]].flatten.compact.all? { |m| evaluate_elastic_condition(m) } &&
         ![self.class.elastic_options[:unless]].flatten.compact.any? { |m| evaluate_elastic_condition(m) }
       end
-
       # percolate this object to see what registered searches match
       # can be done on transient/non-persisted objects!
       # can be done automatically when indexing using :percolate => true config option
@@ -141,8 +107,8 @@ module ElasticSearchable
       def percolate(percolator_query = nil)
         body = {:doc => self.as_json_for_index}
         body[:query] = percolator_query if percolator_query
-        response = ElasticSearchable.request :get, self.class.index_type_path('_percolate'), :json_body => body
-        self.percolations = (response['matches'] || []).map { |match| match['_id'] }
+        response = ElasticSearchable.request :get, self.class.index_mapping_path('_percolate'), :json_body => body
+        self.percolations = response['matches'] || []
         self.percolations
       end
 

data/lib/elastic_searchable/queries.rb

@@ -1,7 +1,6 @@
 module ElasticSearchable
   module Queries
     PER_PAGE_DEFAULT = 20
-    MAX_RETRIES = 5
 
     def per_page
       PER_PAGE_DEFAULT
@@ -15,8 +14,8 @@ module ElasticSearchable
     # http://www.elasticsearch.com/docs/elasticsearch/rest_api/search/
     def search(query, options = {})
       page = (options.delete(:page) || 1).to_i
-      size = (options[:size] ||= per_page_for_search(options))
       options[:fields] ||= '_id'
+      options[:size] ||= per_page_for_search(options)
       options[:from] ||= options[:size] * (page - 1)
       if query.is_a?(Hash)
         options[:query] = query
@@ -36,51 +35,19 @@ module ElasticSearchable
         query[:sort] = sort
       end
 
-      ids_to_delete = []
-      results = []
-      ids = []
-      hits_total = nil
-      retries = MAX_RETRIES
+      response = ElasticSearchable.request :get, index_mapping_path('_search'), :query => query, :json_body => options
+      hits = response['hits']
+      ids = hits['hits'].collect {|h| h['_id'].to_i }
+      results = self.find(ids).sort_by {|result| ids.index(result.id) }
 
-      loop do
-        response = ElasticSearchable.request :get, index_type_path('_search'), :query => query, :json_body => options
-        hits = response['hits']
-        hits_total ||= hits['total'].to_i
-        new_ids = collect_hit_ids(hits)
-        new_results = collect_result_records(new_ids, hits)
-        ids += new_ids
-        results += new_results
-
-        break if results.size >= ids.size || retries <= 0
-
-        retries -= 1
-
-        options[:from] = options[:from] + options[:size]
-        options[:size] = ids.size - results.size
-
-        ids_to_delete += (new_ids - new_results.map(&:id))
-        ids -= ids_to_delete
-      end
-
-      ids_to_delete.each do |id|
-        delete_id_from_index_backgrounded id
+      results.each do |result|
+        result.instance_variable_set '@hit', hits['hits'][ids.index(result.id)]
       end
 
-      ElasticSearchable::Paginator.handler.new(results, page, size, hits_total - ids_to_delete.size)
+      ElasticSearchable::Paginator.handler.new(results, page, options[:size], hits['total'])
     end
 
     private
-
-    def collect_hit_ids(hits)
-      hits['hits'].collect {|h| h['_id'].to_i }
-    end
-
-    def collect_result_records(ids, hits)
-      self.where(:id => ids).to_a.sort_by{ |result| ids.index(result.id) }.each do |result|
-        result.instance_variable_set '@hit', hits['hits'][ids.index(result.id)]
-      end
-    end
-
     # determine the number of search results per page
     # supports will_paginate configuration by using:
     # Model.per_page

data/lib/elastic_searchable/version.rb

@@ -1,3 +1,3 @@
 module ElasticSearchable
-  VERSION = '1.6'
+  VERSION = '2.0.0'
 end

data/lib/elastic_searchable.rb

@@ -4,14 +4,13 @@ require 'logger'
 require 'elastic_searchable/active_record_extensions'
 
 module ElasticSearchable
-  DEFAULT_INDEX = 'elastic_searchable'
   include HTTParty
   format :json
   base_uri 'localhost:9200'
 
   class ElasticError < StandardError; end
   class << self
-    attr_accessor :logger, :default_index, :offline
+    attr_accessor :logger, :index_name, :index_settings, :offline
 
     # execute a block of work without reindexing objects
     def offline(&block)
@@ -34,7 +33,7 @@ module ElasticSearchable
     # ElasticSearchable.debug_output outputs all http traffic to console
     def request(method, url, options = {})
       options.merge! :headers => {'Content-Type' => 'application/json'}
-      options.merge! :body => ElasticSearchable.encode_json(options.delete(:json_body)) if options[:json_body]
+      options.merge! :body => self.encode_json(options.delete(:json_body)) if options[:json_body]
 
       response = self.send(method, url, options)
       logger.debug "elasticsearch request: #{method} #{url} #{"took #{response['took']}ms" if response['took']}"
@@ -47,6 +46,33 @@ module ElasticSearchable
       string.to_s.gsub(/([\(\)\[\]\{\}\?\\\"!\^\+\-\*:~])/,'\\\\\1')
     end
 
+    # create the index
+    # http://www.elasticsearch.org/guide/reference/api/admin-indices-create-index.html
+    def create_index
+      options = {}
+      options[:settings] = self.index_settings if self.index_settings
+      self.request :put, self.request_path, :json_body => options
+    end
+
+    # explicitly refresh the index, making all operations performed since the last refresh
+    # available for search
+    #
+    # http://www.elasticsearch.com/docs/elasticsearch/rest_api/admin/indices/refresh/
+    def refresh_index
+      self.request :post, self.request_path('_refresh')
+    end
+
+    # deletes the entire index
+    # http://www.elasticsearch.com/docs/elasticsearch/rest_api/admin/indices/delete_index/
+    def delete_index
+      self.request :delete, self.request_path
+    end
+
+    # helper method to generate elasticsearch url for this index
+    def request_path(action = nil)
+      ['', index_name, action].compact.join('/')
+    end
+
     private
     # all elasticsearch rest calls return a json response when an error occurs.  ex:
     # {error: 'an error occurred' }
@@ -63,5 +89,5 @@ ElasticSearchable.logger.level = Logger::INFO
 
 # configure default index to be elastic_searchable
 # one index can hold many object 'types'
-ElasticSearchable.default_index = ElasticSearchable::DEFAULT_INDEX
+ElasticSearchable.index_name = 'elastic_searchable'
 

data/test/database.yml

@@ -0,0 +1,3 @@
+sqlite:
+  adapter: sqlite3
+  database: test/elastic_searchable.sqlite3

data/{spec/spec_helper.rb → test/helper.rb}

@@ -1,30 +1,24 @@
 require 'rubygems'
 require 'bundler'
-require 'yaml'
-require 'byebug'
-
 begin
-  Bundler.setup
+  Bundler.setup(:default, :development)
 rescue Bundler::BundlerError => e
   $stderr.puts e.message
   $stderr.puts "Run `bundle install` to install missing gems"
   exit e.status_code
 end
-require 'rspec/matchers'
+require 'test/unit'
+require 'shoulda'
+require 'mocha'
+require 'pry'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
-
 require 'elastic_searchable'
-require 'prefactory'
-
-SINGLE_NODE_CLUSTER_CONFIG = {
-  'number_of_replicas' => 0,
-  'number_of_shards' => 1
-}
-
-Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
+require 'setup_database'
 
-RSpec.configure do |config|
-  config.include Prefactory
+class Test::Unit::TestCase
+  def delete_index
+    ElasticSearchable.delete_index rescue nil
+  end
 end