escargot 0.0.2

data/lib/escargot/distributed_indexing.rb

@@ -0,0 +1,58 @@
+
+module Escargot
+
+  module DistributedIndexing
+
+    def DistributedIndexing.load_dependencies
+      require 'resque'
+    end
+
+    def DistributedIndexing.create_index_for_model(model)
+      load_dependencies
+
+      index_version = model.create_index_version
+
+      model.find_in_batches(:select => model.primary_key) do |batch|
+        Escargot.queue_backend.enqueue(IndexDocuments, model.to_s, batch.map(&:id), index_version)
+      end
+
+      Escargot.queue_backend.enqueue(DeployNewVersion, model.index_name, index_version)
+    end
+
+    class IndexDocuments
+      @queue = :indexing
+
+      def self.perform(model_name, ids, index_version)
+        model = model_name.constantize
+        model.find(:all, :conditions => {model.primary_key => ids}).each do |record|
+          record.local_index_in_elastic_search(:index => index_version)
+        end
+      end
+    end
+
+    class ReIndexDocuments
+      @queue = :nrt
+
+      def self.perform(model_name, ids)
+        model = model_name.constantize
+        ids_found = []
+        model.find(:all, :conditions => {:id => ids}).each do |record|
+          record.local_index_in_elastic_search
+          ids_found << record.id
+        end
+
+        (ids - ids_found).each do |id|
+          model.delete_id_from_index(id)
+        end
+      end
+    end
+
+    class DeployNewVersion
+      @queue = :indexing
+      def self.perform(index, index_version)
+        $elastic_search_client.deploy_index_version(index, index_version)
+      end
+    end
+  end
+
+end

data/lib/escargot/elasticsearch_ex.rb

@@ -0,0 +1,63 @@
+module Escargot
+  
+  module AdminIndexVersions
+
+    # creates an index to store a new index version. Returns its name
+    def create_index_version(index, create_options)
+      index_with_timestamp = "#{index}_#{Time.now.to_f}"
+      $elastic_search_client.create_index(index_with_timestamp, create_options)
+      return index_with_timestamp
+    end
+    
+    # returns the full index name of the current version for this index
+    def current_index_version(index)
+      $elastic_search_client.index_status(index)["indices"].keys.first rescue nil
+    end
+    
+    # "deploys" a new version as the current one
+    def deploy_index_version(index, new_version)
+      $elastic_search_client.refresh(new_version)
+      if current_version = current_index_version(index)
+        $elastic_search_client.alias_index(
+          :add => {new_version => index}, 
+          :remove => {current_version => index}
+        )
+      else
+        $elastic_search_client.alias_index(:add => {new_version => index})
+      end
+    end
+
+    # deletes all index versions older than the current one
+    def prune_index_versions(index)
+      current_version = current_index_version(index)
+      return unless current_version
+      old_versions = index_versions(index).select{|version| version_timestamp(version) < version_timestamp(current_version)}
+      old_versions.each do |version|
+        $elastic_search_client.delete_index(version)
+      end
+    end
+
+    # lists all current, old, an in-progress versions for this index
+    def index_versions(index)
+      $elastic_search_client.index_status()["indices"].keys.grep(/^#{index}_/)
+    end
+    
+    private
+      def version_timestamp(version)
+        version.gsub(/^.*_/, "").to_f
+      end
+    
+  end
+
+  module HitExtensions
+    def to_activerecord
+      model_class = _type.gsub(/-/,'/').classify.constantize
+      begin
+        model_class.find(id) 
+      rescue ActiveRecord::RecordNotFound
+        nil
+      end
+    end
+  end
+
+end

data/lib/escargot/local_indexing.rb

@@ -0,0 +1,19 @@
+module Escargot
+  
+  module LocalIndexing
+    def LocalIndexing.create_index_for_model(model)
+      model = model.constantize if model.kind_of?(String)
+
+      index_version = model.create_index_version
+
+      model.find_in_batches do |batch|
+        batch.each do |record|
+          record.local_index_in_elastic_search(:index => index_version)
+        end
+      end
+
+      $elastic_search_client.deploy_index_version(model.index_name, index_version)
+    end
+  end
+
+end

data/lib/escargot/queue_backend/base.rb

@@ -0,0 +1,7 @@
+module Escargot
+  module QueueBackend
+    # a generic queue 
+    class Base
+    end
+  end
+end

data/lib/escargot/queue_backend/resque.rb

@@ -0,0 +1,9 @@
+module Escargot
+  module QueueBackend
+    class Rescue < Base
+      def enqueue(classname, *arguments)
+        Resque.enqueue(classname, *arguments)
+      end
+    end
+  end
+end

data/lib/escargot/version.rb

@@ -0,0 +1,3 @@
+module Escargot
+  VERSION = "0.0.2"
+end

data/lib/escargot.rb

@@ -0,0 +1,92 @@
+# Escargot
+require 'elasticsearch'
+require 'escargot/activerecord_ex'
+require 'escargot/elasticsearch_ex'
+require 'escargot/local_indexing'
+require 'escargot/distributed_indexing'
+require 'escargot/queue_backend/base'
+require 'escargot/queue_backend/resque'
+
+
+module Escargot
+  def self.register_model(model)
+    return unless model.table_exists?
+    @indexed_models ||= []
+    @indexed_models.delete(model) if @indexed_models.include?(model)
+    @indexed_models << model
+  end
+
+  def self.indexed_models
+    @indexed_models || []
+  end
+
+  def self.queue_backend
+    @queue ||= Escargot::QueueBackend::Rescue.new
+  end
+  
+  def self.flush_all_indexed_models
+    @indexed_models = []
+  end
+
+  # search_hits returns a raw ElasticSearch::Api::Hits object for the search results
+  # see #search for the valid options
+  def self.search_hits(query, options = {}, call_by_instance_method = false)
+    unless call_by_instance_method
+      if (options[:classes])
+        models = Array(options[:classes])
+      else
+        register_all_models
+        models = @indexed_models
+      end
+      options = options.merge({:index => models.map(&:index_name).join(',')})
+    end
+    
+    if query.kind_of?(Hash)
+      query = {:query => query}
+    end
+    $elastic_search_client.search(query, options)
+  end
+
+  # search returns a will_paginate collection of ActiveRecord objects for the search results
+  #
+  # see ElasticSearch::Api::Index#search for the full list of valid options
+  #
+  # note that the collection may include nils if ElasticSearch returns a result hit for a
+  # record that has been deleted on the database  
+  def self.search(query, options = {}, call_by_instance_method = false)
+    hits = Escargot.search_hits(query, options, call_by_instance_method)
+    hits_ar = hits.map{|hit| hit.to_activerecord}
+    results = WillPaginate::Collection.new(hits.current_page, hits.per_page, hits.total_entries)
+    results.replace(hits_ar)
+    results
+  end
+
+  # counts the number of results for this query.
+  def self.search_count(query = "*", options = {}, call_by_instance_method = false)
+    unless call_by_instance_method
+      if (options[:classes])
+        models = Array(options[:classes])
+      else
+        register_all_models
+        models = @indexed_models
+      end
+      options = options.merge({:index => models.map(&:index_name).join(',')})
+    end
+    $elastic_search_client.count(query, options)
+  end
+
+  private
+    def self.register_all_models
+      models = []
+      # Search all Models in the application Rails
+      Dir[File.join("#{RAILS_ROOT}/app/models".split(/\\/), "**", "*.rb")].each do |file|
+        model = file.gsub(/#{RAILS_ROOT}\/app\/models\/(.*?)\.rb/,'\1').classify.constantize
+        unless models.include?(model)
+          require file
+        end
+        models << model
+      end
+    end
+
+
+end

data/lib/tasks/escargot.rake

@@ -0,0 +1,50 @@
+# desc "Explaining what the task does"
+# task :elastic_rails do
+#   # Task goes here
+# end
+
+namespace :escargot do
+  desc "indexes the models"
+  task :index, :models, :needs => [:environment, :load_all_models] do |t, args|
+    each_indexed_model(args) do |model|
+      puts "Indexing #{model}"
+      Escargot::LocalIndexing.create_index_for_model(model)
+    end
+  end
+
+  desc "indexes the models"
+  task :distributed_index, :models, :needs => [:environment, :load_all_models] do |t, args|
+    each_indexed_model(args) do |model|
+      puts "Indexing #{model}"
+      Escargot::DistributedIndexing.create_index_for_model(model)
+    end
+  end
+  
+  desc "prunes old index versions for this models"
+  task :prune_versions, :models, :needs => [:environment, :load_all_models] do |t, args|
+    each_indexed_model(args) do |model|
+      $elastic_search_client.prune_index_versions(model.index_name)
+    end
+  end
+  
+  task :load_all_models do
+    models = ActiveRecord::Base.send(:subclasses)
+    Dir["#{Rails.root}/app/models/*.rb", "#{Rails.root}/app/models/*/*.rb"].each do |file|
+      model = File.basename(file, ".*").classify
+      unless models.include?(model)
+        require file
+      end
+      models << model
+    end
+  end
+  
+  private
+    def each_indexed_model(args)
+      if args[:models]
+        models = args[:models].split(",").map{|m| m.classify.constantize}
+      else
+        models = Escargot.indexed_models
+      end
+      models.each{|m| yield m}
+    end
+end

data/rails/init.rb

@@ -0,0 +1,21 @@
+require 'escargot'
+
+ActiveRecord::Base.class_eval do
+  include Escargot::ActiveRecordExtensions
+end
+
+ElasticSearch::Api::Hit.class_eval do
+  include Escargot::HitExtensions
+end
+
+ElasticSearch::Client.class_eval do
+  include Escargot::AdminIndexVersions
+end
+
+unless File.exists?(Rails.root + "/config/elasticsearch.yml")
+  Rails.logger.warn "No config/elastic_search.yaml file found, connecting to localhost:9200"
+  $elastic_search_client = ElasticSearch.new("localhost:9200")
+else
+  config = YAML.load_file(RAILS_ROOT + "/config/elasticsearch.yml")
+  $elastic_search_client = ElasticSearch.new(config["host"] + ":" + config["port"].to_s, :timeout => 20)
+end

data/test/admin_index_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+
+class AdminIndexTest < Test::Unit::TestCase
+  load_schema
+
+  class User < ActiveRecord::Base
+    elastic_index
+  end
+
+  def test_prune_index
+    index = User.index_name
+    3.times.each do
+      Escargot::LocalIndexing.create_index_for_model(User)
+    end
+    User.refresh_index
+    sleep(1)
+    assert $elastic_search_client.index_versions(index).size > 1
+    assert $elastic_search_client.index_versions(index).include? $elastic_search_client.current_index_version(index)
+    $elastic_search_client.prune_index_versions(index)
+    assert $elastic_search_client.index_versions(index).size == 1
+  end
+
+  def teardown
+    User.delete_index
+  end
+end

data/test/database.yml

@@ -0,0 +1,21 @@
+sqlite:
+  :adapter: sqlite
+  :database: vendor/plugins/escargot/test/escargot_plugins.sqlite.db
+
+sqlite3:
+  :adapter: sqlite3
+  :database: vendor/plugins/escargot/test/escargot_plugins.sqlite3.db
+
+postgresql:
+  :adapter: postgresql
+  :username: postgres
+  :password: postgres
+  :database: escargot
+  :min_messages: ERROR
+
+mysql:
+  :adapter: mysql
+  :host: localhost
+  :username: root
+  :password: password
+  :database: escargot

data/test/define_index_test.rb

@@ -0,0 +1,21 @@
+require 'test_helper'
+
+class ElasticIndexTest < Test::Unit::TestCase
+  
+  class User < ActiveRecord::Base
+    elastic_index
+  end
+
+  def test_index_name
+    assert_equal User.index_name, 'elastic_index_test-user'
+  end
+  
+  def test_search_method_present
+    assert User.respond_to?(:search)
+  end
+  
+  def test_registered_as_indexed_model
+    Escargot.indexed_models.include?(User)
+  end
+  
+end

data/test/distributed_index_creation_test.rb

@@ -0,0 +1,97 @@
+# coding: utf-8
+require 'test_helper'
+
+# tests the behaviour of the index creation tasks that run in the distributed mode
+# without real time support
+
+class DistributedIndexCreation < Test::Unit::TestCase
+
+  load_schema
+  resque_available
+  class User < ActiveRecord::Base
+    elastic_index :updates => false
+  end
+  
+  class LegacyUser < ActiveRecord::Base
+    set_primary_key :legacy_id
+    elastic_index :updates => false
+  end
+
+  def test_distributed_indexing
+    User.delete_all
+    User.delete_index
+
+    User.new(:name => 'John the Long').save!
+    User.new(:name => 'Peter the Young').save!
+    User.new(:name => 'Peter the Old').save!
+    User.new(:name => 'Bob the Skinny').save!
+    User.new(:name => 'Jamie the Flying Machine').save!
+
+    Escargot::DistributedIndexing.create_index_for_model(User)
+    Resque.run!
+    User.refresh_index
+
+    results = User.search("peter")
+    
+    assert_equal results.total_entries, 2
+    assert_equal [results.first.name, results.second.name].sort, ['Peter the Old', 'Peter the Young']
+
+    results = User.search("LONG or SKINNY")
+    assert_equal results.total_entries, 2
+
+    results = User.search("*")
+    assert_equal results.total_entries, 5
+  end
+  
+  # minimal test to ensure that models with a non-default primary key work
+  def test_legacy_model
+    LegacyUser.delete_all
+    LegacyUser.delete_index
+
+    LegacyUser.new(:name => 'John the Long').save!
+    LegacyUser.new(:name => 'Peter the Young').save!
+    LegacyUser.new(:name => 'Peter the Old').save!
+    LegacyUser.new(:name => 'Bob the Skinny').save!
+    LegacyUser.new(:name => 'Jamie the Flying Machine').save!
+    
+    Escargot::DistributedIndexing.create_index_for_model(LegacyUser)
+    Resque.run!
+    LegacyUser.refresh_index
+    
+    results = LegacyUser.search("peter")
+    assert_equal results.total_entries, 2
+  end
+
+  def test_index_rotation
+    # create a first version of the index
+    User.delete_all
+    User.delete_index
+    
+    User.create(:name => 'John the Long')
+    User.create(:name => 'Peter the Fat')
+    User.create(:name => 'Bob the Skinny')
+    User.create(:name => 'Jamie the Flying Machine')
+
+    Escargot::DistributedIndexing.create_index_for_model(User)
+    Resque.run!
+    
+    # create a second version of the index
+
+    User.find(:first).destroy
+    User.find(:first).destroy
+
+    Escargot::DistributedIndexing.create_index_for_model(User)
+    Resque.run!
+    User.refresh_index
+
+    # check that there are no traces of the older index
+    assert_equal User.search_count, 2
+  end
+
+  def teardown
+    User.delete_all
+    User.delete_index
+    LegacyUser.delete_all
+    LegacyUser.delete_index
+  end
+end

data/test/index_options_test.rb

@@ -0,0 +1,34 @@
+# coding: utf-8
+require 'test_helper'
+
+class CustomIndexOptions < Test::Unit::TestCase
+  load_schema
+
+  class User < ActiveRecord::Base
+    elastic_index(
+      :updates => false, 
+      :index_options => {
+        "analysis.analyzer.default.tokenizer" => 'standard',
+        "analysis.analyzer.default.filter" => ["standard", "lowercase", "stop", "asciifolding"]
+      }
+    )
+  end
+  
+  def test_asciifolding_option
+    User.delete_all
+    User.delete_index
+    
+    User.create(:name => "Pedrín el Joven")
+    User.create(:name => 'Pedro el Viejo')
+    User.create(:name => 'Roberto el Delgado')
+    User.create(:name => 'Jamie la Máquina Voladora')
+
+    Escargot::LocalIndexing.create_index_for_model(User)
+    
+    results = User.search("pedrin")
+    assert_equal results.total_entries, 1
+    
+    User.delete_all
+    User.delete_index
+  end
+end

data/test/indexed_content_test.rb

@@ -0,0 +1,39 @@
+# tests the Near Real Time support 
+
+require 'test_helper'
+
+# tests the behaviour of the index creation tasks that run locally (in the "simple" mode)
+
+class IndexedContentTest < Test::Unit::TestCase
+  load_schema
+  
+  class User < ActiveRecord::Base
+    elastic_index
+    
+    def indexed_json_document
+      to_json(:only => :name, :methods => :foo)
+    end
+    
+    def foo
+      "FOO!"
+    end
+  end
+
+  def test_indexed_content
+    User.delete_index
+    User.delete_all
+    
+    User.create(:name => 'Tim the Wise')
+    User.create(:name => 'Peter the Young')
+    User.create(:name => 'Peter the Old')    
+    User.refresh_index
+    
+    assert_equal User.search_count("Peter AND foo:FOO"), 2
+  end
+  
+  def teardown
+    User.delete_all
+    User.delete_index
+  end
+  
+end

data/test/local_index_creation_test.rb

@@ -0,0 +1,51 @@
+require 'test_helper'
+
+# tests the behaviour of the index creation tasks that run locally (in the "simple" mode)
+# without real time support
+
+class LocalIndexCreation < Test::Unit::TestCase
+  load_schema
+  
+  class User < ActiveRecord::Base
+    elastic_index :updates => false
+  end
+
+  def test_fast_index_creation
+    User.delete_all
+    User.delete_index
+    
+    10.times do
+      Escargot::LocalIndexing.create_index_for_model(User)
+    end
+    sleep(1)
+  end
+  
+  def test_indexing_rotation
+    # create a first version of the index
+    User.delete_all
+    User.create(:name => 'John the Long')
+    User.create(:name => 'Peter the Fat')
+    User.create(:name => 'Bob the Skinny')
+    User.create(:name => 'Jamie the Flying Machine')
+  
+    Escargot::LocalIndexing.create_index_for_model(User)
+    
+    # create a second version of the index
+    
+    User.find(:first).destroy
+    User.find(:first).destroy
+    
+    Escargot::LocalIndexing.create_index_for_model(User)
+    
+    # check that there are no trace of the older index
+    
+    results = User.search("*")
+    assert_equal results.total_entries, 2
+  end
+  
+  def teardown
+    User.delete_all
+    User.delete_index
+  end
+  
+end

data/test/mappings_test.rb

@@ -0,0 +1,38 @@
+# coding: utf-8
+require 'test_helper'
+
+class Mappings < Test::Unit::TestCase
+  load_schema
+
+  class User < ActiveRecord::Base
+    elastic_index(
+      :mapping => {
+        :properties => {
+          :name => {:type => "string", :index => "not_analyzed"}
+        }
+      }
+    )
+  end
+  
+  def test_not_analyzed_property
+    
+    User.delete_all
+    User.delete_index
+    Escargot::LocalIndexing.create_index_for_model(User)
+    
+    User.create(:name => 'Pedrín el Joven')
+    User.create(:name => 'Pedro el Viejo')
+    User.create(:name => 'Roberto el Delgado')
+    User.create(:name => 'Jamie la Máquina Voladora')
+
+    User.refresh_index
+
+    assert_equal User.search_count('name=pedro'), 0
+    assert_equal User.search_count('name="Pedro el Viejo"'), 1
+  end
+  
+  def teardown
+    User.delete_all
+    User.delete_index
+  end
+end