supernova 0.3.10 → 0.3.11

data/VERSION

@@ -1 +1 @@
-0.3.10
+0.3.11

data/lib/supernova/solr_indexer.rb

@@ -1,9 +1,12 @@
 require "json"
+require "fileutils"
 
 class Supernova::SolrIndexer
-  attr_accessor :options, :db, :ids
+  attr_accessor :options, :db, :ids, :max_rows_to_direct_index, :local_solr
   attr_writer :index_file_path
   
+  MAX_ROWS_TO_DIRECT_INDEX = 100
+  
   include Supernova::Solr
   
   class << self
@@ -62,6 +65,7 @@ class Supernova::SolrIndexer
     options.each do |key, value|
       self.send(:"#{key}=", value) if self.respond_to?(:"#{key}=")
     end
+    self.max_rows_to_direct_index ||= MAX_ROWS_TO_DIRECT_INDEX
     self.options = options
     self.ids ||= :all
   end
@@ -143,8 +147,23 @@ class Supernova::SolrIndexer
   end
   
   def index_query(query)
-    query_db(query).each do |row|
-      yield(row) if block_given?
+    rows = query_db(query)
+    if self.max_rows_to_direct_index < rows.count
+      index_with_json_file(rows)
+    else
+      index_directly(rows)
+    end
+  end
+  
+  def index_directly(rows)
+    rows.each do |row|
+      Supernova::Solr.connection.add(row)
+    end
+    Supernova::Solr.connection.commit if rows.any?
+  end
+  
+  def index_with_json_file(rows)
+    rows.each do |row|
       write_to_file(row)
     end
     finish
@@ -187,16 +206,18 @@ class Supernova::SolrIndexer
   end
   
   def solr_url
-    Supernova::Solr.url
+    Supernova::Solr.url.present? ? Supernova::Solr.url.to_s.gsub(/\/$/, "") : nil
   end
   
   def do_index_file(options = {})
     raise "solr not configured" if solr_url.nil?
-    cmd = if options[:local]
+    cmd = if self.local_solr
       %(curl -s '#{solr_url}/update/json?commit=true\\&stream.file=#{index_file_path}')
     else
       %(cd #{File.dirname(index_file_path)} && curl -s '#{solr_url}/update/json?commit=true' --data-binary @#{File.basename(index_file_path)} -H 'Content-type:application/json')
     end
-    Kernel.send(:`, cmd)
+    out = Kernel.send(:`, cmd)
+    FileUtils.rm_f(self.index_file_path) if out.to_s.include?(%(<int name=\"status\">0</int>))
+    out
   end
 end

data/spec/integration/solr_spec.rb

@@ -31,6 +31,52 @@ describe "Solr" do
     Offer.search_scope
   end
   
+  describe "#indexing" do
+    before(:each) do
+      Supernova::Solr.truncate!
+      Supernova::Solr.connection.commit
+    end
+    
+    class OfferIndex < Supernova::SolrIndexer
+      has :user_id, :type => :integer
+      has :popularity, :type => :integer
+      
+      clazz Offer
+    end
+    
+    it "indexes all Offers without file" do
+      offer1 = Offer.create!(:user_id => 1, :popularity => 10)
+      offer2 = Offer.create!(:user_id => 2, :popularity => 20)
+      indexer = OfferIndex.new(:db => ActiveRecord::Base.connection)
+      indexer.index!
+      OfferIndex.search_scope.to_a.total_entries.should == 2
+      OfferIndex.search_scope.order("user_id desc").to_a.should == [offer2, offer1]
+      indexer.instance_variable_get("@index_file_path").should be_nil
+    end
+    
+    it "indexes with a file" do
+      offer1 = Offer.create!(:user_id => 1, :popularity => 10)
+      offer2 = Offer.create!(:user_id => 2, :popularity => 20)
+      indexer = OfferIndex.new(:db => ActiveRecord::Base.connection, :max_rows_to_direct_index => 0)
+      indexer.index!
+      indexer.instance_variable_get("@index_file_path").should_not be_nil
+      OfferIndex.search_scope.to_a.total_entries.should == 2
+      OfferIndex.search_scope.order("user_id desc").to_a.should == [offer2, offer1]
+      File.should_not be_exists(indexer.instance_variable_get("@index_file_path"))
+    end
+    
+    it "indexes with a local file" do
+      offer1 = Offer.create!(:user_id => 1, :popularity => 10)
+      offer2 = Offer.create!(:user_id => 2, :popularity => 20)
+      indexer = OfferIndex.new(:db => ActiveRecord::Base.connection, :max_rows_to_direct_index => 0, :local_solr => true)
+      indexer.index!
+      indexer.instance_variable_get("@index_file_path").should_not be_nil
+      OfferIndex.search_scope.to_a.total_entries.should == 2
+      OfferIndex.search_scope.order("user_id desc").to_a.should == [offer2, offer1]
+      File.should_not be_exists(indexer.instance_variable_get("@index_file_path"))
+    end
+  end
+  
   describe "searching" do
     it "returns the correct current_page when nil" do
       new_criteria.to_a.current_page.should == 1
@@ -60,11 +106,13 @@ describe "Solr" do
       end
     end
     
-    it "includes the returned solr_doc" do
-      new_criteria.search("text_t:Hans").to_a.first.instance_variable_get("@solr_doc").should == {
-        "id" => "offers/1", "type" => "Offer", "user_id_i" => 1, "enabled_b" => false, "text_t" => "Hans Meyer", "popularity_i" => 10, 
-        "location_p" => "47,11"
-      }
+    {
+      "id" => "offers/1", "type" => "Offer", "user_id_i" => 1, "enabled_b" => false, "text_t" => "Hans Meyer", 
+      "popularity_i" => 10, "location_p" => "47,11"
+    }.each do |key, value|
+      it "sets #{key} to #{value}" do
+        doc = new_criteria.search("text_t:Hans").to_a.first.instance_variable_get("@solr_doc")[key].should == value
+      end
     end
     
     describe "nearby search" do

data/spec/spec_helper.rb

@@ -18,7 +18,9 @@ end
 Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
 
 RSpec.configure do |config|
-  
+  config.before(:each) do
+    ActiveRecord::Base.connection.execute("TRUNCATE offers")
+  end
 end
 
 ActiveRecord::Base.establish_connection(

data/spec/supernova/solr_indexer_spec.rb

@@ -5,11 +5,11 @@ describe Supernova::SolrIndexer do
   let(:db) { double("db", :query => [to_index]) }
   let(:to_index) { { :id => 1, :title => "Some Title"} }
   let(:file_stub) { double("file").as_null_object }
+  let(:solr) { double("solr").as_null_object }
   
   let(:indexer) do
     indexer = Supernova::SolrIndexer.new
     indexer.db = db
-    Supernova::Solr.url = "http://solr.xx:9333/solr"
     indexer.stub!(:system).and_return true
     indexer
   end
@@ -17,6 +17,8 @@ describe Supernova::SolrIndexer do
   let(:custom_indexer) { indexer_clazz.new }
   
   before(:each) do
+    Supernova::Solr.url = "http://solr.xx:9333/solr"
+    Supernova::Solr.stub!(:connection).and_return solr
     indexer_clazz.has(:title, :type => :text)
     indexer_clazz.has(:artist_id, :type => :integer)
     indexer_clazz.has(:description, :type => :text)
@@ -47,6 +49,10 @@ describe Supernova::SolrIndexer do
     it "sets ids to all when nil" do
       Supernova::SolrIndexer.new.ids.should == :all
     end
+    
+    it "sets max_rows_to_direct_index to 100" do
+      Supernova::SolrIndexer.new.max_rows_to_direct_index.should == 100
+    end
   end
   
   describe "index!" do
@@ -135,24 +141,78 @@ describe Supernova::SolrIndexer do
   describe "#index_query" do
     let(:query) { %(SELECT CONCAT("user_", id) AS id, title FROM people WHERE type = 'User') }
     
-    it "executes the query" do
-      indexer.should_receive(:query_db).with(query).and_return [to_index]
+    it "calls index_with_json_file when rows > max_rows_to_direct_index" do
+      indexer.max_rows_to_direct_index = 0
+      rows = [to_index]
+      indexer.should_receive(:query_db).with(query).and_return rows
+      indexer.should_receive(:index_with_json_file).with(rows)
       indexer.index_query(query)
     end
     
-    it "calls write_to_file on all rows" do
-      rows = [double("1"), double("2")]
-      indexer.stub(:query_db).and_return rows
-      indexer.should_receive(:write_to_file).with(rows.first)
-      indexer.should_receive(:write_to_file).with(rows.at(1))
-      indexer.stub!(:finish)
+    it "calls index_directly with rows when rows = max_rows_to_direct_index" do
+      indexer.max_rows_to_direct_index = 1
+      rows = [to_index]
+      indexer.should_receive(:query_db).with(query).and_return rows
+      indexer.should_receive(:index_directly).with(rows)
       indexer.index_query(query)
     end
     
-    it "calls finish" do
-      indexer.should_receive(:finish)
-      indexer.index_query(query)
+    describe "with number of rows > max_rows_to_direct_index" do
+      before(:each) do
+        indexer.max_rows_to_direct_index = 0
+      end
+      
+      it "calls max_rows_to_direct_index" do
+        indexer.should_receive(:max_rows_to_direct_index).and_return 0
+        indexer.index_query(query)
+      end
+      
+      it "executes the query" do
+        indexer.should_receive(:query_db).with(query).and_return [to_index]
+        indexer.index_query(query)
+      end
+
+      it "calls write_to_file on all rows" do
+        rows = [double("1"), double("2")]
+        indexer.stub(:query_db).and_return rows
+        indexer.should_receive(:write_to_file).with(rows.first)
+        indexer.should_receive(:write_to_file).with(rows.at(1))
+        indexer.stub!(:finish)
+        indexer.index_query(query)
+      end
+
+      it "calls finish" do
+        indexer.should_receive(:finish)
+        indexer.index_query(query)
+      end
+    end
+  end
+  
+  describe "#index_directly" do
+    before(:each) do
+      Supernova::Solr.stub!(:connection).and_return solr
+    end
+    
+    it "calls the correct add statement" do
+      row1 = double("1")
+      row2 = double("2")
+      rows = [row1, row2]
+      solr.should_receive(:add).with(row1)
+      solr.should_receive(:add).with(row2)
+      indexer.index_directly(rows)
+    end
+    
+    it "calls commit" do
+      solr.should_receive(:commit)
+      indexer.index_directly([double("1")])
     end
+    
+    it "does not call commit when rows is empty" do
+      solr.should_not_receive(:commit)
+      indexer.index_directly([])
+    end
+    
+    it "calls a block given given"
   end
   
   describe "#index_file_path" do
@@ -239,8 +299,22 @@ describe Supernova::SolrIndexer do
     end
     
     it "calls the correct curl command" do
-      indexer.index_file_path = "/tmp/some_path.json"
-      Kernel.should_receive(:`).with("curl -s 'http://solr.xx:9333/solr/update/json?commit=true\\&stream.file=/tmp/some_path.json'")
+      indexer = Supernova::SolrIndexer.new(:index_file_path => "/tmp/some_path.json", :local_solr => true)
+      Kernel.should_receive(:`).with("curl -s 'http://solr.xx:9333/solr/update/json?commit=true\\&stream.file=/tmp/some_path.json'").and_return "<\"status\">0"
+      indexer.do_index_file(:local => true)
+    end
+    
+    it "calls rm on file" do
+      indexer = Supernova::SolrIndexer.new(:index_file_path => "/tmp/some_path.json", :local_solr => true)
+      Kernel.should_receive(:`).with("curl -s 'http://solr.xx:9333/solr/update/json?commit=true\\&stream.file=/tmp/some_path.json'").and_return %(<int name="status">0</int>)
+      FileUtils.should_receive(:rm_f).with("/tmp/some_path.json")
+      indexer.do_index_file(:local => true)
+    end
+    
+    it "does not call rm when not successful" do
+      indexer = Supernova::SolrIndexer.new(:index_file_path => "/tmp/some_path.json", :local_solr => true)
+      Kernel.should_receive(:`).with("curl -s 'http://solr.xx:9333/solr/update/json?commit=true\\&stream.file=/tmp/some_path.json'").and_return %(<int name="status">1</int>)
+      FileUtils.should_not_receive(:rm_f).with("/tmp/some_path.json")
       indexer.do_index_file(:local => true)
     end
     
@@ -492,4 +566,11 @@ describe Supernova::SolrIndexer do
       Supernova::SolrIndexer.solr_field_for_field_name_and_mapping(:artist, nil).should == "artist"
     end
   end
+
+  describe "#solr_url" do
+    it "strips slashes from defined solr url" do
+      Supernova::Solr.url = "http://solr.xx:9333/solr/"
+      indexer.solr_url.should == "http://solr.xx:9333/solr"
+    end
+  end
 end

data/supernova.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{supernova}
-  s.version = "0.3.10"
+  s.version = "0.3.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Tobias Schwab"]
-  s.date = %q{2011-06-16}
+  s.date = %q{2011-06-17}
   s.description = %q{Unified search scopes}
   s.email = %q{tobias.schwab@dynport.de}
   s.extra_rdoc_files = [

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: supernova
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 5
   prerelease: 
   segments: 
   - 0
   - 3
-  - 10
-  version: 0.3.10
+  - 11
+  version: 0.3.11
 platform: ruby
 authors: 
 - Tobias Schwab
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-16 00:00:00 Z
+date: 2011-06-17 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   requirement: &id001 !ruby/object:Gem::Requirement