supernova 0.4.15 → 0.4.16

data/VERSION

@@ -1 +1 @@
-0.4.15
+0.4.16

data/lib/supernova/solr_indexer.rb

@@ -77,7 +77,9 @@ class Supernova::SolrIndexer
       response = yield if block_given?
     end
     if @debug == true
-      puts "%s: %s" % [Time.now.iso8601(3), message.gsub("%TIME%", "%.3f" % time)]
+      message.gsub!("%COUNT%", response.count.to_s) if message.include?("%COUNT%") && response.respond_to?(:count)
+      message.gsub!("%TIME%", "%.3f" % time)   if message.include?("%TIME%")
+      puts "%s: %s" % [Time.now.iso8601(3), message]
     end
     response
   end
@@ -104,14 +106,18 @@ class Supernova::SolrIndexer
   end
   
   def map_hash_keys_to_solr(hash)
-    hash["indexed_at_dt"] = Time.now.utc.iso8601
+    @indexed_at ||= Time.now.utc.iso8601.to_s
+    hash["indexed_at_dt"] = @indexed_at
     hash["id_s"] = [self.class.table_name, hash["id"]].compact.join("/") if hash["id"]
     self.class.field_definitions.each do |field, options|
       if hash.has_key?(field.to_s)
         value = hash.delete(field.to_s)
-        if options[:type] == :date 
-          value = Time.utc(value.year, value.month, value.day) if value.is_a?(Date)
-          value = value.utc.iso8601 if value
+        if options[:type] == :date
+          if value.is_a?(Date)
+            value = "#{value}T00:00:00Z" 
+          elsif value.respond_to?(:utc)
+            value = value.utc.iso8601
+          end
         end
         hash["#{field}_#{self.class.suffix_from_type(options[:type])}"] = value
       end
@@ -191,19 +197,14 @@ class Supernova::SolrIndexer
   end
   
   def rows(query = nil)
-    query_db(query || query_to_index)
-  end
-  
-  def solr_rows_to_index_for_query(query)
-    query_db(query).map do |row|
-      map_for_solr(row)
+    debug "fetched rows in %TIME%" do
+      query_db(query || query_to_index)
     end
   end
   
-  def index_query(query)
-    debug "getting rows for #{query[0,100]}"
-    rows = debug "got all rows in %TIME%" do
-      solr_rows_to_index_for_query(query)
+  def index_rows(rows)
+    debug "mapped %COUNT% rows to solr in %TIME%" do
+      rows.map! { |r| map_for_solr(r) }
     end
     if self.max_rows_to_direct_index < rows.count
       debug "indexed #{rows.length} rows with json in %TIME%" do
@@ -216,6 +217,17 @@ class Supernova::SolrIndexer
     end
   end
   
+  def solr_rows_to_index_for_query(query)
+    query_db(query).map do |row|
+      map_for_solr(row)
+    end
+  end
+  
+  def index_query(query)
+    debug "getting rows for #{query[0,100]}"
+    index_rows(query_db(query))
+  end
+  
   def index_directly(rows)
     rows.each do |row|
       row = Supernova::Solr.connection.add(row)
@@ -279,8 +291,12 @@ class Supernova::SolrIndexer
     else
       %(cd #{File.dirname(index_file_path)} && curl -s '#{solr_url}/update/json?commit=true' --data-binary @#{File.basename(index_file_path)} -H 'Content-type:application/json')
     end
+    debug "run command: #{cmd}"
     out = Kernel.send(:`, cmd)
-    raise "unable to index #{index_file_path}: #{out}" if !out.to_s.include?(%(<int name=\"status\">0</int>))
+    if !out.to_s.include?(%(<int name=\"status\">0</int>))
+      debug "ERROR: #{out}"
+      raise "unable to index #{index_file_path}: #{out}" 
+    end
     FileUtils.rm_f(self.index_file_path)
     out
   end

data/spec/spec_helper.rb

@@ -8,6 +8,7 @@ require "fileutils"
 require "ruby-debug"
 require "geokit"
 require "active_record"
+PROJECT_ROOT = Pathname.new(File.expand_path("..", File.dirname(__FILE__)))
 
 if defined?(Debugger) && Debugger.respond_to?(:settings)
   Debugger.settings[:autolist] = 1
@@ -24,6 +25,7 @@ RSpec.configure do |config|
   end
 end
 
+
 ActiveRecord::Base.establish_connection(
   :adapter => "mysql2",
   :host => "localhost", 
@@ -32,7 +34,7 @@ ActiveRecord::Base.establish_connection(
   :encoding => "utf8"
 )
 
-PROJECT_ROOT = Pathname.new(File.expand_path("..", File.dirname(__FILE__)))
+
 FileUtils.mkdir_p(PROJECT_ROOT.join("log"))
 
 ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS offers")

data/spec/supernova/solr_indexer_spec.rb

@@ -225,6 +225,14 @@ describe Supernova::SolrIndexer do
         112
       end
     end
+    
+    it "replaces %COUNT% when responding to .count" do
+      index = CustomSolrIndex.new(:debug => true)
+      index.should_receive(:puts).with(/indexed 2/)
+      index.debug "indexed %COUNT%" do
+        [1, 2]
+      end
+    end
   end
   
   describe "#map_hash_keys_to_solr" do
@@ -331,59 +339,64 @@ describe Supernova::SolrIndexer do
     end
   end
   
-  describe "#index_query" do
-    let(:query) { %(SELECT CONCAT("user_", id) AS id, title FROM people WHERE type = 'User') }
+  describe "#index_rows" do
+    let(:row1) { double("row1") }
+    let(:row2) { double("row2") }
+    let(:mapped1) { double("mapped 1") }
+    let(:mapped2) { double("mapped 2") }
     
-    it "calls solr_rows_to_index_for_query with query" do
-      result = []
-      indexer.should_receive(:solr_rows_to_index_for_query).with(query).and_return(result)
-      indexer.index_query(query)
+    before(:each) do
+      custom_indexer.stub(:map_for_solr).with(row1).and_return(mapped1)
+      custom_indexer.stub(:map_for_solr).with(row2).and_return(mapped2)
     end
     
-    it "calls index_with_json_file when rows > max_rows_to_direct_index" do
-      indexer.max_rows_to_direct_index = 0
-      rows = [to_index]
-      indexer.should_receive(:query_db).with(query).and_return rows
-      indexer.should_receive(:index_with_json_file).with(rows)
-      indexer.index_query(query)
+    it "is callable" do
+      custom_indexer.index_rows([])
+    end
+    
+    it "calls map_for_solr on all rows" do
+      custom_indexer.should_receive(:map_for_solr).with(row1).and_return(mapped1)
+      custom_indexer.should_receive(:map_for_solr).with(row2).and_return(mapped2)
+      custom_indexer.index_rows([row1, row2])
+    end
+    
+    it "calls map_directly when number of rows < max_rows_to_direct_index" do
+      custom_indexer.should_receive(:max_rows_to_direct_index).and_return 100
+      custom_indexer.should_receive(:index_directly).with([mapped1, mapped2])
+      custom_indexer.index_rows([row1, row2])
+    end
+    
+    it "calls map_directly when number of rows < max_rows_to_direct_index" do
+      custom_indexer.should_receive(:max_rows_to_direct_index).and_return 1
+      custom_indexer.should_receive(:index_with_json_file).with([mapped1, mapped2])
+      custom_indexer.index_rows([row1, row2])
     end
+  end
+  
+  describe "#index_query" do
+    let(:query) { %(SELECT CONCAT("user_", id) AS id, title FROM people WHERE type = 'User') }
     
-    it "calls index_directly with rows when rows = max_rows_to_direct_index" do
-      indexer.max_rows_to_direct_index = 1
+    it "calls index_rows with result of query" do
       rows = [to_index]
       indexer.should_receive(:query_db).with(query).and_return rows
-      indexer.should_receive(:index_directly).with(rows)
+      indexer.should_receive(:index_rows).with(rows)
       indexer.index_query(query)
     end
+  end
+  
+  describe "#index_with_json_file" do
+    let(:rows) { [{ "b" => 2 }, { "a" => 1 }] }
     
-    describe "with number of rows > max_rows_to_direct_index" do
-      before(:each) do
-        indexer.max_rows_to_direct_index = 0
-      end
-      
-      it "calls max_rows_to_direct_index" do
-        indexer.should_receive(:max_rows_to_direct_index).and_return 0
-        indexer.index_query(query)
-      end
-      
-      it "executes the query" do
-        indexer.should_receive(:query_db).with(query).and_return [to_index]
-        indexer.index_query(query)
-      end
-
-      it "calls write_to_file on all rows" do
-        rows = [{ "b" => 2 }, { "a" => 1 }]
-        indexer.stub(:query_db).and_return rows
-        indexer.should_receive(:write_to_file).with(rows.first)
-        indexer.should_receive(:write_to_file).with(rows.at(1))
-        indexer.stub!(:finish)
-        indexer.index_query(query)
-      end
+    it "calls write_to_file on all rows" do
+      indexer.should_receive(:write_to_file).with(rows.first)
+      indexer.should_receive(:write_to_file).with(rows.at(1))
+      indexer.stub!(:finish)
+      indexer.index_with_json_file(rows)
+    end
 
-      it "calls finish" do
-        indexer.should_receive(:finish)
-        indexer.index_query(query)
-      end
+    it "calls finish" do
+      indexer.should_receive(:finish)
+      indexer.index_with_json_file(rows)
     end
   end
   
@@ -434,18 +447,25 @@ describe Supernova::SolrIndexer do
       
       it "writes the opening brackets and the first line" do
         file_stub.should_receive(:puts).with("\{")
-        file_stub.should_receive(:print).with("\"add\":{\"doc\":{\"title\":\"Some Title\",\"id\":1}}")
+        file_stub.should_receive(:print).with do |str|
+          str.should include("add")
+          str.should include("\"title\":\"Some Title\"")
+          str.should include("\"id\":1")
+        end
         indexer.write_to_file(to_index)
       end
       
       it "only write fields which are not null" do
-        file_stub.should_receive(:print).with("\"add\":{\"doc\":{\"title\":\"Some Title\",\"id\":1}}")
+        file_stub.stub(:print)
+        file_stub.should_not_receive(:print).with do |str|
+          str.include?("text")
+        end
         indexer.write_to_file(to_index.merge(:text => nil))
       end
       
       it "separates the first and the second line" do
         file_stub.should_receive(:puts).with("\{")
-        file_stub.should_receive(:print).with("\"add\":{\"doc\":{\"title\":\"Some Title\",\"id\":1}}")
+        file_stub.should_receive(:print).with(/\"add\":{\"doc\"/)
         file_stub.should_receive(:print).with(%(,\n"add":{"doc":{"id":2}}))
         indexer.write_to_file(to_index)
         indexer.write_to_file({:id => 2})

data/supernova.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{supernova}
-  s.version = "0.4.15"
+  s.version = "0.4.16"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Tobias Schwab"]
-  s.date = %q{2011-07-28}
+  s.date = %q{2011-08-02}
   s.description = %q{Unified search scopes}
   s.email = %q{tobias.schwab@dynport.de}
   s.executables = ["start_solr"]

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: supernova
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 47
   prerelease: 
   segments: 
   - 0
   - 4
-  - 15
-  version: 0.4.15
+  - 16
+  version: 0.4.16
 platform: ruby
 authors: 
 - Tobias Schwab
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-28 00:00:00 Z
+date: 2011-08-02 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   requirement: &id001 !ruby/object:Gem::Requirement