supernova 0.6.5 → 0.6.6

data/Gemfile

@@ -10,6 +10,7 @@ gem "rsolr"
 gem "will_paginate"
 gem "json"
 gem "activesupport"
+gem "typhoeus"
 
 group :development do
   gem "i18n"

data/Gemfile.lock

@@ -29,6 +29,7 @@ GEM
     json (1.5.3)
     linecache (0.46)
       rbx-require-relative (> 0.0.4)
+    mime-types (1.16)
     mysql2 (0.2.11)
     rake (0.9.2)
     rbx-require-relative (0.0.5)
@@ -48,6 +49,9 @@ GEM
       ruby-debug-base (~> 0.10.4.0)
     ruby-debug-base (0.10.4)
       linecache (>= 0.3)
+    typhoeus (0.2.4)
+      mime-types
+      mime-types
     tzinfo (0.3.29)
     will_paginate (2.3.15)
 
@@ -70,4 +74,5 @@ DEPENDENCIES
   rsolr
   rspec (~> 2.3.0)
   ruby-debug
+  typhoeus
   will_paginate

data/VERSION

@@ -1 +1 @@
-0.6.5
+0.6.6

data/lib/supernova/solr_indexer.rb

@@ -1,9 +1,10 @@
 require "json"
 require "fileutils"
 require "time"
+require "typhoeus"
 
 class Supernova::SolrIndexer
-  attr_accessor :options, :db, :ids, :max_rows_to_direct_index, :local_solr
+  attr_accessor :options, :db, :ids, :max_rows_to_direct_index, :local_solr, :current_json_string
   attr_writer :index_file_path, :debug
   
   MAX_ROWS_TO_DIRECT_INDEX = 100
@@ -221,7 +222,7 @@ class Supernova::SolrIndexer
     end
     if self.max_rows_to_direct_index < rows.count
       debug "indexed #{rows.length} rows with json in %TIME%" do
-        index_with_json_file(rows)
+        index_with_json(rows)
       end
     else
       debug "indexed #{rows.length} rows directly in %TIME%" do
@@ -230,6 +231,10 @@ class Supernova::SolrIndexer
     end
   end
   
+  def index_with_json(rows)
+    options && options[:use_json_file] ? index_with_json_file(rows) : index_with_json_string(rows)
+  end
+  
   def solr_rows_to_index_for_query(query)
     query_db(query).map do |row|
       map_for_solr(row)
@@ -257,6 +262,36 @@ class Supernova::SolrIndexer
     finish
   end
   
+  def append_to_json_string(row)
+    if self.current_json_string.nil?
+      self.current_json_string = "\{\n"
+    else
+      self.current_json_string << ",\n"
+    end
+    self.current_json_string << %("add":#{{:doc => row}.to_json})
+  end
+  
+  def finalize_json_string
+    self.current_json_string << "\n}"
+  end
+  
+  def post_json_string
+    Typhoeus::Request.post("#{solr_update_url}?commit=true", 
+      :body => self.current_json_string, 
+      :headers => { "Content-type" => "application/json; charset=utf-8" }
+    ).tap do |response|
+      self.current_json_string = nil
+    end
+  end
+  
+  def index_with_json_string(rows)
+    rows.each do |row|
+      append_to_json_string(row)
+    end
+    finalize_json_string
+    post_json_string
+  end
+  
   def ids_given?
     self.ids.is_a?(Array)
   end
@@ -297,10 +332,14 @@ class Supernova::SolrIndexer
     Supernova::Solr.url.present? ? Supernova::Solr.url.to_s.gsub(/\/$/, "") : nil
   end
   
+  def solr_update_url
+    "#{solr_url}/update/json"
+  end
+  
   def do_index_file(options = {})
     raise "solr not configured" if solr_url.nil?
     cmd = if self.local_solr
-      %(curl -s '#{solr_url}/update/json?commit=true\\&stream.file=#{index_file_path}')
+      %(curl -s '#{solr_update_url}?commit=true\\&stream.file=#{index_file_path}')
     else
       %(cd #{File.dirname(index_file_path)} && curl -s '#{solr_url}/update/json?commit=true' --data-binary @#{File.basename(index_file_path)} -H 'Content-type:application/json')
     end

data/spec/integration/solr_spec.rb

@@ -31,6 +31,21 @@ describe "Solr" do
     Offer.search_scope
   end
   
+  describe "#index_with_json_string" do
+    it "indexes the correct rows" do
+      Supernova::Solr.truncate!
+      indexer = Supernova::SolrIndexer.new
+      indexer.index_with_json_string([
+          { :title_s => "Title1", :id => 1, :type => "Record" }, 
+          { :title_s => "Title2", :id => 2, :type => "Record" } 
+        ]
+      )
+      response = Supernova::Solr.connection.get('select', :params => { :q=>'*:*', :start=>0, :rows=>10, :sort => "id asc" })
+      response["response"]["docs"].first.should == { "title_s" => "Title1", "id" => "1", "type" => "Record" }
+      response["response"]["docs"].at(1).should == { "title_s" => "Title2", "id" => "2", "type" => "Record" }
+    end
+  end
+  
   describe "#indexing" do
     before(:each) do
       Supernova::Solr.truncate!
@@ -64,6 +79,7 @@ describe "Solr" do
       offer1 = Offer.create!(:user_id => 1, :popularity => 10)
       offer2 = Offer.create!(:user_id => 2, :popularity => 20)
       indexer = OfferIndex.new(:db => ActiveRecord::Base.connection, :max_rows_to_direct_index => 0)
+      indexer.options[:use_json_file] = true
       indexer.index!
       indexer.instance_variable_get("@index_file_path").should_not be_nil
       OfferIndex.search_scope.total_entries.should == 2
@@ -76,6 +92,7 @@ describe "Solr" do
       offer1 = Offer.create!(:user_id => 1, :popularity => 10)
       offer2 = Offer.create!(:user_id => 2, :popularity => 20)
       indexer = OfferIndex.new(:db => ActiveRecord::Base.connection, :max_rows_to_direct_index => 0, :local_solr => true)
+      indexer.options[:use_json_file] = true
       indexer.index!
       indexer.instance_variable_get("@index_file_path").should_not be_nil
       OfferIndex.search_scope.first.instance_variable_get("@original_search_doc")["indexed_at_dt"].should_not be_nil

data/spec/supernova/solr_indexer_spec.rb

@@ -33,6 +33,72 @@ describe Supernova::SolrIndexer do
     Kernel.stub!(:`).and_return true
   end
   
+  describe "#index_with_json_string" do
+    let(:row1) { double("row1") }
+    let(:row2) { double("row2") }
+    let(:rows) { [row1, row2] }
+    
+    before(:each) do
+      indexer.current_json_string = "{"
+      indexer.stub!(:append_to_json_string)
+    end
+    
+    it "calls append to string with all rows" do
+      indexer.should_receive(:append_to_json_string).with(row1)
+      indexer.should_receive(:append_to_json_string).with(row2)
+      indexer.index_with_json_string(rows)
+    end
+    
+    it "calls finalize_json_string" do
+      indexer.should_receive(:finalize_json_string)
+      indexer.index_with_json_string(rows)
+    end
+    
+    it "calls post_json_string" do
+      indexer.should_receive(:post_json_string)
+      indexer.index_with_json_string(rows)
+    end
+  end
+  
+  describe "#post_json_string" do
+    before(:each) do
+      Typhoeus::Request.stub(:post)
+    end
+    
+    it "posts the json string" do
+      indexer.current_json_string = "some string"
+      Typhoeus::Request.should_receive(:post).with("http://solr.xx:9333/solr/update/json?commit=true", :body => "some string", :headers => { "Content-type" => "application/json; charset=utf-8" }).and_return(double("rsp", :body => "text"))
+      indexer.post_json_string
+    end
+    
+    it "resets the current_json_string" do
+      indexer.current_json_string = "some string"
+      indexer.post_json_string
+      indexer.current_json_string.should be_nil
+    end
+  end
+  
+  describe "#append_to_json_string" do
+    it "creates a new string" do
+      indexer.append_to_json_string({"a" => 1})
+      indexer.current_json_string.should == %({\n"add":{"doc":{"a":1}})
+    end
+    
+    it "appends to the existing string" do
+      indexer.append_to_json_string({"a" => 1})
+      indexer.append_to_json_string({"b" => 2})
+      indexer.current_json_string.should == %({\n"add":{"doc":{"a":1}},\n"add":{"doc":{"b":2}})
+    end
+  end
+  
+  describe "#finalize_json_string" do
+    it "adds the last brackets" do
+      indexer.append_to_json_string({"a" => 1})
+      indexer.finalize_json_string
+      indexer.current_json_string.should == %({\n"add":{"doc":{"a":1}}\n})
+    end
+  end
+  
   describe "initialize" do
     it "sets all options" do
       options = { :database => { :database => "dynasty", :username => "dynasty_user" } }
@@ -375,6 +441,19 @@ describe Supernova::SolrIndexer do
     end
   end
   
+  describe "#index_with_json" do
+    it "calls index_with_json_string by default" do
+      indexer.should_receive(:index_with_json_string).with([1])
+      indexer.index_with_json([1])
+    end
+    
+    it "calls index_with_json_file when asked to" do
+      indexer.options[:use_json_file] = true
+      indexer.should_receive(:index_with_json_file).with([1])
+      indexer.index_with_json([1])
+    end
+  end
+  
   describe "#index_rows" do
     let(:row1) { double("row1") }
     let(:row2) { double("row2") }
@@ -404,7 +483,7 @@ describe Supernova::SolrIndexer do
     
     it "calls map_directly when number of rows < max_rows_to_direct_index" do
       custom_indexer.should_receive(:max_rows_to_direct_index).and_return 1
-      custom_indexer.should_receive(:index_with_json_file).with([mapped1, mapped2])
+      custom_indexer.should_receive(:index_with_json).with([mapped1, mapped2])
       custom_indexer.index_rows([row1, row2])
     end
   end

data/supernova.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{supernova}
-  s.version = "0.6.5"
+  s.version = "0.6.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Tobias Schwab"]
-  s.date = %q{2011-09-01}
+  s.date = %q{2011-11-04}
   s.default_executable = %q{start_solr}
   s.description = %q{Unified search scopes}
   s.email = %q{tobias.schwab@dynport.de}
@@ -95,7 +95,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/dynport/supernova}
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.6.2}
+  s.rubygems_version = %q{1.5.2}
   s.summary = %q{Unified search scopes}
 
   if s.respond_to? :specification_version then
@@ -106,6 +106,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<will_paginate>, [">= 0"])
       s.add_runtime_dependency(%q<json>, [">= 0"])
       s.add_runtime_dependency(%q<activesupport>, [">= 0"])
+      s.add_runtime_dependency(%q<typhoeus>, [">= 0"])
       s.add_development_dependency(%q<i18n>, [">= 0"])
       s.add_development_dependency(%q<activerecord>, ["~> 3.0.7"])
       s.add_development_dependency(%q<ruby-debug>, [">= 0"])
@@ -123,6 +124,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<will_paginate>, [">= 0"])
       s.add_dependency(%q<json>, [">= 0"])
       s.add_dependency(%q<activesupport>, [">= 0"])
+      s.add_dependency(%q<typhoeus>, [">= 0"])
       s.add_dependency(%q<i18n>, [">= 0"])
       s.add_dependency(%q<activerecord>, ["~> 3.0.7"])
       s.add_dependency(%q<ruby-debug>, [">= 0"])
@@ -141,6 +143,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<will_paginate>, [">= 0"])
     s.add_dependency(%q<json>, [">= 0"])
     s.add_dependency(%q<activesupport>, [">= 0"])
+    s.add_dependency(%q<typhoeus>, [">= 0"])
     s.add_dependency(%q<i18n>, [">= 0"])
     s.add_dependency(%q<activerecord>, ["~> 3.0.7"])
     s.add_dependency(%q<ruby-debug>, [">= 0"])

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: supernova
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 11
   prerelease: 
   segments: 
   - 0
   - 6
-  - 5
-  version: 0.6.5
+  - 6
+  version: 0.6.6
 platform: ruby
 authors: 
 - Tobias Schwab
@@ -15,12 +15,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-01 00:00:00 +02:00
+date: 2011-11-04 00:00:00 +01:00
 default_executable: start_solr
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rsolr
-  prerelease: false
   version_requirements: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -30,11 +29,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   type: :runtime
   requirement: *id001
 - !ruby/object:Gem::Dependency 
   name: will_paginate
-  prerelease: false
   version_requirements: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -44,11 +43,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   type: :runtime
   requirement: *id002
 - !ruby/object:Gem::Dependency 
   name: json
-  prerelease: false
   version_requirements: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -58,11 +57,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   type: :runtime
   requirement: *id003
 - !ruby/object:Gem::Dependency 
   name: activesupport
-  prerelease: false
   version_requirements: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -72,11 +71,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   type: :runtime
   requirement: *id004
 - !ruby/object:Gem::Dependency 
-  name: i18n
-  prerelease: false
+  name: typhoeus
   version_requirements: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -86,12 +85,26 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  prerelease: false
+  type: :runtime
   requirement: *id005
 - !ruby/object:Gem::Dependency 
-  name: activerecord
-  prerelease: false
+  name: i18n
   version_requirements: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  type: :development
+  requirement: *id006
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  version_requirements: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -102,12 +115,12 @@ dependencies:
         - 0
         - 7
         version: 3.0.7
+  prerelease: false
   type: :development
-  requirement: *id006
+  requirement: *id007
 - !ruby/object:Gem::Dependency 
   name: ruby-debug
-  prerelease: false
-  version_requirements: &id007 !ruby/object:Gem::Requirement 
+  version_requirements: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -116,12 +129,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   type: :development
-  requirement: *id007
+  requirement: *id008
 - !ruby/object:Gem::Dependency 
   name: mysql2
-  prerelease: false
-  version_requirements: &id008 !ruby/object:Gem::Requirement 
+  version_requirements: &id009 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -132,12 +145,12 @@ dependencies:
         - 2
         - 7
         version: 0.2.7
+  prerelease: false
   type: :development
-  requirement: *id008
+  requirement: *id009
 - !ruby/object:Gem::Dependency 
   name: ZenTest
-  prerelease: false
-  version_requirements: &id009 !ruby/object:Gem::Requirement 
+  version_requirements: &id010 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - "="
@@ -148,12 +161,12 @@ dependencies:
         - 5
         - 0
         version: 4.5.0
+  prerelease: false
   type: :development
-  requirement: *id009
+  requirement: *id010
 - !ruby/object:Gem::Dependency 
   name: geokit
-  prerelease: false
-  version_requirements: &id010 !ruby/object:Gem::Requirement 
+  version_requirements: &id011 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -162,12 +175,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   type: :development
-  requirement: *id010
+  requirement: *id011
 - !ruby/object:Gem::Dependency 
   name: autotest
-  prerelease: false
-  version_requirements: &id011 !ruby/object:Gem::Requirement 
+  version_requirements: &id012 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -176,12 +189,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   type: :development
-  requirement: *id011
+  requirement: *id012
 - !ruby/object:Gem::Dependency 
   name: autotest-growl
-  prerelease: false
-  version_requirements: &id012 !ruby/object:Gem::Requirement 
+  version_requirements: &id013 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -190,12 +203,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   type: :development
-  requirement: *id012
+  requirement: *id013
 - !ruby/object:Gem::Dependency 
   name: rspec
-  prerelease: false
-  version_requirements: &id013 !ruby/object:Gem::Requirement 
+  version_requirements: &id014 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -206,12 +219,12 @@ dependencies:
         - 3
         - 0
         version: 2.3.0
+  prerelease: false
   type: :development
-  requirement: *id013
+  requirement: *id014
 - !ruby/object:Gem::Dependency 
   name: bundler
-  prerelease: false
-  version_requirements: &id014 !ruby/object:Gem::Requirement 
+  version_requirements: &id015 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -222,12 +235,12 @@ dependencies:
         - 0
         - 0
         version: 1.0.0
+  prerelease: false
   type: :development
-  requirement: *id014
+  requirement: *id015
 - !ruby/object:Gem::Dependency 
   name: jeweler
-  prerelease: false
-  version_requirements: &id015 !ruby/object:Gem::Requirement 
+  version_requirements: &id016 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -238,12 +251,12 @@ dependencies:
         - 6
         - 0
         version: 1.6.0
+  prerelease: false
   type: :development
-  requirement: *id015
+  requirement: *id016
 - !ruby/object:Gem::Dependency 
   name: rcov
-  prerelease: false
-  version_requirements: &id016 !ruby/object:Gem::Requirement 
+  version_requirements: &id017 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -252,8 +265,9 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  prerelease: false
   type: :development
-  requirement: *id016
+  requirement: *id017
 description: Unified search scopes
 email: tobias.schwab@dynport.de
 executables: 
@@ -366,7 +380,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
 summary: Unified search scopes