supernova-core 0.0.1

data/spec/fixtures/cores_create.json

@@ -0,0 +1 @@
+{"responseHeader":{"status":0,"QTime":665},"core":"test_core","saved":"/opt/solr/solr.xml"}

data/spec/fixtures/cores_status.json

@@ -0,0 +1 @@
+{"responseHeader":{"status":0,"QTime":3},"status":{"project_test":{"name":"project_test","instanceDir":"/Users/tobias/Projects/project/solr/","dataDir":"/opt/solr/project_test/","startTime":"2012-03-26T21:03:51.633Z","uptime":295157,"index":{"numDocs":1013,"maxDoc":1013,"version":1326491264175,"optimized":true,"current":true,"hasDeletions":false,"directory":"org.apache.lucene.store.NIOFSDirectory:org.apache.lucene.store.NIOFSDirectory@/opt/solr/project_test/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@32b3a5a0","lastModified":"2012-03-07T15:32:31Z"}},"supernova_test":{"name":"supernova_test","instanceDir":"/Users/tobias/Projects/supernova/solr/","dataDir":"/opt/solr/supernova_test/","startTime":"2012-03-26T21:03:52.455Z","uptime":294337,"index":{"numDocs":4,"maxDoc":4,"version":1330931224856,"optimized":true,"current":true,"hasDeletions":false,"directory":"org.apache.lucene.store.NIOFSDirectory:org.apache.lucene.store.NIOFSDirectory@/opt/solr/supernova_test/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@303ec561","lastModified":"2012-03-05T12:14:35Z"}}}}

data/spec/integration/supernova_core_spec.rb

@@ -0,0 +1,95 @@
+require "spec_helper"
+require "supernova/solr/core"
+require "supernova/solr/mapper"
+require "supernova/batch_indexer"
+
+describe "Supernova Core Spec" do
+  let(:core) { Supernova::Solr::Core.new("http://localhost:8983/solr", "supernova_core") }
+
+  before(:each) do
+    WebMock.disable!
+    core.create("/opt/solr", "/opt/solr/supernova_core")
+    core.truncate
+    core.commit
+  end
+
+  class LogCatcher
+    def initialize
+      @messages ||= { :info => [] }
+    end
+
+    def messages
+      @messages
+    end
+
+    def method_missing(method, message)
+      @messages[method] ||= []
+      @messages[method] << message
+    end
+  end
+
+  describe "BatchIndexer" do
+    it "should work" do
+      Supernova::BatchIndexer.new(core, 10_000) do |indexer|
+        indexer.index_doc(:id => 1, :title_s => "Hello World")
+      end
+      core.commit
+      core.select["response"]["docs"].should == [{ "id" => "1", "title_s" => "Hello World"}]
+    end
+  end
+
+  describe "managing cores" do
+    it "allows managing cores" do
+      core.exists?.should == true
+      core.unload
+      core.exists?.should == false
+      core.create("/opt/solr", "/opt/solr/supernova_core")
+      core.exists?.should == true
+    end
+  end
+
+  
+
+  it "uses the correct logging" do
+    catcher = LogCatcher.new
+    Supernova.logger = catcher
+    core.select
+    catcher.messages[:info].first.should match(/^SUPERNOVA REQUEST: GET/)
+  end
+
+  it "indexes docs" do
+    core.truncate
+    core.commit
+    core.index_docs([{ :id => 1, :title_s => "Title 1" }, { :id => 3, :title_s => "Title 3" }])
+    core.select["response"]["docs"].should be_empty
+    core.commit
+    core.select["response"]["docs"].should == [
+      { "id" => "1", "title_s" => "Title 1" },
+      { "id" => "3", "title_s" => "Title 3"  },
+    ]
+  end
+
+  it "indexes single docs" do
+    core.truncate
+    core.commit
+    core.index_doc({ :id => 1, :title_s => "Title 1" })
+    core.select["response"]["docs"].should be_empty
+    core.commit
+    core.select["response"]["docs"].should == [
+      { "id" => "1", "title_s" => "Title 1" }
+    ]
+  end
+
+  it "raises an error when core does not exist" do
+    core.unload
+    lambda {
+      core.index_docs([{ :id => 1, :title_s => "Title 1" }, { :id => 3, :title_s => "Title 3" }])
+    }.should raise_error
+  end
+
+  it "raises an error when not able to index" do
+    lambda {
+      core.index_docs([{ :id => 1, :title_s => "Title 1" }, { :title_s => "Title 3" }])
+    }.should raise_error
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,30 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib/supernova'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require File.expand_path("../../lib/supernova.rb", __FILE__)
+require "logger"
+require "fileutils"
+require "webmock/rspec"
+
+def project_root
+  Pathname.new(File.expand_path("..", File.dirname(__FILE__)))
+end
+
+if defined?(Debugger) && Debugger.respond_to?(:settings)
+  Debugger.settings[:autolist] = 1
+  Debugger.settings[:autoeval] = true
+end
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  config.run_all_when_everything_filtered = true
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+
+  config.before(:each) do
+    WebMock.enable!
+  end
+end

data/spec/supernova/batch_indexer_spec.rb

@@ -0,0 +1,69 @@
+require "spec_helper"
+require "supernova/batch_indexer"
+
+describe "Supernova::BatchIndexer" do
+  let(:core) { double("core") }
+  let(:batch_indexer) { Supernova::BatchIndexer.new(core, 1000) }
+
+  it "can be initialized" do
+    indexer = Supernova::BatchIndexer.new(core)
+    indexer.instance_variable_get("@index").should == core
+    indexer.instance_variable_get("@batch_size").should == 1_000
+    indexer = Supernova::BatchIndexer.new(core, 2000)
+    indexer.instance_variable_get("@batch_size").should == 2_000
+  end
+
+  it "calls index docs with the correct amount of docs" do
+    doc1 = { :a => 1 }
+    doc2 = { :a => 2 }
+    doc3 = { :a => 3 }
+
+    core.should_receive(:index_docs).with([doc1, doc2])
+    core.should_receive(:index_docs).with([doc3])
+
+    indexer = Supernova::BatchIndexer.new(core, 2)
+    indexer.index_doc(doc1)
+    indexer.index_doc(doc2)
+    indexer.index_doc(doc3)
+    indexer.finish
+  end
+
+  it "calls index_docs when called with block" do
+    doc1 = { :a => 1 }
+    doc2 = { :a => 2 }
+    doc3 = { :a => 3 }
+
+    core.should_receive(:index_docs).with([doc1, doc2])
+    core.should_receive(:index_docs).with([doc3])
+
+    Supernova::BatchIndexer.new(core, 2) do |indexer|
+      indexer.index_doc(doc1)
+      indexer.index_doc(doc2)
+      indexer.index_doc(doc3)
+    end
+  end
+
+  describe "#do_index" do
+    it "calls index_docs" do
+      doc1 = { :a => 1 }
+      doc2 = { :a => 2 }
+      started = Time.now
+      finished = Time.now + 60
+      Time.stub(:now).and_return(started, finished)
+      batch_indexer.index_doc(doc1)
+      batch_indexer.index_doc(doc2)
+      core.should_receive(:index_docs).with([doc1, doc2])
+      batch_indexer.should_receive(:do_index_log_message).with(started, finished, [doc1, doc2]).and_return("log message")
+
+      Supernova.should_receive(:log).with("log message")
+      batch_indexer.send(:do_index)
+    end
+  end
+
+  it "generates the correct log message" do
+    started = Time.now
+    finished = Time.now + 2
+    docs = [1, 2, 3, 4, 5, 6]
+    batch_indexer.send(:do_index_log_message, started, finished, docs).should == "indexed 6 docs in 2.000 seconds (3.0 docs/second)"
+  end
+end

data/spec/supernova/solr/cli_spec.rb

@@ -0,0 +1,26 @@
+require "spec_helper"
+require "supernova/solr/cli"
+
+describe "Supernova::Solr::CLI" do
+  let(:cli) { Supernova::Solr::CLI.new([]) }
+
+  it "can be initialized" do
+    cli = Supernova::Solr::CLI.new([])
+  end
+
+  describe "#start_solr_cmd" do
+    it "returns a string" do
+      cli.send(:start_solr_cmd).should be_kind_of(String)   
+    end
+
+    it "returns the correct command" do
+      #  >> #{log_path} 2>&1 
+      cli.send(:start_solr_cmd).should include("cd /usr/local/Cellar/solr/3.5.0/libexec/example && /usr/bin/env java -Dsolr.solr.home=/opt/solr -jar start.jar > /opt/solr/solr.log 2>&1")
+    end
+
+    it "uses a custom solr bin when provided" do
+      cli = Supernova::Solr::CLI.new(["--solr-bin", "/path/to/solr/start.jar"])
+      cli.send(:start_solr_cmd).should include("cd /path/to/solr")
+    end
+  end
+end

data/spec/supernova/solr/core_spec.rb

@@ -0,0 +1,81 @@
+require File.expand_path("../../../../spec/spec_helper", __FILE__)
+require "solr/core"
+
+
+describe "Supernova::Solr::Core" do
+  let(:url) { "http://path.to.solr:1122" }
+  let(:core_url) { "http://path.to.solr:1122/solr/my_core" }
+  let(:body) { { :a => 1 }.to_json }
+
+  describe "#initialize" do
+    it "can be initialized" do
+      core = Supernova::Solr::Core.new(url, "my_name")
+      core.solr_url.should == url
+      core.name.should == "my_name"
+    end
+
+    it "removes trailing slashes for the solr_url" do
+      Supernova::Solr::Core.new("http://path.to.solr:1122/solr/", "my_name").url.should == "http://path.to.solr:1122/solr/my_name"
+    end
+  end
+
+  it "returns the correct url" do
+    Supernova::Solr::Core.new(url, "my_name").url.should == "#{url}/my_name"
+  end
+
+  describe "#create" do
+    it "creates the correct core" do
+      instance_path = "/path/to/instance"
+      data_path = "/path/to/data"
+      stub_request(:get, "http://path.to.solr:1122/admin/cores?action=CREATE&dataDir=/path/to/data&instanceDir=/path/to/instance&name=my_name&wt=json")
+        .to_return(:status => 200, :body => body, :headers => {})
+      Supernova::Solr::Core.create(url, "my_name", instance_path, data_path)
+    end
+
+    it "creates the correct core" do
+      stub_request(:get, "http://path.to.solr:1122/admin/cores?action=CREATE&dataDir=/opt/solr/data/my_other_name&instanceDir=/opt/solr&name=my_other_name&wt=json")
+        .to_return(:status => 200, :body => body, :headers => {})
+      Supernova::Solr::Core.create(url, "my_other_name")
+    end
+  end
+
+  it "correctly queries for core status" do
+    instance_path = "/path/to/instance"
+    data_path = "/path/to/data"
+    stub_request(:get, "http://path.to.solr:1122/admin/cores?action=STATUS&core=my_name&wt=json")
+      .to_return(:status => 200, :body => body, :headers => {})
+    Supernova::Solr::Core.status(url, "my_name")
+  end
+
+  it "unloads the correct core" do
+    instance_path = "/path/to/instance"
+    data_path = "/path/to/data"
+    stub_request(:get, "http://path.to.solr:1122/admin/cores?action=UNLOAD&core=my_name&deleteIndex=true&wt=json")
+    .to_return(:status => 200, :body => body, :headers => {})
+    Supernova::Solr::Core.unload(url, "my_name")
+  end
+
+  it "delegates unload for instance methods" do
+    instance_path = "/path/to/instance"
+    data_path = "/path/to/data"
+    stub_request(:get, "http://path.to.solr:1122/admin/cores?action=UNLOAD&core=my_name&deleteIndex=true&wt=json")
+    .to_return(:status => 200, :body => body, :headers => {})
+    Supernova::Solr::Core.new(url, "my_name").unload
+  end
+
+  describe "#exists?" do
+    it "true when returned hash points to core" do
+      response = { "status" => { "my_name" => { "some" => "entry" } } }
+      stub_request(:get, "http://path.to.solr:1122/admin/cores?action=STATUS&core=my_name&wt=json").
+         to_return(:status => 200, :body => response.to_json, :headers => {})
+      Supernova::Solr::Core.new(url, "my_name").exists?.should == true
+    end
+
+    it "true when returned hash points to core" do
+      response = { "status" => { "my_name" => { } } }
+      stub_request(:get, "http://path.to.solr:1122/admin/cores?action=STATUS&core=my_name&wt=json").
+         to_return(:status => 200, :body => response.to_json, :headers => {})
+      Supernova::Solr::Core.new(url, "my_name").exists?.should == false
+    end
+  end
+end

data/spec/supernova/solr/mapper_spec.rb

@@ -0,0 +1,25 @@
+require "spec_helper"
+require "supernova/solr/mapper"
+
+describe "Supernova::Solr::Mapper" do
+  let(:mapping) do 
+    { :name => :name_s, :login_count => :login_count_i }
+  end
+
+  it "sets the mapping" do
+    mapper = Supernova::Solr::Mapper.new(mapping)
+    mapper.send(:mapping).should == mapping.merge(:id => :id)
+  end
+
+  it "allows using integer, string et al" do
+    mapper = Supernova::Solr::Mapper.new(:name => :string, :number => :number_i)
+    mapper.send(:mapping).should == { :id => :id, :name => :name_s, :number => :number_i }
+  end
+
+  describe "#map" do
+    it "returns the correct hash" do
+      mapper = Supernova::Solr::Mapper.new(mapping)
+      mapper.map(:name => "Hans", :login_count => 10, :a => 1, :id => 11).should == { :name_s => "Hans", :login_count_i => 10, :id => 11 }
+    end
+  end
+end

data/spec/supernova/solr/server_spec.rb

@@ -0,0 +1,144 @@
+require File.expand_path("../../../../spec/spec_helper", __FILE__)
+require "solr/server"
+
+describe "Supernova::Solr::Server" do
+	let(:url) { "http://path.to.solr:112" }
+  let(:body) { "{\"responseHeader\":{\"status\":0,\"QTime\":3}}" }
+
+  describe "#initialize" do
+    it "can be initialized" do
+      Supernova::Solr::Server.new(url).url.should == url
+    end
+
+    it "strips the trailing slash" do
+      Supernova::Solr::Server.new("http://path.to.solr:112/").url.should == "http://path.to.solr:112"
+    end
+  end
+
+  describe "#core_names" do
+    it "returns the correct array" do
+      stub_request(:get, "http://path.to.solr:112/admin/cores?action=STATUS&wt=json")
+        .to_return(:status => 200, :body => project_root.join("spec/fixtures/cores_status.json").read, :headers => {})
+      Supernova::Solr::Server.core_names(url).should == %w(project_test supernova_test)
+    end
+  end
+
+  describe "post_update" do
+    it "returns a hash" do
+      response = { "a" => 1 }
+      stub_request(:post, "http://path.to.solr:112/update/update/json?wt=json")
+        .with(:headers => {'Content-Type'=>'application/json'})
+        .to_return(:status => 200, :body => response.to_json, :headers => {})
+      Supernova::Solr::Server.send(:post_update, "http://path.to.solr:112/update", "").should == response
+    end
+  end
+
+  it "indexes single docs" do
+    doc = { "id" => 1, "title_s" => "Peter" }
+    stub_request(:post, "http://path.to.solr:112/update/json?wt=json")
+      .with(
+        :body => "{\"add\":{\"doc\":{\"id\":1,\"title_s\":\"Peter\"}}}",
+        :headers => {'Content-Type'=>'application/json'}
+      )
+      .to_return(:status => 200, :body => body, :headers => {})
+    Supernova::Solr::Server.index_doc(url, doc)
+  end
+
+  describe "index_docs" do
+    it "indexes the correct rows" do
+      rows = [
+        { "id" => 1, "title_s" => "Peter" },
+        { "id" => 2, "title_s" => "Thomas" },
+      ]
+      stub_request(:post, "http://path.to.solr:112/update/json?wt=json")
+        .with(
+          :body => "{\"add\":{\"doc\":{\"id\":1,\"title_s\":\"Peter\"}}}\n{\"add\":{\"doc\":{\"id\":2,\"title_s\":\"Thomas\"}}}",
+          :headers => {'Content-Type'=>'application/json'}
+        )
+        .to_return(:status => 200, :body => body, :headers => {})
+      Supernova::Solr::Server.index_docs(url, rows)
+    end
+
+    it "indexes the correct rows with commit" do
+      rows = [
+        { "id" => 1, "title_s" => "Peter" },
+        { "id" => 2, "title_s" => "Thomas" },
+      ]
+      stub_request(:post, "http://path.to.solr:112/update/json?wt=json&commit=true")
+        .with(
+          :body => "{\"add\":{\"doc\":{\"id\":1,\"title_s\":\"Peter\"}}}\n{\"add\":{\"doc\":{\"id\":2,\"title_s\":\"Thomas\"}}}",
+          :headers => {'Content-Type'=>'application/json'}
+        )
+        .to_return(:status => 200, :body => body, :headers => {})
+      Supernova::Solr::Server.index_docs(url, rows, true)
+    end
+
+    it "indexes the correct rows with commitWithin" do
+      rows = [
+        { "id" => 1, "title_s" => "Peter" },
+        { "id" => 2, "title_s" => "Thomas" },
+      ]
+      stub_request(:post, "http://path.to.solr:112/update/json?wt=json&commitWithin=10000")
+        .with(
+          :body => "{\"add\":{\"doc\":{\"id\":1,\"title_s\":\"Peter\"}}}\n{\"add\":{\"doc\":{\"id\":2,\"title_s\":\"Thomas\"}}}",
+          :headers => {'Content-Type'=>'application/json'}
+        )
+        .to_return(:status => 200, :body => body, :headers => {})
+      Supernova::Solr::Server.index_docs(url, rows, 10_000)
+    end
+  end
+
+  describe "#select" do
+    let(:body) { { "a" => 1 }.to_json }
+    it "selects with default parameters" do
+      stub_request(:get, "http://path.to.solr:112/select?q=*:*&wt=json")
+        .to_return(:status => 200, :body => body, :headers => {})
+      Supernova::Solr::Server.select(url).should == { "a" => 1 }
+    end
+
+    it "delegates for instance methods" do
+      stub_request(:get, "http://path.to.solr:112/select?q=*:*&wt=json")
+        .to_return(:status => 200, :body => body, :headers => {})
+      Supernova::Solr::Server.new("http://path.to.solr:112").select.should == { "a" => 1 }
+    end
+  end
+
+  it "calls the commit statement" do
+    stub_request(:post, "http://path.to.solr:112/update/json?wt=json")
+      .with(:body => "{\"commit\":{}}", :headers => {'Content-Type'=>'application/json'})
+      .to_return(:status => 200, :body => body, :headers => {})
+    Supernova::Solr::Server.commit(url)
+  end
+
+  it "delegates commit for instance methods" do
+    stub_request(:post, "http://path.to.solr:112/my_name/update/json?wt=json")
+      .with(:body => "{\"commit\":{}}", :headers => {'Content-Type'=>'application/json'})
+      .to_return(:status => 200, :body => body, :headers => {})
+    core = Supernova::Solr::Server.new("#{url}/my_name")
+    core.url.should == "#{url}/my_name"
+    core.commit
+  end
+
+  it "calls the optimize statement" do
+    stub_request(:post, "http://path.to.solr:112/update/json?wt=json")
+      .with(:body => "{\"optimize\":{}}", :headers => {'Content-Type'=>'application/json'})
+      .to_return(:status => 200, :body => body, :headers => {})
+    Supernova::Solr::Server.optimize(url)
+  end
+
+  describe "#delete" do
+    it "deletes by query" do
+      stub_request(:post, "http://path.to.solr:112/update/json?wt=json")
+        .with(:body => "{\"delete\":{\"query\":\"id:1\"}}", :headers => {'Content-Type'=>'application/json'})
+        .to_return(:status => 200, :body => body, :headers => {})
+      Supernova::Solr::Server.delete_by_query(url, "id:1")
+    end
+
+    it "truncates the index" do
+      stub_request(:post, "http://path.to.solr:112/update/json?wt=json&commit=true")
+        .with(:body => "{\"delete\":{\"query\":\"*:*\"}}", :headers => {'Content-Type'=>'application/json'})
+        .to_return(:status => 200, :body => body, :headers => {})
+      Supernova::Solr::Server.truncate(url, true)
+    end
+  end
+end