rsolr-async 0.1.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Matt Mitchell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,23 @@
+= rsolr-async
+rsolr-async enhances the RSolr core library by adding the ability to use EventMachine as a connection adapter.
+
+=Requirements
+Ruby 1.9 is required, as well as the em-http-request and eventmachine gems.
+
+=How
+Just pass-in :async to the RSolr.connect method:
+  require 'rsolr-async'
+  rsolr = RSolr.connect(:async, :url => 'http://localhost:8983/solr')
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a future version unintentionally.
+* Commit, do not mess with rakefile, version, or history
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Matt Mitchell. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "rsolr-async"
+    gem.summary = %Q{An EventMachine based connection adapter for RSolr}
+    gem.description = %Q{Provides asynchronous connections to Solr}
+    gem.email = "goodieboy@gmail.com"
+    gem.homepage = "http://github.com/mwmitchell/rsolr-async"
+    gem.authors = ["Matt Mitchell", "Mike Perham"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_dependency "rsolr", ">= 0.12.1"
+    gem.add_dependency "eventmachine", ">= 0.12.10"
+    gem.add_dependency "em-http-request", ">= 0.2.6"
+    
+    gem.files = FileList['lib/**/*.rb', 'examples/**', 'LICENSE', 'README.rdoc', 'VERSION']
+    gem.test_files = ['spec/*', 'Rakefile', 'solr/example/**/*']
+    
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  # Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "rsolr-async #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/examples/bulk_indexer.rb

@@ -0,0 +1,65 @@
+require 'benchmark'
+require 'rsolr'
+require 'rsolr-async'
+
+# Bulk load a number of documents into Solr.
+# Ensures that Solr commits the changes once per minute.
+class BulkIndexer
+  
+  def index(docs)
+    puts "indexing #{docs.size} docs"
+    Benchmark.measure do
+      docs.each_with_index do |doc,index|
+        doc.merge!(:id => Time.now.to_f.to_s)
+        result = rsolr.update(rsolr.message.add(doc, :commitWithin => 60000), :wt => 'ruby')
+        raise RuntimeError, result if result !~ /'status'=>0/
+      end
+    end
+  end
+  
+  private
+  
+  def rsolr
+    @solr ||= RSolr.connect(:async, :url => 'http://localhost:8983/solr')
+  end
+end
+
+# Use like so:
+#
+# ruby -Ilib examples/bulk_indexer.rb
+#
+# You will need to change doc to reflect the fields in your Solr's schema.xml.
+doc = {
+  'content' => <<-EOS
+  Teams have doubled the number of scouts on the payroll over the last two decades to an average of 12 while their marketing departments, which are charged with keeping the revenue rolling in, have tripled in size to about 20 employees each. Moreover, teams have spent millions on training facilities, video-editing technology, nutritionists, specialized catering services and high-tech meeting rooms.
+
+  Indianapolis linebackers coach Mike Murphy, who has been an NFL coach for more than two decades, says he remembers when computers, introduced in the 1990s, first eliminated the hours coaches used to spend splicing game film. But instead of going home at a reasonable hour, he says, coaches started working more. "We're so paranoid that we explore every possibility, every angle," he says. "You have so much information that you can confuse yourself. You can go nuts."
+EOS
+}
+docs = Array.new(1000, doc)
+
+EM.run do
+  # We create 5 fibers, which allows us to run 5 I/O operations
+  # in parallel.  In practice, you won't see a huge speedup if Solr
+  # is running on localhost.  On my machine:
+  # Indexing done in 4.698831 seconds with 1 concurrency
+  # Indexing done in 3.898857 seconds with 2 concurrency
+  # Indexing done in 3.520972 seconds with 5 concurrency
+  
+  a = Time.now
+  concurrency = 5
+  completed = 0
+  concurrency.times do |idx|
+   
+    Fiber.new do
+      bl = BulkIndexer.new
+      bl.index(docs.slice(idx*(1000/concurrency), 1000/concurrency))
+      completed += 1
+      if completed == concurrency
+        puts "Indexing done in #{Time.now - a} seconds with #{concurrency} concurrency"
+        EM.stop
+      end
+    end.resume
+
+  end
+end

data/lib/rsolr-async.rb

@@ -0,0 +1,102 @@
+raise RuntimeError, "EventMachine connection requires Ruby 1.9" if RUBY_VERSION < '1.9'
+
+require 'rubygems'
+require 'rsolr'
+require 'em-http'
+require 'fiber'
+
+#
+# Connection for EventMachine
+#
+module RSolr::Async
+  
+  module Connectable
+    
+    def connect *args, &blk
+      if args.first == :async
+        rsolr = RSolr::Client.new(RSolr::Async::Connection.new(*args[1..-1]))
+        block_given? ? (yield rsolr) : rsolr
+      else
+        super *args, &blk
+      end
+    end
+    
+  end
+  
+  RSolr.extend RSolr::Async::Connectable
+  
+  #
+  # Evented Connection for standard HTTP Solr server
+  #
+  class Connection
+    
+    include RSolr::Connection::Requestable
+    
+    REQUEST_CLASS = EM::HttpRequest
+    
+    protected
+    
+    def connection path
+      REQUEST_CLASS.new("#{@uri.to_s}#{path}")
+    end
+    
+    def timeout
+      opts[:timeout] || 5
+    end
+
+    def get path, params={}
+      # this yield/resume business is complicated by em-http's mocking support which
+      # yields to the callback immediately rather than from another fiber.
+      yielding = true
+      fiber = Fiber.current
+      http_response = self.connection(path).get :query => params, :timeout => timeout
+      http_response.callback do
+        yielding = false
+        fiber.resume if Fiber.current != fiber
+      end
+      http_response.errback do
+        yielding = false
+        fiber.resume if Fiber.current != fiber
+      end
+      Fiber.yield if yielding
+      create_http_context http_response, path, params
+    end
+    
+    def post path, data, params={}, headers={}
+      yielding = true
+      fiber = Fiber.current
+      http_response = self.connection(path).post :query => params, :body => data, :head => headers, :timeout => timeout
+      http_response.callback do
+        yielding = false
+        fiber.resume if Fiber.current != fiber
+      end
+      http_response.errback do
+        yielding = false
+        fiber.resume if Fiber.current != fiber
+      end
+      Fiber.yield if yielding
+      create_http_context http_response, path, params, data, headers
+    end
+    
+    def create_http_context http_response, path, params, data=nil, headers={}
+      full_url = "#{@uri.to_s}#{path}"
+      {
+        :status_code=>http_response.response_header.status,
+        :url=>full_url,
+        :body=>encode_utf8(http_response.response),
+        :path=>path,
+        :params=>params,
+        :data=>data,
+        :headers=>headers,
+      }
+    end
+    
+    # accepts a path/string and optional hash of query params
+    def build_url path, params={}
+      full_path = @uri.path + path
+      super full_path, params, @uri.query
+    end
+
+  end
+  
+end

data/spec/rsolr-async_spec.rb

@@ -0,0 +1,69 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+require 'em-http'
+require 'em-http/mock'
+
+describe RSolr::Async do
+  
+  context 'initialization' do
+    
+    it 'should modifiy RSolr' do
+      RSolr.should be_a(RSolr::Async::Connectable)
+    end
+    
+    it 'should not change the default connect behavior' do
+      RSolr.connect.connection.should be_a(RSolr::Connection::NetHttp)
+    end
+    
+    it 'should create an instance of RSolr::Async::Connection when :async is used' do
+      RSolr.connect(:async, :url=>'http://localhost:8983/solr').connection.should be_a(RSolr::Async::Connection)
+    end
+    
+  end
+  
+  RSolr::Async::Connection::REQUEST_CLASS = EM::MockHttpRequest
+  
+  let(:new_connection){ RSolr::Async::Connection.new }
+  
+  context '#request' do
+    it 'should forward simple, non-data calls to #get' do
+      
+      EM.run do
+        
+        EM.add_timer(1) do
+          EM.stop
+        end
+        
+        EM::MockHttpRequest.pass_through_requests = false
+        body = <<-EOM
+HTTP/1.1 200 OK
+Date: Mon, 16 Nov 2009 20:39:15 GMT
+Expires: -1
+Cache-Control: private, max-age=0
+Content-Type: text/xml; charset=utf-8
+Connection: close
+
+<?xml version="1.0" encoding="UTF-8"?>
+<response>
+<lst name="responseHeader"><int name="status">0</int><int name="QTime">1</int><lst name="params"><str name="q">a</str></lst></lst><result name="response" numFound="0" start="0"/>
+</response>
+EOM
+        EM::MockHttpRequest.register 'http://127.0.0.1:8983/solr/select?q=a', :get, body
+        
+        Fiber.new do
+          begin
+            http = new_connection
+            resp = http.request('/select', :q=>'a')
+            resp[:status_code].should == 200
+          rescue Exception => ex
+            puts ex.message
+            puts ex.backtrace.join("\n")
+          end
+          EM.stop
+        end.resume
+      end
+      
+    end
+  end
+  
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rsolr-async'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification 
+name: rsolr-async
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Matt Mitchell
+- Mike Perham
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-02-04 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.9
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rsolr
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.1
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.10
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: em-http-request
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.2.6
+  type: :runtime
+  version_requirements: *id004
+description: Provides asynchronous connections to Solr
+email: goodieboy@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- LICENSE
+- README.rdoc
+- VERSION
+- examples/bulk_indexer.rb
+- lib/rsolr-async.rb
+- spec/rsolr-async_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- Rakefile
+homepage: http://github.com/mwmitchell/rsolr-async
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.6.1
+signing_key: 
+specification_version: 3
+summary: An EventMachine based connection adapter for RSolr
+test_files: 
+- spec/rsolr-async_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- Rakefile