solrizer-rabbit 0.0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+*.swp

data/.rvmrc

@@ -0,0 +1 @@
+rvm use default@solrizer-rabbit

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in solrizer-rabbit.gemspec
+gemspec
+
+gem 'solrizer-fedora', :git=>'git://github.com/projecthydra/solrizer-fedora.git', :ref=>'1dbb815'

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# Solrizer::Rabbit
+
+Solrizer-rabbit is a gem for indexing ActiveFedora objects into solr by using RabbitMQ.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'solrizer-rabbit'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install solrizer-rabbit
+
+## Usage
+
+<code>
+# search fedora for a list of pids, write them into the queue
+rake solrizer:rabbit:enqueue
+
+# read the pids from fedora and index them. threads defaults to 1
+rake solrizer:rabbit:index threads=7
+</code>
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/lib/solrizer-rabbit.rb

@@ -0,0 +1,59 @@
+require "solrizer-rabbit/version"
+
+require "solrizer-fedora"
+require "solrizer-rabbit/queue_index_worker"
+require "solrizer-rabbit/buffered_indexer"
+
+module Solrizer
+  module Rabbit
+    def self.queue_name
+      ENV['queue'] || 'index'
+    end
+
+    def self.enqueue
+      q = Carrot.queue(queue_name)
+      
+      connections.each do |conn|
+        conn.search(nil) do |object|
+          q.publish(object.pid)
+        end
+      end 
+
+      Carrot.stop
+    end
+
+    def self.work
+      worker_count = (ENV['threads'] || 1).to_i
+
+      workers = []
+      threads = []
+      worker_count.times do |n|
+        worker = Solrizer::Rabbit::QueueIndexWorker.new
+        workers << worker
+        threads << Thread.new { worker.run }
+      end
+
+      Signal.trap("INT") { workers.each {|w| w.stop} }
+
+      threads.each do |thread|
+        thread.join
+      end 
+      puts "Sending commit to solr"
+      ActiveFedora::SolrService.instance.conn.commit
+    end
+
+
+    private 
+
+    def self.connections
+      if ActiveFedora.config.sharded?
+        return ActiveFedora.config.credentials.map { |cred| ActiveFedora::RubydoraConnection.new(cred).connection}
+      else
+        return [ActiveFedora::RubydoraConnection.new(ActiveFedora.config.credentials).connection]
+      end
+    end
+  end
+end
+
+load File.join(File.dirname(__FILE__),"tasks/solrizer-rabbit.rake") if defined?(Rake)
+

data/lib/solrizer-rabbit/buffered_indexer.rb

@@ -0,0 +1,89 @@
+module Solrizer::Rabbit
+  class BufferedIndexer 
+    include ActiveSupport::Benchmarkable
+    BUFFER_SIZE = 1000
+    COMMIT_EVERY = 0
+
+
+    def initialize(conn)
+      @count = 0
+      @batch_count = 0
+      @add_buffer = []
+      @delete_buffer = []
+      @solr = conn
+    end
+
+    def flush(commit = false)
+      try_to_add unless @add_buffer.empty?
+      @add_buffer = []
+      try_to_delete unless @delete_buffer.empty?
+      @delete_buffer = []
+      @count = 0
+      maybe_commit()
+    end
+
+    def try_to_add
+      tries = 0
+      begin 
+        benchmark "#{$$} -- #{Rails.env} solr add" do
+          solr.add @add_buffer
+        end
+      rescue TimeoutError 
+        ## The timeout is set in this parameter.  It is 60 seconds by default.
+        # rsolr.connection.connection.read_timeout = 60
+        tries += 1 
+        puts "Timeout #{tries}" 
+        sleep(10 * tries) # wait a little longer each time through
+        retry if tries < 5 
+        raise "Adding docs timed out #{tries} times. Qutting." 
+      end
+    end
+
+    def try_to_delete
+      tries = 0
+      begin 
+        benchmark "#{$$} -- #{Rails.env} solr delete" do
+          solr.delete_by_id @delete_buffer
+        end
+      rescue TimeoutError 
+        ## The timeout is set in this parameter.  It is 60 seconds by default.
+        # rsolr.connection.connection.read_timeout = 60
+        tries += 1 
+        puts "Timeout #{tries}" 
+        sleep(10 * tries) # wait a little longer each time through
+        retry if tries < 5 
+        raise "Adding docs timed out #{tries} times. Qutting." 
+      end
+    end
+
+    def maybe_commit(force=false)
+      return if COMMIT_EVERY == 0 && !force
+      @batch_count += 1
+      if force || @batch_count > COMMIT_EVERY
+        solr.commit
+        @batch_count =0
+      end
+    end
+
+    def add(doc)
+      @add_buffer << doc
+      increment
+    end
+    def delete_by_id(doc)
+      @delete_buffer << doc
+      increment
+    end
+
+    private
+
+    def increment
+      @count += 1
+      flush if @count >= BUFFER_SIZE
+    end
+
+    def solr
+      @solr
+    end
+  end
+end
+

data/lib/solrizer-rabbit/queue_index_worker.rb

@@ -0,0 +1,43 @@
+require 'carrot'
+module Solrizer
+  module Rabbit
+    class QueueIndexWorker
+      def initialize
+        Thread.current[:carrot] = Carrot.new()#:host=>'mediashelf.eu')
+        @q = Carrot.queue(Solrizer::Rabbit.queue_name)
+
+        @buff = BufferedIndexer.new(ActiveFedora::SolrService.instance.conn)
+        @stopped = false
+      end
+
+      def stop
+        puts "finishing writes"
+        @stopped=true
+      end
+
+      def run
+        indexer = Solrizer::Fedora::Indexer.new
+        while !@stopped && msg = @q.pop
+          begin
+            obj = Solrizer::Fedora::Repository.get_object(msg)
+            solr_doc = indexer.create_document( obj )
+            @buff.add(solr_doc)
+          rescue RSolr::Error::Http, Errno::ECONNREFUSED => exception
+            puts "Fatal #{exception.class}, exception see log"
+            logger.fatal( "\n\n#{exception.class} (#{exception.message})\n\n")
+            logger.flush if logger.respond_to? :flush #Rails logger is flushable, mediashelf-loggable isn't
+            exit!
+          rescue StandardError => exception
+            puts "Caught #{exception.class}, while procesing `#{msg}` see log"
+            logger.fatal( "\n\n#{exception.class} (#{exception.message}) while procesing `#{msg}`:\n    " + exception.backtrace.join("\n    ") + "\n\n")
+            logger.flush if logger.respond_to? :flush #Rails logger is flushable, mediashelf-loggable isn't
+          end
+        end
+        puts "flushing buffers"
+        @buff.flush(true)
+        Carrot.stop
+        puts "done"
+      end
+    end
+  end
+end

data/lib/solrizer-rabbit/sharded_indexer.rb

@@ -0,0 +1,33 @@
+module Solrizer::Rabbit
+  class ShardedIndexer
+    attr_accessor :shards
+    def initialize
+      self.shards = YAML.load_file(Rails.root + 'config/shards.yml')[Rails.env]
+      @buffers = []
+      shards.each do |conf|
+        @buffers << DTU::BufferedIndexer.new(RSolr.connect(:url=>conf))
+      end
+    end
+
+    def add(doc)
+      buffer(doc['id']).add(doc)
+    end
+
+    def delete(doc)
+      buffer(doc['id']).delete_by_id(doc['id'])
+    end
+
+    def flush(commit = false)
+      @buffers.each {|b| b.flush(commit)}
+    end
+
+    def buffer(id)
+      raise "No id" unless id
+      n = Digest::MD5.hexdigest(id.to_s).hex % @buffers.count
+      @buffers[n]
+    end
+
+  end
+end
+
+

data/lib/solrizer-rabbit/version.rb

@@ -0,0 +1,5 @@
+module Solrizer
+  module Rabbit
+    VERSION = "0.0.1"
+  end
+end

data/lib/tasks/solrizer-rabbit.rake

@@ -0,0 +1,14 @@
+namespace :solrizer do
+  namespace :rabbit do
+    desc "Enqueue all pids"
+    task :enqueue => :environment do
+      Solrizer::Rabbit.enqueue
+    end
+
+    desc "Run the index worker"
+    task :index => :environment do
+      Solrizer::Rabbit.work
+    end
+  end
+end
+

data/solrizer-rabbit.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/solrizer-rabbit/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Justin Coyne"]
+  gem.email         = ["justin.coyne@yourmediashelf.com"]
+  gem.description   = %q{Solrize fedora objects using a queue}
+  gem.summary       = %q{Solrize fedora objects using a queue}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "solrizer-rabbit"
+  gem.require_paths = ["lib"]
+  gem.version       = Solrizer::Rabbit::VERSION
+
+  gem.add_dependency('solrizer-fedora', '~> 2.1') 
+  gem.add_dependency('carrot') 
+
+  gem.add_development_dependency('rspec')
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+require 'solrizer-rabbit'
+

data/spec/unit/queue_index_worker_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+describe Solrizer::Rabbit::QueueIndexWorker do
+
+  before do
+    @mock_buffer = mock()
+    Solrizer::Rabbit::BufferedIndexer.should_receive(:new).with(kind_of RSolr::Client).and_return(@mock_buffer)
+    @stub_queue = stub('queue')
+    @stub_queue.should_receive(:pop).and_return('foo:123', 'foo:231', nil)
+    Carrot.should_receive(:queue).and_return(@stub_queue)
+  end
+  it "should run" do
+    @mock_buffer.should_receive(:flush)
+    @mock_buffer.should_receive(:add).with("document 1")
+    @mock_buffer.should_receive(:add).with("document 2")
+
+    obj1 = stub('obj1', :pid=>'foo:123')
+    obj2 = stub('obj2', :pid=>'foo:231')
+    @mock_indexer = mock("indexer")
+    @mock_indexer.should_receive(:create_document).with(obj1).and_return("document 1") 
+    @mock_indexer.should_receive(:create_document).with(obj2).and_return("document 2")
+    Solrizer::Fedora::Indexer.should_receive(:new).and_return(@mock_indexer)
+    Solrizer::Fedora::Repository.should_receive(:get_object).with('foo:123').and_return(obj1)
+    Solrizer::Fedora::Repository.should_receive(:get_object).with('foo:231').and_return(obj2)
+    
+    subject.run
+  end
+
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: solrizer-rabbit
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Justin Coyne
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-05-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: solrizer-fedora
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: carrot
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Solrize fedora objects using a queue
+email:
+- justin.coyne@yourmediashelf.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/solrizer-rabbit.rb
+- lib/solrizer-rabbit/buffered_indexer.rb
+- lib/solrizer-rabbit/queue_index_worker.rb
+- lib/solrizer-rabbit/sharded_indexer.rb
+- lib/solrizer-rabbit/version.rb
+- lib/tasks/solrizer-rabbit.rake
+- solrizer-rabbit.gemspec
+- spec/spec_helper.rb
+- spec/unit/queue_index_worker_spec.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Solrize fedora objects using a queue
+test_files:
+- spec/spec_helper.rb
+- spec/unit/queue_index_worker_spec.rb