harvestdor-indexer 0.0.11 → 0.0.13

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YjA3NGNjNzhkOGQ1NTcxMTEyOWIyNGJkY2NlYTFmNzhhN2U0NGUyZQ==
-  data.tar.gz: !binary |-
-    YzU0NzdkMGIyN2I4YjgyOGJmMGMyMDcyZWQyNjNkZjc3M2ZmNmFiYw==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    ZmQyZjkwYThhN2MzNGZjZGJmNTU0NjNjN2FiMWI0YTBmZjJlN2M0Y2UwNDU5
-    YmQzZGI0MjZkOWM0ZGU4MmY5OGFjZTA5YTE2M2Y0ZjUzOGM5ZmNiMTI2YWFl
-    MzUwODUyOTdiMTE5ZmZiYWE5Nzg0MzRkOTRiMGM2ZDYxY2NmZjk=
-  data.tar.gz: !binary |-
-    MmE0YTVhMjdjNjQ3YzViNTk4NTI2ZWYzYjU1ZTFjMWYzNTU0NWM3YWRlMmMz
-    N2YwMzI4ZDM2NmY1ZGVkYjRmYzczNWE2NDQwYzFkYmJiYmM3Mzk3Nzk1ZjZk
-    MTA0ZDA4YTBlZDIzNjQzNjg4ODZkYTcwZjhiMTBiNTg2NzI1ZWM=
+SHA1:
+  metadata.gz: a14d39d89afa8e167b50facf06cf7cf0a6c5fa76
+  data.tar.gz: 63478e959742b0640189aaf90b2f185315b4b9c3
+SHA512:
+  metadata.gz: eb6f6830232b5d7609a29779c8ca58139cbad499d5c51daa9bfeb0252fe52fc633296a360ccd5e2e9c7b2b68313402c7fb9b6e4532a414a2d130c3f0fe867a69
+  data.tar.gz: f97897beffaebca548732810ed0d3281b169feead5203aaa9ec1a63167cdbf4813a5619052b9ab1cdf8f2fc54bb5068d827487ec9e668b27a1f8b85acd375552

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+script: rake rspec
+rvm:
+  - 2.1.1
+  - 2.0.0
+  - 1.9.3
+  - jruby-19mode # JRuby in 1.9 mode
+notifications:
+  email:
+    - ndushay@stanford.edu
+    - laneymcg@stanford.edu
+    - bess@stanford.edu

data/Gemfile

@@ -1,5 +1,4 @@
 source 'https://rubygems.org'
-source "http://sul-gems.stanford.edu"
 
 # See harvestdor-indexer.gemspec for this gem's dependencies
 gemspec

data/README.rdoc

@@ -1,4 +1,9 @@
 = Harvestdor::Indexer
+{<img src="https://travis-ci.org/sul-dlss/harvestdor-indexer.svg" alt="Build Status" />}[https://travis-ci.org/sul-dlss/harvestdor-indexer]
+{<img src="https://coveralls.io/repos/sul-dlss/harvestdor-indexer/badge.png" alt="Coverage Status" />}[https://coveralls.io/r/sul-dlss/harvestdor-indexer]{<img src="https://gemnasium.com/sul-dlss/harvestdor-indexer.svg" alt="Dependency Status" />}[https://gemnasium.com/sul-dlss/harvestdor-indexer]{<img src="https://badge.fury.io/rb/harvestdor-indexer.svg" alt="Gem Version" />}[http://badge.fury.io/rb/harvestdor-indexer]
+
+
+
 
 A Gem to harvest meta/data from DOR and the skeleton code to index it and write to Solr.
 
@@ -24,13 +29,22 @@ You must override the index method and provide configuration options.  It is rec
 
 Create a yml config file for your collection going to a Solr index.  
 
-See  spec/config/ap.yml for an example.
+Note: Because of an update to underlying HTTP libraries, versions of this gem > 0.0.12 require an updated syntax. Errors like "unknown method timeout" might be because you're using an older version of a config file. The new configuration looks like this:
+
+  http_options:
+    ssl:
+      verify: false
+    # timeouts are in seconds;  timeout -> open/read, open_timeout -> connection open
+    request:
+      timeout: 180
+      open_timeout: 180
 
+See  spec/config/ap.yml for an example.
 You will want to copy that file and change the following settings:
 1. log_name
 2. default_set (in OAI harvesting params section)
-2a. other OAI harvesting params
-3. blacklist or whitelist if you are using them
+3. other OAI harvesting params
+4. blacklist or whitelist if you are using them
 
 You can also pass in non-default configurations as a hash
 
@@ -68,27 +82,23 @@ end
 
 I suggest you write a script to run the code.  Your script might look like this:
 
-	#!/usr/bin/env ruby
-
-	$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
-	$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-
-	require 'rubygems'
-	begin
-	  require 'your_indexer'
-	rescue LoadError
-	  require 'bundler/setup'
-	  require 'your_indexer'
-	end
-
-	config_yml_path = ARGV.pop
-	if config_yml_path.nil?
-	  puts "** You must provide the full path to a config yml file **"
-	  exit
-	end
-  
-	indexer = Harvestdor::Indexer.new(config_yml_path, opts)
-	indexer.harvest_and_index
+  #!/usr/bin/env ruby
+  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
+  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+  require 'rubygems'
+    begin
+      require 'your_indexer'
+    rescue LoadError
+      require 'bundler/setup'
+      require 'your_indexer'
+    end
+  config_yml_path = ARGV.pop
+  if config_yml_path.nil?
+    puts "** You must provide the full path to a config yml file **"
+    exit
+  end
+  indexer = Harvestdor::Indexer.new(config_yml_path, opts)
+  indexer.harvest_and_index
 
 Then you run the script like so:
 
@@ -108,6 +118,9 @@ I suggest you run your code on harvestdor-dev, as it is already set up to be abl
 
 == Releases
 
+* <b>0.0.13</b> Upgrade to latest faraday HTTP client syntax; Use retries gem (https://github.com/ooyala/retries) to make retrying of index process more robust
+* <b>0.0.12</b> fix total_object nil error
+* <b>0.0.11</b> fix error_count and success_count, allow setting of max-tries (retry solr add if error)
 * <b>0.0.7</b> adding additional logging of error, success counts, and time to index and harvest
 * <b>0.0.6</b> tweak error handling for public xml pieces
 * <b>0.0.5</b> make rake release a no-op

data/Rakefile

@@ -7,9 +7,6 @@ require 'rspec/core/rake_task'
 require 'yard'
 require 'yard/rake/yardoc_task'
 
-require 'dlss/rake/dlss_release'
-Dlss::Release.new
-
 begin
   Bundler.setup(:default, :development)
 rescue Bundler::BundlerError => e
@@ -18,10 +15,6 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 
-desc "DO NOT USE! use dlss_release"
-task :release do
-end
-
 task :default => :ci  
 
 desc "run continuous integration suite (tests, coverage, docs)" 

data/harvestdor-indexer.gemspec

@@ -18,9 +18,8 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   
   gem.add_dependency 'rsolr'
-
-  # sul-gems
-  gem.add_dependency 'harvestdor'
+  gem.add_dependency 'retries'
+  gem.add_dependency 'harvestdor', '>=0.0.14'
   gem.add_dependency 'stanford-mods'
   
   # Runtime dependencies
@@ -29,15 +28,13 @@ Gem::Specification.new do |gem|
   # Development dependencies
   # Bundler will install these gems too if you've checked out solrmarc-wrapper source from git and run 'bundle install'
   # It will not add these as dependencies if you require solrmarc-wrapper for other projects
-  gem.add_development_dependency "lyberteam-gems-devel", ">= 1.0"
   gem.add_development_dependency "rake"
   # docs
   gem.add_development_dependency "rdoc"
   gem.add_development_dependency "yard"
   # tests
 	gem.add_development_dependency 'rspec'
-	gem.add_development_dependency 'simplecov'
-	gem.add_development_dependency 'simplecov-rcov'
+	gem.add_development_dependency 'coveralls'
 	# gem.add_development_dependency 'ruby-debug19'
   
 end

data/lib/harvestdor-indexer.rb

@@ -1,6 +1,7 @@
 # external gems
 require 'confstruct'
 require 'rsolr'
+require 'retries'
 
 # sul-dlss gems
 require 'harvestdor'
@@ -21,7 +22,7 @@ module Harvestdor
     def initialize yml_path, options = {}
       @success_count=0    # the number of objects successfully indexed
       @error_count=0      # the number of objects that failed
-      @max_retries=5      # the number of times to retry an object 
+      @max_retries=10      # the number of times to retry an object 
       @total_time_to_solr=0
       @total_time_to_parse=0
       @yml_path = yml_path
@@ -54,13 +55,13 @@ module Harvestdor
       total_time=elapsed_time(start_time)
       total_objects=@success_count+@error_count
       logger.info("Finished harvest_and_index at #{Time.now}: final Solr commit returned")
-      logger.info("Total elapsed time for harvest and index: #{(total_time/60.0)} minutes")
-      logger.info("Avg solr commit time per object (successful): #{@total_time_to_solr/@success_count} seconds") unless (@total_time_to_solr == 0 || @success_count == 0)
-      logger.info("Avg solr commit time per object (all): #{@total_time_to_solr/total_objects} seconds") unless (@total_time_to_solr == 0 || @error_count == 0 || total_objects == 0)
-      logger.info("Avg parse time per object (successful): #{@total_time_to_parse/@success_count} seconds") unless (@total_time_to_parse == 0 || @success_count == 0)
-      logger.info("Avg parse time per object (all): #{@total_time_to_parse/total_objects} seconds") unless (@total_time_to_parse == 0 || @error_count == 0 || total_objects == 0)
-      logger.info("Avg complete index time per object (successful): #{total_time/@success_count} seconds") unless (@success_count == 0)
-      logger.info("Avg complete index time per object (all): #{total_time/total_objects} seconds") unless (@error_count == 0 || total_object == 0)
+      logger.info("Total elapsed time for harvest and index: #{(total_time/60.0).round(2)} minutes")
+      logger.info("Avg solr commit time per object (successful): #{(@total_time_to_solr/@success_count).round(2)} seconds") unless (@total_time_to_solr == 0 || @success_count == 0)
+      logger.info("Avg solr commit time per object (all): #{(@total_time_to_solr/total_objects).round(2)} seconds") unless (@total_time_to_solr == 0 || @error_count == 0 || total_objects == 0)
+      logger.info("Avg parse time per object (successful): #{(@total_time_to_parse/@success_count).round(2)} seconds") unless (@total_time_to_parse == 0 || @success_count == 0)
+      logger.info("Avg parse time per object (all): #{(@total_time_to_parse/total_objects).round(2)} seconds") unless (@total_time_to_parse == 0 || @error_count == 0 || total_objects == 0)
+      logger.info("Avg complete index time per object (successful): #{(total_time/@success_count).round(2)} seconds") unless (@success_count == 0)
+      logger.info("Avg complete index time per object (all): #{(total_time/total_objects).round(2)} seconds") unless (@error_count == 0 || total_objects == 0)
       logger.info("Successful count: #{@success_count}")
       logger.info("Error count: #{@error_count}")
       logger.info("Total records processed: #{total_objects}")
@@ -78,27 +79,22 @@ module Harvestdor
       return @druids
     end
 
-    #add the document to solr, retry if an error occurs
-    def solr_add(doc, id, do_retry=true)
-      #if do_retry is false, skip retrying 
-      tries=do_retry ? 0 : 999
-      max_tries=@max_retries ? @max_retries : 5 #if @max_retries isn't set, use 5
-      while tries < max_tries
-      begin
-        tries+=1
-        solr_client.add(doc)
-        #return if successful
-        return
-      rescue => e
-        if tries<max_tries
-          logger.warn "#{id}: #{e.message}, retrying"
-        else
-          @error_count+=1
-          logger.error "Failed saving #{id}: #{e.message}"
-          logger.error e.backtrace
-          return
-        end
+    # Add the document to solr, retry if an error occurs.
+    # See https://github.com/ooyala/retries for docs on with_retries.
+    # @param [Hash] doc a Hash representation of the solr document
+    # @param [String] id the id of the document being sent, for logging
+    def solr_add(doc, id)
+      max_tries=@max_retries ? @max_retries : 10 #if @max_retries isn't set, use 10
+      
+      handler = Proc.new do |exception, attempt_number, total_delay|
+        logger.debug "#{exception.class} on attempt #{attempt_number} for #{id}"
+        # logger.debug exception.backtrace
       end
+      
+      with_retries(:max_tries => max_tries, :handler => handler, :base_sleep_seconds => 1, :max_sleep_seconds => 5) do |attempt|
+        logger.debug "Attempt #{attempt} for #{id}"
+        solr_client.add(doc)
+        logger.info "Successfully indexed #{id} on attempt #{attempt}"
       end
     end
 

data/lib/harvestdor-indexer/version.rb

@@ -1,6 +1,6 @@
 module Harvestdor
   class Indexer
     # this is the Ruby Gem version
-    VERSION = "0.0.11"
+    VERSION = "0.0.13"
   end
 end

data/spec/config/ap.yml

@@ -57,5 +57,6 @@ http_options:
   ssl:
     verify: false
   # timeouts are in seconds;  timeout -> open/read, open_timeout -> connection open
-  timeout: 180
-  open_timeout: 180
+  request:
+    timeout: 180
+    open_timeout: 180

data/spec/spec_helper.rb

@@ -1,16 +1,5 @@
-# for test coverage
-require 'simplecov'
-require 'simplecov-rcov'
-class SimpleCov::Formatter::MergedFormatter
-  def format(result)
-     SimpleCov::Formatter::HTMLFormatter.new.format(result)
-     SimpleCov::Formatter::RcovFormatter.new.format(result)
-  end
-end
-SimpleCov.formatter = SimpleCov::Formatter::MergedFormatter
-SimpleCov.start do
-  add_filter "/spec/"
-end
+require 'coveralls'
+Coveralls.wear!
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))

data/spec/unit/harvestdor-indexer_spec.rb

@@ -13,6 +13,24 @@ describe Harvestdor::Indexer do
     @whitelist_path = File.join(File.dirname(__FILE__), "../config/ap_whitelist.txt")
   end
   
+  # The method that sends the solr document to solr
+  describe "#solr_add" do
+    before(:each) do
+      doc_hash = {
+        :modsxml => 'whatever',
+        :title_display => 'title',
+        :pub_year_tisim => 'some year',
+        :author_person_display => 'author',
+        :format => 'Image',
+        :language => 'English'
+      }
+    end
+    it "sends an add request to the solr_client" do
+      expect(@indexer.solr_client).to receive(:add)
+      @indexer.solr_add(@doc_hash, "abc123")
+    end
+  end
+  
   describe "access methods" do
     it "initializes success count" do
       @indexer.success_count.should == 0
@@ -21,7 +39,7 @@ describe Harvestdor::Indexer do
       @indexer.error_count.should == 0
     end
     it "initializes max_retries" do
-      @indexer.max_retries.should == 5
+      expect(@indexer.max_retries).to eql(10)
     end
     it "allows overriding of max_retries" do
       @indexer.max_retries=6

metadata

@@ -1,167 +1,153 @@
 --- !ruby/object:Gem::Specification
 name: harvestdor-indexer
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.13
 platform: ruby
 authors:
 - Naomi Dushay
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-18 00:00:00.000000000 Z
+date: 2014-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rsolr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: harvestdor
+  name: retries
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: stanford-mods
+  name: harvestdor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.14
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.14
 - !ruby/object:Gem::Dependency
-  name: confstruct
+  name: stanford-mods
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: lyberteam-gems-devel
+  name: confstruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: simplecov-rcov
+  name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Harvest DOR object metadata via a relationship (e.g. hydra:isGovernedBy
@@ -173,8 +159,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .yardopts
+- ".gitignore"
+- ".travis.yml"
+- ".yardopts"
 - Gemfile
 - LICENSE.txt
 - README.rdoc
@@ -196,17 +183,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.7
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Harvest DOR object metadata and index it to Solr