harvestdor-indexer 1.0.4 → 2.0.0

data/lib/harvestdor/indexer/metrics.rb

@@ -0,0 +1,53 @@
+module Harvestdor
+  ##
+  # Harvest metrics tracker
+  class Indexer::Metrics
+    attr_accessor :error_count, :success_count, :logger
+
+    def initialize options = {}
+      @success_count=0    # the number of objects successfully indexed
+      @error_count=0      # the number of objects that failed
+      @logger = options[:logger] || Logger.new(STDERR)
+    end
+
+    ##
+    # Wrap an operation in tally block; if the block completes without throwing
+    # an exception, tally a success. If the block throws an exception, catch it 
+    # and tally a failure.
+    #
+    # Callers can provide an :on_error handler to receive the exception and process
+    # it appropriately.
+    #
+    # @param [Hash] options
+    # @option options [#call] Callback that will receive any exception thrown by the block
+    def tally options = {}, &block      
+      begin
+        block.call
+        success!
+      rescue => e
+        error!
+        logger.error "Failed to process: #{e.message}"
+        options[:on_error].call e if options[:on_error]
+      end
+    end
+
+    ##
+    # Record a successful run
+    def success!
+      @success_count += 1
+    end
+
+    ##
+    # Record an error
+    def error!
+      @error_count += 1
+    end
+
+    ##
+    # Total number of runs
+    def total
+      @success_count + @error_count
+    end
+
+  end
+end

data/lib/harvestdor/indexer/resource.rb

@@ -0,0 +1,174 @@
+require 'active_support/benchmarkable'
+
+module Harvestdor
+  class Indexer::Resource
+    include ActiveSupport::Benchmarkable
+
+    attr_reader :indexer, :druid, :options
+  
+    def initialize indexer, druid, options = {}
+      @indexer = indexer
+      @druid = druid
+      @options = options
+    end
+
+    def bare_druid
+      @bare_druid ||= druid.gsub("druid:", "")
+    end
+
+    ##
+    # The harvestdor client used for retrieving resources
+    def harvestdor_client
+      indexer.harvestdor_client
+    end
+
+    def dor_fetcher_client
+      indexer.dor_fetcher_client
+    end
+
+    ##
+    # Get the logger
+    def logger
+      options[:logger] || (indexer.logger if indexer.respond_to? :logger) || Logger.new(STDERR)
+    end
+
+    ##
+    # Is this resource a collection?
+    def collection?
+      identity_metadata.xpath("/identityMetadata/objectType").any? { |x| x.text == "collection" }
+    end
+
+    # get the druids from isMemberOfCollection relationships in rels-ext from public_xml
+    # @return [Array<String>] the druids (e.g. ww123yy1234) this object has isMemberOfColletion relationship with, or nil if none
+    def collections
+      @collections ||= begin
+        ns_hash = {'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', 'fedora' => "info:fedora/fedora-system:def/relations-external#", '' => ''}
+        is_member_of_nodes ||= public_xml.xpath('/publicObject/rdf:RDF/rdf:Description/fedora:isMemberOfCollection/@rdf:resource', ns_hash)
+
+        is_member_of_nodes.reject { |n| n.value.empty? }.map do |n| 
+          Harvestdor::Indexer::Resource.new(indexer, n.value.gsub("info:fedora/", ""))
+        end
+      end
+    end
+
+    ##
+    # Return the items in this collection
+    def items
+      @items ||= begin
+        druids = dor_fetcher_client.druid_array(dor_fetcher_client.get_collection(bare_druid, {}))
+        druids.map { |x| Harvestdor::Indexer::Resource.new(indexer, x) }
+      end
+    end
+    
+    # given a druid, get its objectLabel from its purl page identityMetadata
+    # @param [String] druid, e.g. ab123cd4567
+    # @return [String] the value of the <objectLabel> element in the identityMetadata for the object
+    def identity_md_obj_label
+      logger.error("#{druid} missing identityMetadata") unless identity_metadata
+      identity_metadata.xpath('identityMetadata/objectLabel').text
+    end
+    
+    
+    # return the MODS for the druid as a Stanford::Mods::Record object
+    # @param [String] druid e.g. ab123cd4567
+    # @return [Stanford::Mods::Record] created from the MODS xml for the druid
+    def smods_rec
+      @smods_rec ||= benchmark "smods_rec(#{druid})", level: :debug do
+        ng_doc = mods
+        raise "Empty MODS metadata for #{druid}: #{ng_doc.to_xml}" if ng_doc.root.xpath('//text()').empty?
+        mods_rec = Stanford::Mods::Record.new
+        mods_rec.from_nk_node(ng_doc.root)
+        mods_rec
+      end
+    end
+
+    def mods
+      @mods ||= harvestdor_client.mods bare_druid
+    end
+    
+    # the public xml for this DOR object, from the purl page
+    # @param [String] druid e.g. ab123cd4567
+    # @return [Nokogiri::XML::Document] the public xml for the DOR object
+    def public_xml
+      @public_xml ||= benchmark "public_xml(#{druid})", level: :debug do
+        ng_doc = harvestdor_client.public_xml bare_druid
+        raise "No public xml for #{druid}" if !ng_doc
+        raise "Empty public xml for #{druid}: #{ng_doc.to_xml}" if ng_doc.root.xpath('//text()').empty?
+        ng_doc
+      end
+    end
+
+    ##
+    # Has the public_xml been previously retrieved?
+    def public_xml?
+      !!@public_xml
+    end
+
+    ##
+    # Get the public_xml, if retrieved, or the druid. This is used to short-circuit
+    # retrieving metadata out of the public xml.
+    def public_xml_or_druid
+      if public_xml?
+        public_xml
+      else
+        bare_druid
+      end
+    end
+    
+    # the contentMetadata for this DOR object, ultimately from the purl public xml
+    # @param [Object] object a String containing a druid (e.g. ab123cd4567), or 
+    #  a Nokogiri::XML::Document containing the public_xml for an object
+    # @return [Nokogiri::XML::Document] the contentMetadata for the DOR object
+    def content_metadata
+      ng_doc = benchmark "content_metadata (#{druid})", level: :debug do
+        harvestdor_client.content_metadata public_xml_or_druid
+      end
+      raise "No contentMetadata for \"#{druid}\"" if !ng_doc || ng_doc.children.empty?
+      ng_doc
+    end
+    
+    # the identityMetadata for this DOR object, ultimately from the purl public xml
+    # @param [Object] object a String containing a druid (e.g. ab123cd4567), or 
+    #  a Nokogiri::XML::Document containing the public_xml for an object
+    # @return [Nokogiri::XML::Document] the identityMetadata for the DOR object
+    def identity_metadata
+      ng_doc = benchmark "identity_metadata (#{druid})", level: :debug do
+        harvestdor_client.identity_metadata public_xml_or_druid
+      end
+      raise "No identityMetadata for \"#{druid}\"" if !ng_doc || ng_doc.children.empty?
+      ng_doc
+    end
+    
+    # the rightsMetadata for this DOR object, ultimately from the purl public xml
+    # @param [Object] object a String containing a druid (e.g. ab123cd4567), or 
+    #  a Nokogiri::XML::Document containing the public_xml for an object
+    # @return [Nokogiri::XML::Document] the rightsMetadata for the DOR object
+    def rights_metadata
+      ng_doc = benchmark "rights_metadata (#{druid})", level: :debug do
+        harvestdor_client.rights_metadata public_xml_or_druid
+      end
+      raise "No rightsMetadata for \"#{druid}\"" if !ng_doc || ng_doc.children.empty?
+      ng_doc
+    end
+    
+    # the RDF for this DOR object, ultimately from the purl public xml
+    # @param [Object] object a String containing a druid (e.g. ab123cd4567), or 
+    #  a Nokogiri::XML::Document containing the public_xml for an object
+    # @return [Nokogiri::XML::Document] the RDF for the DOR object
+    def rdf
+      ng_doc = benchmark "rdf (#{druid})", level: :debug do
+        harvestdor_client.rdf public_xml_or_druid
+      end
+      raise "No RDF for \"#{druid}\"" if !ng_doc || ng_doc.children.empty?
+      ng_doc
+    end
+
+    def eql?(other)
+      other.is_a? Harvestdor::Indexer::Resource and other.indexer == indexer and other.druid == druid
+    end
+
+    def hash
+      druid.hash ^ indexer.hash
+    end
+  end
+end

data/lib/harvestdor/indexer/solr.rb

@@ -0,0 +1,39 @@
+module Harvestdor
+  class Indexer::Solr
+    attr_accessor :client, :indexer, :config
+
+    def initialize indexer, config = {}
+      @indexer = indexer
+      @client = RSolr.connect(config)
+      @config = Confstruct::Configuration.new config
+      @config.max_retries ||= 10
+    end
+    
+    def logger
+      indexer.logger
+    end
+
+    def commit!
+      client.commit
+    end
+    
+    # Add the document to solr, retry if an error occurs.
+    # See https://github.com/ooyala/retries for docs on with_retries.
+    # @param [Hash] doc a Hash representation of the solr document
+    def add(doc)
+      id = doc[:id]
+      
+      handler = Proc.new do |exception, attempt_number, total_delay|
+        logger.debug "#{exception.class} on attempt #{attempt_number} for #{id}"
+        # logger.debug exception.backtrace
+      end
+      
+      with_retries(:max_tries => config.max_retries, :handler => handler, :base_sleep_seconds => 1, :max_sleep_seconds => 5) do |attempt|
+        logger.debug "Attempt #{attempt} for #{id}"
+        client.add(doc)
+        logger.info "Successfully indexed #{id} on attempt #{attempt}"
+      end
+    end
+    
+  end
+end

data/lib/{harvestdor-indexer → harvestdor/indexer}/version.rb

@@ -1,6 +1,6 @@
 module Harvestdor
   class Indexer
     # this is the Ruby Gem version
-    VERSION = "1.0.4"
+    VERSION = "2.0.0"
   end
 end

data/spec/config/ap.yml

@@ -1,28 +1,6 @@
-# You will want to copy this file and change the following settings:
-# 1. log_dir, log_name
-# 2. default_set
-# 3. blacklist or whitelist if you are using them
-# 4. Solr baseurl
-
-# log_dir:  directory for log file  (default logs, relative to harvestdor gem path)
-log_dir: spec/test_logs  
-
-# log_name: name of log file  (default: harvestdor.log)
-log_name: ap-test.log
-
-# purl: url for the DOR purl server (used to get ContentMetadata, etc.)
-purl: http://purl.stanford.edu
-
-# ---------- White and Black list parameters -----
-
-# name of file containing druids that will NOT be processed even if they are harvested 
-#  via DorFetcher either give absolute path or path relative to where the command will 
-#  be executed
-#blacklist: config/ap_blacklist.txt
-
-# name of file containing druids that WILL be processed (all others will be ignored)
-#  either give absolute path or path relative to where the command will be executed
-#whitelist: config/ap_whitelist.txt
+dor_fetcher:
+  service_url: http://127.0.0.1:3000
+  skip_heartbeat: true
 
 # ----------- SOLR index (that we're writing INTO) parameters ------------
 solr:
@@ -31,25 +9,35 @@ solr:
   # timeouts are in seconds;  read_timeout -> open/read, open_timeout -> connection open
   read_timeout: 60
   open_timeout: 60
+  max_retries: 10
 
-# default_set:  default set for harvest  (default: nil)
-#   can be overridden on calls to harvest_ids and harvest_records
-default_set: is_governed_by_yg867hg1375
-
-# default_metadata_prefix:  default metadata prefix to be used for harvesting  (default: mods)
-#   can be overridden on calls to harvest_ids and harvest_records
-
-# default_from_date:  default from date for harvest  (default: nil)
-#   can be overridden on calls to harvest_ids and harvest_records
+# ---------- White and Black list parameters -----
 
-# default_until_date:  default until date for harvest  (default: nil)
-#   can be overridden on calls to harvest_ids and harvest_records
+# name of file containing druids that WILL be processed (all others will be ignored)
+#  either give absolute path or path relative to where the command will be executed
+#whitelist: config/ap_whitelist.txt
 
-# Additional options to pass to Faraday http client (https://github.com/technoweenie/faraday)
-http_options:
-  ssl:
-    verify: false
-  # timeouts are in seconds;  timeout -> open/read, open_timeout -> connection open
-  request:
-    timeout: 180
-    open_timeout: 180
+harvestdor:
+  # You will want to copy this file and change the following settings:
+  # 1. log_dir, log_name
+  # 2. default_set
+  # 3. blacklist or whitelist if you are using them
+  # 4. Solr baseurl
+
+  # log_dir:  directory for log file  (default logs, relative to harvestdor gem path)
+  log_dir: spec/test_logs  
+
+  # log_name: name of log file  (default: harvestdor.log)
+  log_name: ap-test.log
+
+  # purl: url for the DOR purl server (used to get ContentMetadata, etc.)
+  purl: http://purl.stanford.edu
+
+  # Additional options to pass to Faraday http client (https://github.com/technoweenie/faraday)
+  http_options:
+    ssl:
+      verify: false
+    # timeouts are in seconds;  timeout -> open/read, open_timeout -> connection open
+    request:
+      timeout: 180
+      open_timeout: 180

data/spec/fixtures/vcr_cassettes/get_collection_druids_call.yml

@@ -53,4 +53,100 @@ http_interactions:
         social customs, and people."}],"counts":{"collections":1,"items":5,"total_count":6}}'
     http_version:
   recorded_at: Wed, 12 Nov 2014 19:34:03 GMT
+- request:
+    method: get
+    uri: http://purl.stanford.edu/yg867hg1375.xml
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Date:
+      - Wed, 17 Dec 2014 19:39:37 GMT
+      Server:
+      - Apache/2.2.15 (Red Hat)
+      X-Powered-By:
+      - Phusion Passenger (mod_rails/mod_rack) 3.0.19
+      X-Ua-Compatible:
+      - IE=Edge,chrome=1
+      Etag:
+      - '"67aa6d1481ba1537ae63af5aaf493f84"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Request-Id:
+      - f2e753d56bf896cde6e941be0f51d05a
+      X-Runtime:
+      - '0.007983'
+      X-Rack-Cache:
+      - miss
+      Status:
+      - '200'
+      Content-Length:
+      - '2180'
+      Content-Type:
+      - application/xml; charset=utf-8
+    body:
+      encoding: UTF-8
+      string: |
+        <publicObject id="druid:yg867hg1375" published="2013-11-11T15:34:32-08:00">
+          <identityMetadata>
+            <objectId>druid:yg867hg1375</objectId>
+            <objectCreator>DOR</objectCreator>
+            <objectLabel>Francis E. Stafford photographs, 1909-1933</objectLabel>
+            <objectType>collection</objectType>
+            <adminPolicy>druid:vb546ms7107</adminPolicy>
+            <otherId name="catkey">9615156</otherId>
+            <otherId name="uuid">8f1feb20-4b29-11e3-8e31-0050569b3c3c</otherId>
+            <tag>Remediated By : 3.25.3</tag>
+          </identityMetadata>
+          <xml/>
+          <rightsMetadata>
+            <access type="discover">
+              <machine>
+                <world/>
+              </machine>
+            </access>
+            <access type="read">
+              <machine>
+                <world/>
+              </machine>
+            </access>
+            <use>
+              <human type="useAndReproduction"/>
+              <human type="creativeCommons"/>
+              <machine type="creativeCommons"/>
+            </use>
+            <copyright>
+              <human/>
+            </copyright>
+          </rightsMetadata>
+          <rdf:RDF xmlns:fedora-model="info:fedora/fedora-system:def/model#" xmlns:hydra="http://projecthydra.org/ns/relations#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+            <rdf:Description rdf:about="info:fedora/druid:yg867hg1375">
+            </rdf:Description>
+          </rdf:RDF>
+          <oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:srw_dc="info:srw/schema/1/dc-schema" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+            <dc:title>Francis E. Stafford photographs, 1909-1933</dc:title>
+            <dc:contributor>Stafford, Francis E., 1884-1938</dc:contributor>
+            <dc:type>Collection</dc:type>
+            <dc:date>1909-1933</dc:date>
+            <dc:language>und</dc:language>
+            <dc:format>3 oversize boxes.</dc:format>
+            <dc:description>Photographs of scenes in China, mainly between 1909 and 1915.</dc:description>
+            <dc:rights>Closed. Digital use copies available.</dc:rights>
+            <dc:description type="biographical/historical">American missionary in China, 1909-1915 and 1932-1933.</dc:description>
+            <dc:coverage>China</dc:coverage>
+          </oai_dc:dc>
+        </publicObject>
+    http_version: 
+  recorded_at: Wed, 17 Dec 2014 19:39:38 GMT
 recorded_with: VCR 2.9.3