harvestdor-indexer 2.4.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2615efed3ea7e94441d7457e39cd5bc3503a5cd9
-  data.tar.gz: 508ff570b19083f8f78ea29d3f820c09f698f392
+  metadata.gz: 7c1cecc67cfdfbd0f57a40c1bc667b775143d237
+  data.tar.gz: 7a0452f584a683dd7f9146b5e1d165fa7c276341
 SHA512:
-  metadata.gz: 8960f3c459bd794532ddc94e0d80d31f5f85692f9e08eca0a540054290054dfeee9082c2bc68f9d3ee5e5e516be03b43fdebe8a2fba9834c47a8e21dfc2ea65a
-  data.tar.gz: 10cf83435837099f544094927182de39a2ae624e66cdff7f0477158de923d22cd27c174d5f1d8127ead92df9f4162a3f3fc50a03bc98e7ce68eb59e94393b804
+  metadata.gz: ca9a9d59768d8125528e8770a0251a6b8b9480833264046b462ef7e07fb72ca4fe27ab490e35f00ac08fba14e895506e9d3a55aaa893e2b411f4924e78461997
+  data.tar.gz: 7238bca995fc84045046e3891cd3c45f02e49eccf18fda48622811cd2d76ae53db445dc265087e1d79acc258c958ce05eaf0da0697f49c8c30865cd74af3ac8a

data/.travis.yml

@@ -2,7 +2,7 @@ language: ruby
 sudo: false
 script: rake
 rvm:
-  - 2.2.3
+  - 2.3.1
 
 notifications:
   email:

data/harvestdor-indexer.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'dor-fetcher', '=1.0.5'
   gem.add_dependency 'activesupport'
   gem.add_dependency 'parallel'
+  gem.add_dependency 'faraday'
 
   # Runtime dependencies
   gem.add_runtime_dependency 'confstruct'
@@ -36,8 +37,8 @@ Gem::Specification.new do |gem|
   # tests
   gem.add_development_dependency 'rspec', '~> 3.0'
   gem.add_development_dependency 'coveralls'
-  gem.add_development_dependency 'rubocop'
-  gem.add_development_dependency 'rubocop-rspec'
+  gem.add_development_dependency 'rubocop', '0.42.0'
+  gem.add_development_dependency 'rubocop-rspec', '1.5.1'
   gem.add_development_dependency 'vcr'
   gem.add_development_dependency 'webmock'
 

data/lib/harvestdor/indexer.rb

@@ -20,6 +20,7 @@ module Harvestdor
   # Base class to harvest from DOR via harvestdor gem and then index
   class Indexer
     require 'harvestdor/indexer/metrics'
+    require 'harvestdor/indexer/purl_fetcher'
     require 'harvestdor/indexer/resource'
     require 'harvestdor/indexer/solr'
 
@@ -142,6 +143,12 @@ module Harvestdor
       @dor_fetcher_client ||= DorFetcher::Client.new(config.dor_fetcher)
     end
 
+    def purl_fetcher_client
+      return unless config.purl_fetcher
+
+      @purl_fetcher_client ||= Harvestdor::Indexer::PurlFetcher.new(config.purl_fetcher)
+    end
+
     def solr
       @solr ||= Harvestdor::Indexer::Solr.new self, config.solr.to_hash
     end
@@ -167,4 +174,4 @@ module Harvestdor
       raise msg
     end
   end # Indexer class
-end # Harvestdor module
+end # Harvestdor module

data/lib/harvestdor/indexer/purl_fetcher.rb

@@ -0,0 +1,37 @@
+module Harvestdor
+  class Indexer
+    # Client for working with the PURL Fetcher API
+    class PurlFetcher
+      attr_reader :config
+
+      def initialize(config = {})
+        @config = config
+      end
+
+      def druids_from_collection(collection)
+        return to_enum(:druids_from_collection, collection) unless block_given?
+
+        page = 1
+
+        loop do
+          response = client.get("/collections/#{collection}/purls", page: page, per_page: 100)
+          data = JSON.parse(response.body)
+
+          break if data['purls'].blank?
+
+          data['purls'].each { |d| yield d['druid'] }
+
+          page += 1
+
+          break if data['pages']['next_page'].nil?
+        end
+      end
+
+      private
+
+      def client
+        @client ||= Faraday.new(config)
+      end
+    end
+  end
+end

data/lib/harvestdor/indexer/resource.rb

@@ -14,6 +14,14 @@ module Harvestdor
       @options = options
     end
 
+    def namespaced_druid
+      if druid =~ /^druid:/
+        druid
+      else
+        "druid:#{druid}"
+      end
+    end
+
     # @return [String] string of form  oo123oo1234
     def bare_druid
       @bare_druid ||= druid.gsub('druid:', '')
@@ -29,6 +37,10 @@ module Harvestdor
       indexer.dor_fetcher_client
     end
 
+    def purl_fetcher_client
+      indexer.purl_fetcher_client
+    end
+
     ##
     # Get the logger
     def logger
@@ -66,7 +78,7 @@ module Harvestdor
       return [] unless collection?
 
       # return an enumerator, with an estimated size of the collection
-      return to_enum(:items) { items_druids.length } unless block_given?
+      return to_enum(:items) { items_druids.count } unless block_given?
 
       items_druids.each do |x|
         yield Harvestdor::Indexer::Resource.new(indexer, x)
@@ -74,7 +86,12 @@ module Harvestdor
     end
 
     def items_druids
-      @items_druids ||= dor_fetcher_client.druid_array(dor_fetcher_client.get_collection(bare_druid, {}))
+      if purl_fetcher_client
+        # we don't need to memoize purl_fetcher_client, since it natively uses enumerables
+        purl_fetcher_client.druids_from_collection(namespaced_druid)
+      else
+        @items_druids ||= dor_fetcher_client.druid_array(dor_fetcher_client.get_collection(bare_druid, {}))
+      end
     end
 
     # given a druid, get its objectLabel from its purl page identityMetadata
@@ -167,4 +184,4 @@ module Harvestdor
       druid.hash ^ indexer.hash
     end
   end
-end
+end

data/lib/harvestdor/indexer/version.rb

@@ -1,6 +1,6 @@
 module Harvestdor
   class Indexer
     # this is the Ruby Gem version
-    VERSION = '2.4.0'.freeze
+    VERSION = '2.5.0'.freeze
   end
 end

data/spec/unit/harvestdor/indexer/purl_fetcher_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe Harvestdor::Indexer::PurlFetcher do
+  describe '#druids_from_collection' do
+    let(:client) { subject.send(:client) }
+
+    before do
+      allow(client).to receive(:get).with('/collections/druid:oo000oo0000/purls', page: 1, per_page: 100).and_return(
+        instance_double(Faraday::Response, body: { purls: [{ druid: 'druid:oo000oo0001' }], pages: { next_page: 2 } }.to_json)
+      )
+      allow(client).to receive(:get).with('/collections/druid:oo000oo0000/purls', page: 2, per_page: 100).and_return(
+        instance_double(Faraday::Response, body: { purls: [{ druid: 'druid:oo000oo0002' }], pages: { next_page: 3 } }.to_json)
+      )
+      allow(client).to receive(:get).with('/collections/druid:oo000oo0000/purls', page: 3, per_page: 100).and_return(
+        instance_double(Faraday::Response, body: { purls: [{ druid: 'druid:oo000oo0003' }], pages: { next_page: nil } }.to_json)
+      )
+    end
+
+    it 'returns an enumerable of druids from a collection' do
+      results = subject.druids_from_collection('druid:oo000oo0000')
+
+      expect(results.to_a).to match_array %w(druid:oo000oo0001 druid:oo000oo0002 druid:oo000oo0003)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: harvestdor-indexer
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.0
 platform: ruby
 authors:
 - Naomi Dushay
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-27 00:00:00.000000000 Z
+date: 2017-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rsolr
@@ -110,6 +110,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: confstruct
   requirement: !ruby/object:Gem::Requirement
@@ -198,30 +212,30 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.42.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.42.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.1
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -274,6 +288,7 @@ files:
 - lib/harvestdor-indexer.rb
 - lib/harvestdor/indexer.rb
 - lib/harvestdor/indexer/metrics.rb
+- lib/harvestdor/indexer/purl_fetcher.rb
 - lib/harvestdor/indexer/resource.rb
 - lib/harvestdor/indexer/solr.rb
 - lib/harvestdor/indexer/version.rb
@@ -289,6 +304,7 @@ files:
 - spec/unit/harvestdor-indexer-solr_spec.rb
 - spec/unit/harvestdor-indexer_spec.rb
 - spec/unit/harvestdor/indexer/metrics_spec.rb
+- spec/unit/harvestdor/indexer/purl_fetcher_spec.rb
 homepage: https://github.com/sul-dlss/harvestdor-indexer
 licenses: []
 metadata: {}
@@ -308,7 +324,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Harvest DOR object metadata and index it to Solr
@@ -325,3 +341,4 @@ test_files:
 - spec/unit/harvestdor-indexer-solr_spec.rb
 - spec/unit/harvestdor-indexer_spec.rb
 - spec/unit/harvestdor/indexer/metrics_spec.rb
+- spec/unit/harvestdor/indexer/purl_fetcher_spec.rb