spotlight-dor-resources 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c0c448f6276a5bbd515fecaaf0e0d836e533f246
-  data.tar.gz: 13048c1c02a606c0c4d30e07768c79bcd6385834
+  metadata.gz: c130b41c0a1a72df4461aeb827bc66f7959244f6
+  data.tar.gz: 2c91800eb334e092f90e5cb290fa69ace4163b0e
 SHA512:
-  metadata.gz: 326665ea9d207234a6fa97b8c3993a5cff159a154d78e61d813de6352b39ece8db5d8364a331830fb0857979e97b12299eded8b77d0b9d513e0155e02a1fab71
-  data.tar.gz: 7a9ade62a20f3c233864fcbccf1454ad220f98297a3556a42550e6e2c30fc887a4040cac6346d4af6ee3ce716a8fd8edad5314b31aaca399f3d9e176acbe3682
+  metadata.gz: e8ab91d1840f91f56a32a6eefe3d249fb7727fa985aad7324db746d805a3e3c7c2b02ca06e8ac1f49d5fad5ca97645ea107663c9a4cb984e14b51dc03d525050
+  data.tar.gz: 812b4522e38b815d846ce4a240a0792c64cd2b3139895a990d195eb938882828970c6c533ee61f35dc196e591798b8c06dd80466f7fe9a3ff6a34b382f7d73ed

data/README.md

@@ -51,8 +51,12 @@ public_xml=purl.resource.public_xml # nokogiri doc with all public XML
 mods=purl.resource.mods # nokogiri doc with just the MODs
 smods_rec=purl.resource.smods_rec # a Stanford::Mods::Record Object (which inherits from the MODS gem)
 
+puts purl.resource.bare_druid # will not have the "druid:" prefix
+puts purl.resource.druid # might have the "druid:" prefix (if returned as the result of an is_collection call)
+
 purl.resource.indexer.config.solr.url # the configured solr server (should be localhost in development)
-purl.save # save and index to solr
+
+purl.save # save to database and index to solr
 purl.reindex # reindex once saved the first time
 ```
 

data/lib/spotlight/dor/indexer.rb

@@ -1,7 +1,10 @@
 # rubocop:disable Metrics/ClassLength
+# rubocop:disable Metrics/AbcSize
+
 # external gems
 require 'gdor/indexer'
 require 'solrizer'
+require 'faraday'
 
 module Spotlight::Dor
   # Base class to harvest from DOR via harvestdor gem
@@ -125,7 +128,7 @@ module Spotlight::Dor
         add_thumbnail_fields(images.first, solr_doc) if images.first
 
         images.each do |image|
-          add_image_fields(image, solr_doc)
+          add_image_fields(image, solr_doc, sdb.bare_druid)
         end
       end
 
@@ -140,9 +143,9 @@ module Spotlight::Dor
         Solrizer.insert_field(solr_doc, 'content_metadata_first_image_height', image_data['height'], :displayable)
       end
 
-      def add_image_fields(node, solr_doc)
+      def add_image_fields(node, solr_doc, bare_druid)
         file_id = node.attr('id').gsub('.jp2', '')
-        base_url = stacks_iiif_url(solr_doc[:id], file_id)
+        base_url = stacks_iiif_url(bare_druid, file_id)
 
         Solrizer.insert_field(solr_doc, 'content_metadata_image_iiif_info', "#{base_url}/info.json", :displayable)
         Solrizer.insert_field(solr_doc, 'thumbnail_square_url', "#{base_url}/square/100,100/0/default.jpg", :displayable)
@@ -151,8 +154,8 @@ module Spotlight::Dor
         Solrizer.insert_field(solr_doc, 'full_image_url', "#{base_url}/full/full/0/default.jpg", :displayable)
       end
 
-      def stacks_iiif_url(druid, file_name)
-        "#{Spotlight::Dor::Resources::Engine.config.stacks_iiif_url}/#{druid}%2F#{file_name}"
+      def stacks_iiif_url(bare_druid, file_name)
+        "#{Spotlight::Dor::Resources::Engine.config.stacks_iiif_url}/#{bare_druid}%2F#{file_name}"
       end
     end
 
@@ -189,19 +192,16 @@ module Spotlight::Dor
 
       # search for configured full text files, and if found, add them to the full text (whole document) solr field
       def add_object_full_text(sdb, solr_doc)
-        object_level_full_text_urls(sdb).each do |file_url|
-          # append content from each file to the field, creating field if it doesn't exist yet
-          # ruby note: the construct below allows us to append a string to a variable that starts out as nil
-          # because .to_s of nil is empty string
-          solr_doc['full_text_tesim'] = solr_doc['full_text_tesim'].to_s + get_file_content(file_url)
-        end
+        full_text_urls = object_level_full_text_urls(sdb)
+        return if full_text_urls.size == 0
+        solr_doc['full_text_tesim'] = full_text_urls.map { |file_url| get_file_content(file_url) }
       end
 
       # go grab the supplied file url, grab the file, encode and return
-      # TODO: thse should also be able to also deal with .rtf and .xml files
+      # TODO: this should also be able to deal with .rtf and .xml files, scrubbing/converting as necessary to get plain text
       def get_file_content(file_url)
-        response = Net::HTTP.get_response(URI.parse(file_url))
-        response.body.scrub.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        response = Faraday.get(file_url)
+        response.body.scrub.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?').gsub(/\s+/, ' ')
       rescue
         logger.warn("Error indexing full text - couldn't load file #{file_url}")
         nil
@@ -213,7 +213,7 @@ module Spotlight::Dor
         files = []
         object_level_full_text_filenames(sdb).each do |xpath_location|
           files += sdb.public_xml.xpath(xpath_location).map do |txt_file|
-            "#{Spotlight::Dor::Resources::Engine.config.stacks_file_url}/#{sdb.druid}/#{txt_file['id']}"
+            "#{Spotlight::Dor::Resources::Engine.config.stacks_file_url}/#{sdb.bare_druid}/#{txt_file['id']}"
           end
         end
         files
@@ -223,7 +223,7 @@ module Spotlight::Dor
       #  add as many as you need, all will be searched
       def object_level_full_text_filenames(sdb)
         [
-          "//contentMetadata/resource/file[@id=\"#{sdb.druid}.txt\"]" # feigenbaum style - full text in .txt named for druid
+          "//contentMetadata/resource/file[@id=\"#{sdb.bare_druid}.txt\"]" # feigenbaum style - full text in .txt named for druid
         ]
       end
     end

data/lib/spotlight/dor/resources/version.rb

@@ -2,7 +2,7 @@ module Spotlight
   module Dor
     # :nodoc:
     module Resources
-      VERSION = '0.0.5'
+      VERSION = '0.0.6'
     end
   end
 end

data/spec/lib/spotlight/dor/indexer_spec.rb

@@ -50,6 +50,7 @@ describe Spotlight::Dor::Indexer do
   describe '#add_content_metadata_fields' do
     before do
       allow(r).to receive(:public_xml).and_return(public_xml)
+      allow(sdb).to receive(:bare_druid).and_return(fake_druid)
 
       # stacks url calculations require the druid
       solr_doc[:id] = fake_druid
@@ -500,6 +501,9 @@ describe Spotlight::Dor::Indexer do
   # rubocop:enable Metrics/LineLength
 
   describe '#add_object_full_text' do
+    before do
+      allow(sdb).to receive(:bare_druid).and_return(fake_druid)
+    end
     let!(:expected_text) { 'SOME full text string that is returned from the server' }
     let!(:full_file_path) { 'https://stacks.stanford.edu/file/oo000oo0000/oo000oo0000.txt' }
     it 'indexes the full text into the appropriate field if a recognized file pattern is found' do
@@ -523,7 +527,7 @@ describe Spotlight::Dor::Indexer do
       allow(subject).to receive(:get_file_content).with(full_file_path).and_return(expected_text)
       subject.send(:add_object_full_text, sdb, solr_doc)
       expect(subject.object_level_full_text_urls(sdb)).to eq [full_file_path]
-      expect(solr_doc['full_text_tesim']).to eq expected_text
+      expect(solr_doc['full_text_tesim']).to eq [expected_text]
     end
     it 'does not index the full text if no recognized pattern is found' do
       public_xml_with_no_recognized_full_text = Nokogiri::XML <<-EOF
@@ -566,7 +570,7 @@ describe Spotlight::Dor::Indexer do
       allow(subject).to receive(:get_file_content).with(full_file_path).and_return(expected_text)
       subject.send(:add_object_full_text, sdb, solr_doc)
       expect(subject.object_level_full_text_urls(sdb)).to eq [full_file_path, full_file_path]
-      expect(solr_doc['full_text_tesim']).to eq(expected_text + expected_text) # same file twice
+      expect(solr_doc['full_text_tesim']).to eq [expected_text, expected_text] # same file twice in a 2 element array
     end
   end # add_object_full_text
 end

data/spec/test_app_templates/gdor.yml

@@ -1,3 +1,8 @@
+dor_fetcher:
+  service_url: http://dorfetcher-url
+  # if skip_heartbeat set to true, this will skip a check that the dorfetcher service is alive before making API calls
+  skip_heartbeat: true
+
 harvestdor:
   # log_dir:  directory for log file  (default logs, relative to harvestdor gem path)
   log_dir: log

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spotlight-dor-resources
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Chris Beer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-04 00:00:00.000000000 Z
+date: 2015-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday