spotlight-dor-resources 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8b1e1f0ddb963f81f8fd302a284ef53592146ef9
-  data.tar.gz: 03687ee4c19706fa434993ce33ad59beac3b522a
+  metadata.gz: c0c448f6276a5bbd515fecaaf0e0d836e533f246
+  data.tar.gz: 13048c1c02a606c0c4d30e07768c79bcd6385834
 SHA512:
-  metadata.gz: 01970798dc91a7dd7b3f451244e71573d6c1129eb2c59a7f7758b3fdfb4d7d70c3d5bf8e3d99d1a7300b404dbc7142869898b1792729f8ab1099beb1f7113cd1
-  data.tar.gz: 121941bdd33d142d20f27bcacd25d112327324d5e1f84e0f190896bb9fa0c932562cdaeea2a29dae846d84b940085e3bcdd3b92a4f832854e74c24f46269621d
+  metadata.gz: 326665ea9d207234a6fa97b8c3993a5cff159a154d78e61d813de6352b39ece8db5d8364a331830fb0857979e97b12299eded8b77d0b9d513e0155e02a1fab71
+  data.tar.gz: 7a9ade62a20f3c233864fcbccf1454ad220f98297a3556a42550e6e2c30fc887a4040cac6346d4af6ee3ce716a8fd8edad5314b31aaca399f3d9e176acbe3682

data/.gitignore

@@ -20,5 +20,7 @@ pickle-email-*.html
 .project
 config/initializers/secret_token.rb
 spec/internal
+.internal_test_app
 Gemfile.lock
-jetty
+jetty
+./spec/internal

data/.rubocop.yml

@@ -8,7 +8,7 @@ AllCops:
     - 'bin/**/*'
     - 'db/**/*'
     - 'config/**/*'
-    - 'spec/internal/**/*'
+    - '.internal_test_app/**/*'
     - 'spec/test_app_templates/**/*'
     - 'spec/spec_helper.rb'
     - 'spec/teaspoon_env.rb'

data/Gemfile

@@ -12,21 +12,28 @@ group :test do
   gem 'pry-byebug', require: false
 end
 
-# the below comes from engine_cart, a gem used to test this Rails engine gem in the context of a Rails app
-file = File.expand_path("Gemfile", ENV['ENGINE_CART_DESTINATION'] || ENV['RAILS_ROOT'] || File.expand_path("../spec/internal", __FILE__))
+# BEGIN ENGINE_CART BLOCK
+# engine_cart: 0.8.0
+# engine_cart stanza: 0.8.0
+# the below comes from engine_cart, a gem used to test this Rails engine gem in the context of a Rails app.
+file = File.expand_path("Gemfile", ENV['ENGINE_CART_DESTINATION'] || ENV['RAILS_ROOT'] || File.expand_path(".internal_test_app", File.dirname(__FILE__)))
 if File.exist?(file)
-  puts "Loading #{file} ..." if $DEBUG # `ruby -d` or `bundle -v`
-  instance_eval File.read(file)
+  begin
+    eval_gemfile file
+  rescue Bundler::GemfileError => e
+    Bundler.ui.warn '[EngineCart] Skipping Rails application dependencies:'
+    Bundler.ui.warn e.message
+  end
 else
-  gem 'rails', ENV['RAILS_VERSION']
+  Bundler.ui.warn "[EngineCart] Unable to find test application dependencies in #{file}, using placeholder dependencies"
+
+  gem 'rails', ENV['RAILS_VERSION'] if ENV['RAILS_VERSION']
 
-  # explicitly include sass-rails to get compatible sprocket dependencies
   if ENV['RAILS_VERSION'].nil? || ENV['RAILS_VERSION'] =~ /^4.2/
-    gem 'coffee-rails', '~> 4.1.0'
-    gem 'sass-rails', '~> 5.0'
     gem 'responders', "~> 2.0"
+    gem 'sass-rails', ">= 5.0"
   else
     gem 'sass-rails', "< 5.0"
-    gem 'coffee-rails', "~> 4.0.0"
   end
 end
+# END ENGINE_CART BLOCK

data/README.md

@@ -24,7 +24,7 @@ Within a Spotlight application with `spotlight-dor-resources` installed, you cou
 
 Note that Spotlight:
 
-* is a Rails engine and needs to be used in the context of a Rails application. We use [engine_cart](https://github.com/cbeer/engine_cart) to create an internal test application at spec/internal.
+* is a Rails engine and needs to be used in the context of a Rails application. We use [engine_cart](https://github.com/cbeer/engine_cart) to create an internal test application at .internal_test_app.
 * uses Solr as part of its integration tests. We use [jettywrapper](https://github.com/projecthydra/jettywrapper) to manage the Solr instance used for development and test.
 
 Our `$ rake ci` task utilizes Solr and the testing rails app, with Spotlight installed, automatically.
@@ -36,16 +36,16 @@ For access to a Rails console with the gem loaded up for testing purposes, you c
     $ bundle
     $ bundle exec rake:ci
 
-This will download a test jetty instance (to run Solr), generate a testing app at ```spec/internal``` and run the tests.
+This will download a test jetty instance (to run Solr), generate a testing app at ```.internal_test_app``` and run the tests.
 
 ### Indexing with the generated test app
 
-    $ cd spec/internal
-
-    $ bundle exec rails console
+    $ bundle exec rake engine_cart:console
 
 ```ruby
-purl=Spotlight::Resources::Purl.new({:url=>'https://purl.stanford.edu/cx709ty7769'})
+druid="ty202yt2402" # a feigenbaum druid
+druid="cx709ty7769" # a revs druid
+purl=Spotlight::Resources::Purl.new({:url=>"https://purl.stanford.edu/#{druid}"})
 puts purl.to_solr.first # the solr document as a hash
 public_xml=purl.resource.public_xml # nokogiri doc with all public XML
 mods=purl.resource.mods # nokogiri doc with just the MODs

data/lib/spotlight/dor/indexer.rb

@@ -182,6 +182,52 @@ module Spotlight::Dor
       end
     end
 
+    concerning :FullTextIndexing do
+      included do
+        before_index :add_object_full_text
+      end
+
+      # search for configured full text files, and if found, add them to the full text (whole document) solr field
+      def add_object_full_text(sdb, solr_doc)
+        object_level_full_text_urls(sdb).each do |file_url|
+          # append content from each file to the field, creating field if it doesn't exist yet
+          # ruby note: the construct below allows us to append a string to a variable that starts out as nil
+          # because .to_s of nil is empty string
+          solr_doc['full_text_tesim'] = solr_doc['full_text_tesim'].to_s + get_file_content(file_url)
+        end
+      end
+
+      # go grab the supplied file url, grab the file, encode and return
+      # TODO: thse should also be able to also deal with .rtf and .xml files
+      def get_file_content(file_url)
+        response = Net::HTTP.get_response(URI.parse(file_url))
+        response.body.scrub.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+      rescue
+        logger.warn("Error indexing full text - couldn't load file #{file_url}")
+        nil
+      end
+
+      # these are the file locations where full txt files can be found at the object level
+      # this method returns an array of fully qualified public URLs that can be accessed to find full text countent
+      def object_level_full_text_urls(sdb)
+        files = []
+        object_level_full_text_filenames(sdb).each do |xpath_location|
+          files += sdb.public_xml.xpath(xpath_location).map do |txt_file|
+            "#{Spotlight::Dor::Resources::Engine.config.stacks_file_url}/#{sdb.druid}/#{txt_file['id']}"
+          end
+        end
+        files
+      end
+
+      # xpaths to locations in the contentMetadata where full text object level files can be found,
+      #  add as many as you need, all will be searched
+      def object_level_full_text_filenames(sdb)
+        [
+          "//contentMetadata/resource/file[@id=\"#{sdb.druid}.txt\"]" # feigenbaum style - full text in .txt named for druid
+        ]
+      end
+    end
+
     concerning :CartographicIndexing do
       included do
         before_index :mods_cartographics_indexing

data/lib/spotlight/dor/resources/engine.rb

@@ -8,7 +8,9 @@ module Spotlight::Dor::Resources
       Spotlight::Engine.config.resource_providers << Spotlight::Resources::Searchworks
       Spotlight::Engine.config.resource_providers << Spotlight::Resources::Purl
       Spotlight::Dor::Resources::Engine.config.parallel_options = { in_threads: 1 }
-      Spotlight::Dor::Resources::Engine.config.stacks_iiif_url = 'https://stacks.stanford.edu/image/iiif'
+      Spotlight::Dor::Resources::Engine.config.base_stacks_url = 'https://stacks.stanford.edu'
+      Spotlight::Dor::Resources::Engine.config.stacks_file_url = "#{config.base_stacks_url}/file"
+      Spotlight::Dor::Resources::Engine.config.stacks_iiif_url = "#{config.base_stacks_url}/image/iiif"
     end
   end
 end

data/lib/spotlight/dor/resources/version.rb

@@ -2,7 +2,7 @@ module Spotlight
   module Dor
     # :nodoc:
     module Resources
-      VERSION = '0.0.4'
+      VERSION = '0.0.5'
     end
   end
 end

data/spec/lib/spotlight/dor/indexer_spec.rb

@@ -498,4 +498,75 @@ describe Spotlight::Dor::Indexer do
     end # each
   end # add_folder_name
   # rubocop:enable Metrics/LineLength
+
+  describe '#add_object_full_text' do
+    let!(:expected_text) { 'SOME full text string that is returned from the server' }
+    let!(:full_file_path) { 'https://stacks.stanford.edu/file/oo000oo0000/oo000oo0000.txt' }
+    it 'indexes the full text into the appropriate field if a recognized file pattern is found' do
+      public_xml_with_feigenbaum_full_text = Nokogiri::XML <<-EOF
+        <publicObject id="druid:oo000oo0000" published="2015-10-17T18:24:08-07:00">
+          <contentMetadata objectId="oo000oo0000" type="book">
+            <resource id="oo000oo0000_4" sequence="4" type="object">
+              <label>Document</label>
+              <file id="oo000oo0000.pdf" mimetype="application/pdf" size="6801421"></file>
+              <file id="oo000oo0000.txt" mimetype="text/plain" size="23376"></file>
+            </resource>
+            <resource id="oo000oo0000_5" sequence="5" type="page">
+              <label>Page 1</label>
+              <file id="oo000oo0000_00001.jp2" mimetype="image/jp2" size="1864266"><imageData width="2632" height="3422"/></file>
+            </resource>
+            </contentMetadata>
+          </publicObject>
+        EOF
+      allow(sdb).to receive(:public_xml).and_return(public_xml_with_feigenbaum_full_text)
+      # don't actually attempt a call to the stacks
+      allow(subject).to receive(:get_file_content).with(full_file_path).and_return(expected_text)
+      subject.send(:add_object_full_text, sdb, solr_doc)
+      expect(subject.object_level_full_text_urls(sdb)).to eq [full_file_path]
+      expect(solr_doc['full_text_tesim']).to eq expected_text
+    end
+    it 'does not index the full text if no recognized pattern is found' do
+      public_xml_with_no_recognized_full_text = Nokogiri::XML <<-EOF
+        <publicObject id="druid:oo000oo0000" published="2015-10-17T18:24:08-07:00">
+          <contentMetadata objectId="oo000oo0000" type="book">
+            <resource id="oo000oo0000_4" sequence="4" type="object">
+              <label>Document</label>
+              <file id="oo000oo0000.pdf" mimetype="application/pdf" size="6801421"></file>
+            </resource>
+            <resource id="oo000oo0000_5" sequence="5" type="page">
+              <label>Page 1</label>
+              <file id="oo000oo0000_00001.jp2" mimetype="image/jp2" size="1864266"><imageData width="2632" height="3422"/></file>
+            </resource>
+            </contentMetadata>
+          </publicObject>
+        EOF
+      allow(sdb).to receive(:public_xml).and_return(public_xml_with_no_recognized_full_text)
+      subject.send(:add_object_full_text, sdb, solr_doc)
+      expect(subject.object_level_full_text_urls(sdb)).to eq []
+      expect(solr_doc['full_text_tesim']).to be_nil
+    end
+    it 'indexes the full text from two files if two recognized patterns are found' do
+      public_xml_with_two_recognized_full_text_files = Nokogiri::XML <<-EOF
+        <publicObject id="druid:oo000oo0000" published="2015-10-17T18:24:08-07:00">
+          <contentMetadata objectId="oo000oo0000" type="book">
+            <resource id="oo000oo0000_4" sequence="4" type="object">
+              <label>Document</label>
+              <file id="oo000oo0000.pdf" mimetype="application/pdf" size="6801421"></file>
+              <file id="oo000oo0000.txt" mimetype="text/plain" size="23376"></file>
+            </resource>
+            <resource id="oo000oo0000_5" sequence="5" type="page">
+              <label>Page 1</label>
+              <file id="oo000oo0000_00001.jp2" mimetype="image/jp2" size="1864266"><imageData width="2632" height="3422"/></file>
+              <file id="oo000oo0000.txt" mimetype="text/plain" size="23376"></file>
+            </resource>
+            </contentMetadata>
+          </publicObject>
+        EOF
+      allow(sdb).to receive(:public_xml).and_return(public_xml_with_two_recognized_full_text_files)
+      allow(subject).to receive(:get_file_content).with(full_file_path).and_return(expected_text)
+      subject.send(:add_object_full_text, sdb, solr_doc)
+      expect(subject.object_level_full_text_urls(sdb)).to eq [full_file_path, full_file_path]
+      expect(solr_doc['full_text_tesim']).to eq(expected_text + expected_text) # same file twice
+    end
+  end # add_object_full_text
 end

data/spec/spec_helper.rb

@@ -10,7 +10,7 @@ EngineCart.load_application!
 require 'rspec/rails'
 require 'capybara/poltergeist'
 Capybara.javascript_driver = :poltergeist
-Capybara.default_wait_time = 5
+Capybara.default_max_wait_time = 5
 
 if ENV["COVERAGE"] or ENV["CI"]
   require 'simplecov'
@@ -57,4 +57,4 @@ RSpec.configure do |config|
   config.include Warden::Test::Helpers, type: :feature
   config.after(:each, type: :feature) { Warden.test_reset! }
   config.include Capybara::DSL
-end
+end

data/spec/test_app_templates/gdor.yml

@@ -1,6 +1,6 @@
 harvestdor:
   # log_dir:  directory for log file  (default logs, relative to harvestdor gem path)
-  log_dir: spec/internal/log/gdor_logs
+  log_dir: log
 
   # log_name: name of log file  (default: harvestdor.log)
   log_name: gdor.log

data/spec/test_app_templates/lib/generators/test_app_generator.rb

@@ -1,7 +1,7 @@
 require 'rails/generators'
 
 class TestAppGenerator < Rails::Generators::Base
-  source_root "../../spec/test_app_templates"
+  source_root "../spec/test_app_templates"
 
   def add_gems
     gem 'blacklight', '~> 5.1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spotlight-dor-resources
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Chris Beer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-30 00:00:00.000000000 Z
+date: 2015-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -321,7 +321,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: Spotlight resource indexer for DOR resources.