cul_hydra 1.9.2 → 1.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7c942afbb5baf9638461f80723af40b30633892a7774d426ad4edabe6dbdab4
-  data.tar.gz: 549c735df93fed430ebb33708a59829f7f7228e664309d1f19f323f6de7ef4a6
+  metadata.gz: 8f3619958e270bc6bec9a1e7452032b1859264350705dc0b87be69492fb1ce75
+  data.tar.gz: c8a57028e065ad4dcd862714fe5d1885708ea91705e26f0c11f6f2358d1fd634
 SHA512:
-  metadata.gz: 0d5eba1acff06eb1789b529d0f4871a4d40ce48f76d6e1cbd5acff3162495b156fce85ed2910c581fc7f71548f0c42de3551343b5e85aeddbfbe63f6ea8062d1
-  data.tar.gz: 57d089ae06742089dd3e140307cf3cf787615196e69e8fbbc9a271fab3ff38f0ab4bf97554cab459d9cf1fa7ea66dedb2a17dbb49cb2480706e5712b5e61749b
+  metadata.gz: 328b4fc330baf639835cf543bc1994bdd5fff32d5030b657d48a85b14481bbf894fff1ccb121039d0191a22e3c8b5dbaffa8020969067a52dc0c6f00422de2f8
+  data.tar.gz: 4c142006b25ade25def658155a10ad8ea5ae4d896647f140c06ed7b16d99027b6d1deec68f56340e2e73bb38fcac530098604b886ff3c5c1d83a6f820206035a

data/app/models/cul/hydra/datastreams/encoded_text_datastream.rb

@@ -2,11 +2,13 @@ module Cul
 module Hydra
 module Datastreams
 class EncodedTextDatastream < ::ActiveFedora::Datastream
-  DEFAULT_PRIORITIES = [ Encoding::UTF_8, Encoding::ISO_8859_1, Encoding::WINDOWS_1252 ]
+  DEFAULT_PRIORITIES = [ Encoding::UTF_8, Encoding::WINDOWS_1252, Encoding::ISO_8859_1 ]
+
   def initialize(digital_object=nil, dsid=nil, options={})
     @encoding_priorities = options.delete(:encodings) || DEFAULT_PRIORITIES
     super
   end
+
   def content=(value)
     super(utf8able!(value).encode!(Encoding::UTF_8))
   end
@@ -21,6 +23,7 @@ class EncodedTextDatastream < ::ActiveFedora::Datastream
 
   def self.utf8able!(data, encoding_priorities = DEFAULT_PRIORITIES)
     return unless data
+    data = data.read if data.is_a? IO
     content_encoding = encoding_priorities.detect do |enc|
       begin
         data.force_encoding(enc).valid_encoding?
@@ -29,7 +32,6 @@ class EncodedTextDatastream < ::ActiveFedora::Datastream
       end
     end
     raise "could not encode text datastream content" unless content_encoding
-    puts "using encoding #{content_encoding}"
     data.force_encoding(content_encoding)
   end
 end

data/app/models/generic_resource.rb

@@ -68,7 +68,8 @@ class GenericResource < ::ActiveFedora::Base
     solr_doc["fulltext_tesim"] = []
     unless self.datastreams["fulltext"].nil?
       solr_doc["fulltext_tesim"].concat(solr_doc["title_display_ssm"]) unless solr_doc["title_display_ssm"].nil? or solr_doc["title_display_ssm"].length == 0
-      solr_doc["fulltext_tesim"] << self.datastreams["fulltext"].content
+      utf8able = Cul::Hydra::Datastreams::EncodedTextDatastream.utf8able!(self.datastreams["fulltext"].content)
+      solr_doc["fulltext_tesim"] << utf8able.encode(Encoding::UTF_8)
     end
     relationships(:original_name).each do |original_name|
       solr_doc["original_name_tesim"] ||= []

data/fixtures/spec/BLOB/description-cp1252.txt

@@ -0,0 +1 @@
+In 1917, Harrison founded the first organization (The Liberty League) and the first newspaper (The Voice) of the �New Negro Movement� and he published his first book, The Negro and the Nation. He opposed positions taken by Joel E. Spingarn and W.E.B. Du Bois of the NAACP during the First World War and, along with William Monroe Trotter and others he organized the 1918 Liberty Congress. The Congress, the major Black protest effort during the war, demanded enforcement of the Thirteenth, Fourteenth, and Fifteenth Amendments and federal anti-lynching legislation. Beginning in 1920, he became the principal editor of Marcus Garvey's Negro World, which he reshaped into a leading political and literary publication of the era. In its pages, he discussed history, politics, theater, international affairs, religion, and science. He also created a "Poetry for the People" feature, a �West Indian News Notes� column, and what he described as the first regular book review section by a Black author in �Negro newspaperdom.� In 1920 he also published his second book, When Africa Awakes: The �Inside Story� of the Stirrings and Strivings of the New Negro in the Western World. Later, he would criticize Garvey's methods and actions.

data/lib/cul_hydra/indexer.rb

@@ -2,7 +2,9 @@ module Cul::Hydra::Indexer
 
   NUM_FEDORA_RETRY_ATTEMPTS = 3
   DELAY_BETWEEN_FEDORA_RETRY_ATTEMPTS = 5.seconds
-
+  DEFAULT_INDEX_OPTS = {
+    skip_resources: false, verbose_output: false, softcommit: true, reraise: false
+  }.freeze
   def self.descend_from(pid, pids_to_omit=nil, verbose_output=false)
     if pid.blank?
       raise 'Please supply a pid (e.g. rake recursively_index_fedora_objects pid=ldpd:123)'
@@ -60,14 +62,37 @@ module Cul::Hydra::Indexer
   end
   def self.recursively_index_fedora_objects(top_pid, pids_to_omit=nil, skip_generic_resources=false, verbose_output=false)
 
+    index_opts = { skip_generic_resources: skip_generic_resources, verbose_output: verbose_output }
     descend_from(top_pid, pids_to_omit, verbose_output) do |pid|
-      self.index_pid(pid, skip_generic_resources, verbose_output)
+      self.index_pid(pid, index_opts)
     end
+  end
 
+  # this is a compatibility method for bridging the previously used postional arguments to
+  # keyword arguments by extracting an opts hash from varargs
+  # legacy positional opts signature: skip_resources = false, verbose_output = false, softcommit = true
+  # keyword defaults are in DEFAULT_INDEX_OPTS
+  # @param args [Array] a list of arguments ending with an options hash
+  # @return options hash
+  def self.extract_index_opts(args)
+    args = args.dup # do not modify the original list
+    # extract opts hash
+    index_opts = (args.last.is_a? Hash) ? args.pop : {}
+    # symbolize keys and reverse merge defaults
+    index_opts = index_opts.map {|k,v| [k.to_sym, v] }.to_h
+    index_opts = DEFAULT_INDEX_OPTS.merge(index_opts)
+    # assign any legacy positional arguments, permitting explicit nils
+    unless args.empty?
+      index_opts[:skip_resources] = args[0] if args.length > 0
+      index_opts[:verbose_output] = args[1] if args.length > 1
+      index_opts[:softcommit] = args[2] if args.length > 2
+    end
+    index_opts
   end
 
-  def self.index_pid(pid, skip_generic_resources=false, verbose_output=false, softcommit=true)
+  def self.index_pid(pid, *args)
     # We found an object with the desired PID. Let's reindex it
+    index_opts = extract_index_opts(args)
     begin
       active_fedora_object = nil
 
@@ -77,19 +102,19 @@ module Cul::Hydra::Indexer
           if skip_generic_resources && active_fedora_object.is_a?(GenericResource)
             puts 'Object was skipped because GenericResources are being skipped and it is a GenericResource.'
           else
-            if softcommit
+            if index_opts[:softcommit]
               active_fedora_object.update_index
             else
               # Using direct solr query to update document without soft commiting
               ActiveFedora::SolrService.add(active_fedora_object.to_solr)
             end
-            puts 'done.' if verbose_output
+            puts 'done.' if index_opts[:verbose_output]
           end
           break
         rescue RestClient::RequestTimeout, Errno::EHOSTUNREACH => e
           remaining_attempts = (NUM_FEDORA_RETRY_ATTEMPTS-1) - i
           if remaining_attempts == 0
-            raise e
+            raise
           else
             Rails.logger.error "Error: Could not connect to fedora. (#{e.class.to_s + ': ' + e.message}).  Will retry #{remaining_attempts} more #{remaining_attempts == 1 ? 'time' : 'times'} (after a #{DELAY_BETWEEN_FEDORA_RETRY_ATTEMPTS} second delay)."
             sleep DELAY_BETWEEN_FEDORA_RETRY_ATTEMPTS
@@ -102,15 +127,18 @@ module Cul::Hydra::Indexer
             sleep 5
           else
             # Other RuntimeErrors should be passed on
-            raise e
+            raise
           end
         end
       end
     rescue SystemExit, Interrupt => e
       # Allow system interrupt (ctrl+c)
-      raise e
+      raise
     rescue Exception => e
       puts "Encountered problem with #{pid}.  Skipping record.  Exception class: #{e.class.name}.  Message: #{e.message}"
+      if index_opts[:reraise]
+        raise
+      end
     end
   end
 end

data/lib/cul_hydra/version.rb

@@ -1,6 +1,6 @@
 module Cul
   module Hydra
-    VERSION = '1.9.2'
+    VERSION = '1.9.3'
     def self.version
       VERSION
     end

data/lib/tasks/index.rake

@@ -91,10 +91,11 @@ namespace :cul_hydra do
       pool = Thread.pool(thread_pool_size)
       mutex = Mutex.new
 
+      index_opts = { skip_generic_resources: skip_generic_resources, verbose_output: false }
       pids.each do |pid|
         pool.process {
           
-          Cul::Hydra::Indexer.index_pid(pid, skip_generic_resources, false)
+          Cul::Hydra::Indexer.index_pid(pid, index_opts)
           
           mutex.synchronize do
             counter += 1
@@ -124,7 +125,8 @@ namespace :cul_hydra do
         next
       end
 
-      skip_generic_resources = (ENV['skip_generic_resources'] == 'true')
+      index_opts = { verbose_output: false }
+      index_opts[:skip_generic_resources] = (ENV['skip_generic_resources'] == 'true')
 
       start_time = Time.now
       pids = Cul::Hydra::RisearchMembers.get_publish_target_member_pids(publish_target_pid, true)
@@ -133,7 +135,7 @@ namespace :cul_hydra do
       counter = 0
 
       pids.each do |pid|
-        Cul::Hydra::Indexer.index_pid(pid, skip_generic_resources, false)
+        Cul::Hydra::Indexer.index_pid(pid, index_opts)
         counter += 1
         puts "Indexed #{counter} of #{total} | #{Time.now - start_time} seconds"
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cul_hydra
 version: !ruby/object:Gem::Version
-  version: 1.9.2
+  version: 1.9.3
 platform: ruby
 authors:
 - Benjamin Armintor
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-12 00:00:00.000000000 Z
+date: 2020-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -451,6 +451,7 @@ files:
 - fixtures/cmodels/ldpd_sdep.StaticImageCore.xml
 - fixtures/cmodels/ore_Proxy.xml
 - fixtures/cmodels/pcdm_Collection.xml
+- fixtures/spec/BLOB/description-cp1252.txt
 - fixtures/spec/BLOB/description-utf8.txt
 - fixtures/spec/BLOB/dlc.md
 - fixtures/spec/BLOB/test001.jpg