bulk_ops 0.1.21 → 0.1.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00c3c3a7c2256497f2b11f6a177b4c5ef83ffd97bfc68aa624ff9cc03245c41d
-  data.tar.gz: ad0ea1d063dc3e73bf9623ada0bc68288dfcb19f56a40999b05f0b9d59123977
+  metadata.gz: d0b68ae5923dae38908ff5382d52aba0fa514d0cf9e714eefcfb46013ec9ce02
+  data.tar.gz: 175b647ebf14fb4e432861c0185d9bf142d666e4df29129631d064517f55f588
 SHA512:
-  metadata.gz: 1c8fe5f805e77388d2825935d9876ab2a19786f05c5eccdf2dc3fc149b112010a790e129aad2ff77717dd65c1af06fff9182ec5f0a6b99e68c37aff7e7583391
-  data.tar.gz: a2ab3d414597fb7e6add10ceb86ed339c082eb8f8604b6f6ec51df16b303e59639c0354ac85e63bd03f2f810748a6254c00a7a1a3f930c9a9b4fa746735ebb43
+  metadata.gz: 4209b3080bd3e6b4e5df1ef727e1db943252b0d32169bb080db61e6ed01b4ba93fff56303a38d6e655eda22f82377e06a1d847c7d47df8d664f7af6e5bf49d75
+  data.tar.gz: ac685fa72de08e26d4fe403b57143399602986aa3cf2600c00f2c55a86788bdefa5cfc0dc250d364a8b808ed1f856ea8f84c5e7dd8c88107859d21783b6ca55c

data/lib/bulk_ops/error.rb

@@ -137,6 +137,37 @@ class BulkOps::Error
         message += "An example of a missing filename is: #{errors.first.file}\n"
       end
       
+   when :relationship_error
+      message = "\n-- Errors resolving relationships --\n "
+      message += "There were issues resolving #{errors.count} relationships.\n"
+      if errors.count < max_error
+        message += "errors:\n"
+        message += errors.map{|er| "Row #{er.row_number}, relationship ##{er.object_id}: #{er.message}"}.join("\n")
+      else
+        message += "An example of an error is: Row #{er.first.row_number}, relationship ##{er.first.object_id}: #{er.first.message}\n"
+      end
+      
+    when :ingest_failure
+      message = "\n-- Ingested File is Broken or Missing --\n "
+      message += "After the ingest completed, we had issues finding and re-saving the ingested works associated with #{errors.count} rows.\n"
+      if errors.count < max_error
+        message += "Problem rows:\n"
+        message += errors.map{|er| "#{er.row_number} - proxy ##{er.object_id}"}.join("\n")
+      else
+        message += "An example of a failed ingest is row #{errors.first.row_number} with work proxy #{errors.first.object_id} \n"
+      end
+
+    when :id_not_unique
+      message = "\n-- Multiple works shared a supposedly unique identifier, and we don't know which one to edit --\n "
+      if errors.count < max_error
+        message += "Problem rows:\n"
+        message += errors.map{|er| "#{er.row_number} - proxy ##{er.object_id} - #{er.options_name}: #{er.option_values}"}.join("\n")
+      else
+        message += "An example of a row that identifies multiple works is #{errors.first.row_number} with work proxy #{errors.first.object_id} using the identifier:  #{er.options_name} - #{er.option_values} \n"
+      end
+
+    else
+      message = "\n-- There were other errors of an unrecognized type. Check the application logs --\n "      
     end
     return message
   end

data/lib/bulk_ops/operation.rb

@@ -125,18 +125,26 @@ module BulkOps
       update(stage: "finishing")
 
       # Attempt to resolve each dangling (objectless) relationships
-      BulkOps::Relationship.where(:status => "pending").each do |relationship|
-        relationship.resolve! if relationship.work_proxy.operation_id == id
+      relationships = work_proxies.reduce([]){|all_rels,proxy| all_rels + proxy.relationships.select{|rel| rel.status == "pending"}}
+      relationships.each do |rel| 
+        begin
+          rel.resolve! 
+        rescue StandardError => e
+          @operation_errors << BulkOps::Error.new(:relationship_error, row_number: proxy.row_number, object_id: relationship.id, message: "#{e.class} - #{e.message}" )
+        end
       end
       
-      work_proxies.each do |proxy|
-        wrk = Work.find(proxy.work_id)
-        wrk.save if wrk.members.any?{|mem| mem.class.to_s != "FileSet"}
-        sd = SolrDocument.find(wrk.id)
-        wrk.save if sd['hasRelatedImage_ssim'].present? && sd['relatedImageId_ss'].blank?
+      work_proxies.each do |proxy| 
+        work = nil
+        begin
+          work = Work.find(proxy.work_id).save
+        rescue StandardError => e
+          @operation_errors << BulkOps::Error.new(:ingest_failure, row_number: proxy.row_number, object_id: proxy.id, message: "#{e.class} - #{e.message}")
+        end
       end
 
-      update(stage: (accumulated_errors.blank? ? "complete" : "errors" ))
+      new_stage = accumulated_errors.blank? ? "complete" : "errors"
+      update(stage: new_stage)
       report_errors!
       lift_holds
     end
@@ -284,9 +292,9 @@ module BulkOps
     end
 
     def busy?
-      return true if work_proxies.where(status: "running").count > 0
-      return true if work_proxies.where(status: "queued").count > 0
-      return true if work_proxies.where(status: "starting").count > 0
+      return true if work_proxies.any?{|prx| prx.status.downcase == "running"}
+      return true if work_proxies.any?{|prx| prx.status.downcase == "queued"}
+      return true if work_proxies.any?{|prx| prx.status.downcase == "starting"}
       return false
     end
 

data/lib/bulk_ops/parser.rb

@@ -8,9 +8,9 @@ class BulkOps::Parser
   def self.is_file_set? metadata, row_number
     return false unless metadata[row_number].present?
     # If the work type is explicitly specified, use that
-    if (type_key = metadata.keys.find{|key| key.downcase.gsub(/[_\-\s]/,"").include?("worktype") })
-      return true if metadata[type_key].downcase == "fileset" 
-      return false if metadata[type_key].present?
+    if (type_key = metadata[row_number].to_h.keys.find{|key| key.downcase.gsub(/[_\-\s]/,"").include?("worktype") })
+      return true if metadata[row_number][type_key].downcase == "fileset" 
+      return false if metadata[row_number][type_key].present?
     end
 #    Otherwise, if there are any valid fields other than relationship or file fields, call it a work
     metadata[row_number].each do |field, value|
@@ -73,8 +73,11 @@ class BulkOps::Parser
 
   def find_work_id_from_unique_metadata field_name, value
     field_solr_name = schema.get_field(field_name).solr_name
-    query = "_query_:\"{!raw f=#{field_name}}#{value}\""
+    query = "_query_:\"{!dismax qf=#{field_solr_name}}#{value}\""
     response = ActiveFedora::SolrService.instance.conn.get(ActiveFedora::SolrService.select_path, params: { fq: query, rows: 1, start: 0})["response"]
+    if response["numFound"] > 1
+      report_error( :id_not_unique , "",  row_number: row_number, object_id: @proxy.id, options_name: field_name, option_values: value ) unless label
+    end
     return response["docs"][0]["id"]
   end
 

data/lib/bulk_ops/version.rb

@@ -1,3 +1,3 @@
 module BulkOps
-  VERSION = "0.1.21"
+  VERSION = "0.1.22"
 end

data/lib/bulk_ops/work_job.rb

@@ -23,7 +23,11 @@ class BulkOps::WorkJob < ActiveJob::Base
       # Delete any UploadedFiles. These take up tons of unnecessary disk space.
       @work.file_sets.each do |fileset|
         if uf = Hyrax::UploadedFile.find_by(file: fileset.label)
-          uf.destroy!
+          begin
+            uf.destroy!
+          rescue StandardError => e
+            Rails.logger.warn("Could not delete uploaded file. #{e.class} - #{e.message}")
+          end
         end
       end
 
@@ -49,28 +53,30 @@ class BulkOps::WorkJob < ActiveJob::Base
       return
     end
 
-    return unless define_work(workClass)
+    return unless (work_action = define_work(workClass))
 
     user = User.find_by_email(user_email)
     update_status "running", "Started background task at #{DateTime.now.strftime("%d/%m/%Y %H:%M")}"
     ability = Ability.new(user)
     env = Hyrax::Actors::Environment.new(@work, ability, attributes)
-    update_status "complete", Hyrax::CurationConcern.actor.send(type,env)
+    update_status "complete", Hyrax::CurationConcern.actor.send(work_action,env)
   end
 
   private
 
 
-  def define_work
+  def define_work(workClass="Work")
     if (@work_proxy.present? && @work_proxy.work_id.present? && record_exists?(@work_proxy.work_id))
       begin
         @work = ActiveFedora::Base.find(@work_proxy.work_id)
+        return :update
       rescue ActiveFedora::ObjectNotFoundError
         report_error "Could not find work to update in Fedora (though it shows up in Solr). Work id: #{@work_proxy.work_id}"
         return false
       end
     else
       @work = workClass.capitalize.constantize.new
+      return :ingest
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulk_ops
 version: !ruby/object:Gem::Version
-  version: 0.1.21
+  version: 0.1.22
 platform: ruby
 authors:
 - Ned Henry, UCSC Library Digital Initiatives
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-26 00:00:00.000000000 Z
+date: 2019-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails