bulk_ops 0.1.11 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 58a1fcfbb6f75f1a90103383bf9d77afc7912db5e837d57e7f514a0d2620646b
-  data.tar.gz: c6e74c188710b832d14a62c8622c7d4b116b3fc24069bce83ef044f3c1de7bf7
+  metadata.gz: d5769c63fe85e0a97a16da3f147f2788af3ce5114ac84fd9ea1e3078f187ce11
+  data.tar.gz: 6164cafacea7e4f88cb39464ad2cf19cd38268f63cf4fdccda6f0bee35cc3515
 SHA512:
-  metadata.gz: 629745639d8de5bdb9ced3eaa7473721d33ed87beb2d6e7af100306e5177545ee65419ee172a7eaa900e2e94434437f6236209f6fe00c5b26dd4b21975096687
-  data.tar.gz: 38ecec91afaccc17de448fd42d6aab6d295dab4e40a25d786dd57c77af642e978bfdb0f6805341e57494a135a04c8fdd2dc3da4020fe4a2cc9681cec8d7d4e16
+  metadata.gz: e93c25e5bf47b3d9230b78b42695871bb4c7905270c9f9eeda92e459f030e4a74cc32e5a362e2224a1817ee0dd3f639b36aa573032d90cea00aaf0c3b4c9717a
+  data.tar.gz: 307ad0126fd62baa1a21b7f0ca5073c0847a27435a0665955aa3df68df24a28b85b96396ba01ef966e82d9b218e6b728ef46cd03ec8bcfbb369b18c56d24d32e

data/db/migrate/20181017180436_create_bulk_ops_tables.rb

@@ -33,6 +33,7 @@ class CreateBulkOpsTables < ActiveRecord::Migration[5.0]
       t.string   :identifier_type
       t.string   :relationship_type
       t.string   :status
+      t.string   :next
       t.timestamps
     end
 

data/lib/bulk_ops/error.rb

@@ -1,7 +1,8 @@
 class BulkOps::Error
   attr_accessor :type, :row_number, :object_id, :message, :option_name, :file, :option_values, :field, :url
 
-  MAX_ERROR = 50
+  MAX_ERROR = 5000
+  MAX_ERROR_SHORT = 50
 
   def initialize type:, row_number: nil, object_id: nil, message: nil, options_name: nil, option_values: nil, field: nil, url: nil , file: nil
     @type = type
@@ -39,12 +40,13 @@ class BulkOps::Error
     return error_file_name
   end
 
-  def self.error_message type, errors
+  def self.error_message type, errors, short=false
+    max_error = short ? MAX_ERROR_SHORT : MAX_ERROR
     case type
     when :mismatched_auth_terms
       message = "\n-- Controlled Authority IDs and Labels don't match -- \n"
       message += "The operation is set to create an error if the provided URLs for controlled authority terms do not resolve to the provided labels.\n"
-      if errors.count < MAX_ERROR
+      if errors.count < max_error
         message += "The following rows were affected:\n"
         message += errors.map{|error| error.row_number}.join(",")+"\n"
       else
@@ -53,7 +55,7 @@ class BulkOps::Error
     when :upload_error
       message = "\n-- Errors uploading files -- \n"
       message += "Your files looked ok when we checked earlier, but we couldn't access them when we were trying to actually start the operation.\n"
-      if errors.count < MAX_ERROR
+      if errors.count < max_error
         message += "The following files were affected:\n"
         message += errors.map{|error| "Row #{row_number}, Filename: #{error.file}"}.join("\n")+"\n" 
       else
@@ -90,7 +92,7 @@ class BulkOps::Error
     when :cannot_retrieve_label 
       message = "\n-- Errors Retrieving Remote Labels --\n"
       urls = errors.map{|error| error.url}.uniq
-      if urls.count < MAX_ERROR
+      if urls.count < max_error
         urls.each do |url|
           url_errors = errors.select{|er| er.url == url}
           message +=  "Error retrieving label for remote url #{url}. \nThis url appears in #{url_errors.count} instances in the spreadsheet.\n"
@@ -104,7 +106,7 @@ class BulkOps::Error
     when :cannot_retrieve_url
       message = "\n-- Errors Retrieving Remote URLs --\n"
       urls = errors.map{|error| error.url}.uniq
-      if urls.count < MAX_ERROR
+      if urls.count < max_error
         urls.each do |url|
           url_errors = errors.select{|er| er.url == url}
           message +=  "Error retrieving URL for remote authority term #{url}. \nThis term appears in #{url_errors.count} instances in the spreadsheet.\n"
@@ -118,7 +120,7 @@ class BulkOps::Error
     when :bad_object_reference 
       message = "\n-- Error: bad object reference --\m" 
       message += "We enountered #{errors.count} problems resolving object references.\n"
-      if errors.count < MAX_ERROR
+      if errors.count < max_error
          message += "The row numbers with problems were:\n"
          message += errors.map{|er| "row number #{er.row_number} references the object #{er.object_id}"}.join("\n")
       else
@@ -128,7 +130,7 @@ class BulkOps::Error
     when :cannot_find_file
       message = "\n-- Missing File Errors --\n "
       message += "We couldn't find the files listed on #{errors.count} rows.\n"
-      if errors.count < MAX_ERROR
+      if errors.count < max_error
         message += "Missing filenames:\n"
         message += errors.map{|er| er.file}.join("\n")
       else

data/lib/bulk_ops/operation.rb

@@ -141,12 +141,21 @@ module BulkOps
     def check_if_finished
       return unless stage == "running" && !busy?
 
+      update(stage: "finishing")
+
       # Attempt to resolve each dangling (objectless) relationships
       BulkOps::Relationship.where(:status => "pending").each do |relationship|
-        relationship.resolve!
+        relationship.resolve! if relationship.work_proxy.operation_id == id
+      end
+      
+      work_proxies.each do |proxy|
+        wrk = Work.find(proxy.work_id)
+        wrk.save if wrk.members.any?{|mem| mem.class.to_s !== "FileSet"}
+        sd = SolrDocument.find(wrk.id)
+        wrk.save if sd['hasRelatedImage_ssim'].present? && sd['relatedImageId_ss'].blank?
       end
 
-      update(stage: accumulated_errors.blank? ? "complete" : "errors" )
+      update(stage: (accumulated_errors.blank? ? "complete" : "errors" ))
       report_errors!
       lift_holds
     end
@@ -272,7 +281,7 @@ module BulkOps
     def options
       return {} if name.nil?
       return @options if @options
-      branch = running? ? "master" : nil
+      branch = (running? || complete?) ? "master" : nil
       @options ||= git.load_options(branch: branch)
     end
 

data/lib/bulk_ops/queue_work_ingests_job.rb

@@ -0,0 +1,21 @@
+class BulkOps::QueueWorkIngestsJob <  ActiveJob::Base
+  attr_accessor :operation
+
+  queue_as :ingest
+
+  def perform(op)
+    @operation = op
+    metadata = op.final_spreadsheet
+    op.work_proxies.where(status:'queued').each do |proxy|
+      data = proxy.interpret_data(metadata[proxy.row_number])
+      next unless proxy.proxy_errors.blank?
+      proxy.update(status: 'sidekiq', message: "interpreted at #{DateTime.now.strftime("%d/%m/%Y %H:%M")} " + proxy.message)
+      BulkOps::CreateWorkJob.perform_later(proxy.work_type || "Work",
+                                           op.user.email,
+                                           data,
+                                           proxy.id,
+                                           proxy.visibility)
+    end
+  end
+
+end

data/lib/bulk_ops/relationship.rb

@@ -3,6 +3,7 @@ class BulkOps::Relationship < ActiveRecord::Base
 
   self.table_name = "bulk_ops_relationships"
   belongs_to :work_proxy, class_name: "BulkOps::WorkProxy", foreign_key: "work_proxy_id"
+  delegate :operation, :operation_id, to: :work_proxy
 
   def initialize *args
     super *args
@@ -13,7 +14,7 @@ class BulkOps::Relationship < ActiveRecord::Base
   end
 
   def findObject
-    case identifier_type
+    case identifier_type.downcase
     when "id"
       begin
       object = ActiveFedora::Base.find(object_identifier)
@@ -25,7 +26,7 @@ class BulkOps::Relationship < ActiveRecord::Base
       #          TODO clean up solr query and add work type to it
       query = "{!field f=title_tesim}#{object_identifier}"
       objects = ActiveFedora::SolrService.instance.conn.get(ActiveFedora::SolrService.select_path,
-                                                            params: { fq: query, rows: 100})["response"]["docs"].first
+                                                            params: { fq: query, rows: 100})["response"]["docs"]
       if objects.present?
         return ActiveFedora::Base.find(objects.first["id"])
       elsif relationship_type.downcase == "collection"
@@ -39,9 +40,13 @@ class BulkOps::Relationship < ActiveRecord::Base
       return false if objects.blank?
       return ActiveFedora::Base.find(objects.first["id"])
     when "row"
-      object_proxy = BulkOps::WorkProxy.find_by(operation_id: work_proxy.operation.id, 
+      object_proxy = BulkOps::WorkProxy.find_by(operation_id: work_proxy.operation_id, 
                                                 row_number: (object_identifier.to_i - 2))
       ActiveFedora::Base.find(object_proxy.work_id)
+    when "proxy_id"
+      return false unless (proxy = BulkOps::WorkProxy.find(proxy_id))
+      return false unless proxy.work_id.present?
+      ActiveFedora::Base.find(proxy.work_id)
     end
   end
 
@@ -52,20 +57,47 @@ class BulkOps::Relationship < ActiveRecord::Base
     end
     implement_relationship! relationship_type, subject, object
   end
+
+  def insert_among_siblings(object,new_member)
+    return nil unless ["parent"].include?(relationship_type.downcase)
+    prev_sib_id = previous_sibling
+    # This is the id of the WorkProxy associate with the most recent sibling work
+    # that might be fully ingested. If is it not fully ingested, we will move on 
+    # to the preceding sibling.
+    while prev_sib_id.present? 
+      prev_sib_proxy = BulkOps::WorkProxy.find(previous_sibling)
+      # Check if the previous sibling is fully ingested 
+      # and get its index among its siblings (if it has been successfully attached to the parent)
+      prev_sib_index = object.ordered_member_ids.index(prev_sib_proxy.work_id) if prev_sib_proxy.work_id.present?      # Insert the new member among its siblings if we found the right place
+      return object.ordered_members.to_a.insert(prev_sib_index+1, new_member) if prev_sib_index.present?
+      # Otherwise, pull up the sibling's relationship field to check if it sibling has a sibling before it
+      sib_relationship = prev_sib_proxy.relationships.find_by(relationship_type: relationship_type, object_identifier: object_identifier)
+      # If we can't find an ingested sibling among the ordered members,
+      # make this work the first member.
+      break unless sib_relationship.present?
+      prev_sib_id = sib_relationship.previous_sibling
+    end
+    return  [new_member]+ordered_members.to_a
+  end
   
   def implement_relationship!(type,subject,object)
-    case type
+    case type.downcase
     when "parent"
-      object.ordered_members << subject
-      object.save
+      unless object.member_ids.include? subject.id
+        object.ordered_members = insert_among_siblings(object, new_member)
+        object.save
+      end
     when "child"
-      subject.ordered_members << object
-      subject.save
+      #CAVEAT ordering not fully implemented in this case
+      unless subject.member_ids.include? object.id
+        subject.ordered_members << object
+        subject.save
+      end
     when "collection"
-      object.add_members([subject.id])
-      object.save
-    when "next"
-    #TODO - implement this - related to ordering of filesets
+      unless object.member_object_ids.include? subject.id
+        object.add_members([subject.id])
+        object.save
+      end
     when "order"
       #TODO - implement this - related to ordering of filesets
       

data/lib/bulk_ops/verification.rb

@@ -200,7 +200,7 @@ module BulkOps
           if ref_id == "row" || (ref_id == "id/row" && obj_id.is_a?(Integer))
             obj_id = obj_id.to_i
             # This is a row number reference. It should be an integer in the range of possible row numbers.
-            unless obj_id.present?(obj_id > 0) && (obj_id <= metadata.count)
+            unless obj_id.present? && (obj_id > 0) && (obj_id <= metadata.count)
               @verification_errors << BulkOps::Error.new({type: :bad_object_reference, object_id: obj_id, row_number: row_num + ROW_OFFSET})
             end  
           elsif ref_id == "id" || ref_id == "hyrax id" || (ref_id == "id/row" && (obj_id.is_a? Integer))

data/lib/bulk_ops/version.rb

@@ -1,3 +1,3 @@
 module BulkOps
-  VERSION = "0.1.11"
+  VERSION = "0.1.12"
 end

data/lib/bulk_ops/work_job.rb

@@ -9,42 +9,40 @@ class BulkOps::WorkJob < ActiveJob::Base
   after_perform do |job|
     
     # update BulkOperationsWorkProxy status
-    @work ||= ActiveFedora::Base.find(@work_proxy.work_id)
-    if  @work.id.nil?
-      status = "error"
+    if  @work.nil? || @work.id.nil?
+      update_status "error"
     else
       @work_proxy.work_id = @work.id
-      status = "complete"
-    end
-    update_status status
+      update_status "complete"
 
-    # Attempt to resolve all of the relationships defined in this row   
-    @work_proxy.relationships.each do |relationship|
-      relationship.resolve!
-    end
+      # Attempt to resolve all of the relationships defined in this row   
+      @work_proxy.relationships.each do |relationship|
+        relationship.resolve!
+      end
 
-    # Delete any UploadedFiles. These take up tons of unnecessary disk space.
-    @work.file_sets.each do |fileset|
-      if uf = Hyrax::UploadedFile.find_by(file: fileset.label)
-        uf.destroy!
+      # Delete any UploadedFiles. These take up tons of unnecessary disk space.
+      @work.file_sets.each do |fileset|
+        if uf = Hyrax::UploadedFile.find_by(file: fileset.label)
+          uf.destroy!
+        end
       end
-    end
 
-    # Remove any edit holds placed on an item
-    @work_proxy.lift_hold
+      # Remove any edit holds placed on an item
+      @work_proxy.lift_hold
 
-    # Check if the parent operation is finished
-    # and do any cleanup if so    
-    if @work_proxy.operation.present? && @work_proxy.operation.respond_to?(:check_if_finished)
-      @work_proxy.operation.check_if_finished 
-    end
-  
+      # Check if the parent operation is finished
+      # and do any cleanup if so    
+      if @work_proxy.operation.present? && @work_proxy.operation.respond_to?(:check_if_finished)
+        @work_proxy.operation.check_if_finished 
+      end
+    end 
   end
 
   def perform(workClass,user_email,attributes,work_proxy_id,visibility=nil)
     return if status == "complete"
     update_status "starting", "Initializing the job"
     attributes['visibility']= visibility if visibility.present?
+    attributes['title'] = ['Untitled'] if attributes['title'].blank?
     @work_proxy = BulkOps::WorkProxy.find(work_proxy_id)
     unless @work_proxy
       report_error("Cannot find work proxy with id: #{work_proxy_id}") 
@@ -79,9 +77,10 @@ class BulkOps::WorkJob < ActiveJob::Base
     :create
   end
 
-  def update_status status, message=false
+  def update_status stat, message=false
+    @status = stat
     return false unless @work_proxy
-    atts = {status: status}
+    atts = {status: stat}
     atts[:message] = message if message
     @work_proxy.update(atts)
   end

data/lib/bulk_ops/work_proxy.rb

@@ -2,7 +2,7 @@ class BulkOps::WorkProxy < ActiveRecord::Base
 
   require 'uri'
   OPTION_FIELDS = ['visibility','work type']
-  RELATIONSHIP_FIELDS = ['parent','child','collection','next','order']
+  RELATIONSHIP_FIELDS = ['parent','child','collection','order']
   REFERENCE_IDENTIFIER_FIELDS = ['Reference Identifier','ref_id','Reference ID','Relationship ID','Relationship Identifier','Reference Identifier Type','Reference ID Type','Ref ID Type','relationship_identifier_type','relationship_id_type']
   FILE_FIELDS = ['file','files','filename','filenames']
   FILE_ACTIONS = ['add','upload','remove','delete']
@@ -49,6 +49,7 @@ class BulkOps::WorkProxy < ActiveRecord::Base
     metadata.merge! interpret_relationship_fields(raw_data )
     metadata.merge! interpret_option_fields(raw_data)
     metadata = setAdminSet(metadata)
+    metadata = setMetadataInheritance(metadata)
     return metadata
   end
 
@@ -208,7 +209,7 @@ class BulkOps::WorkProxy < ActiveRecord::Base
       field = schema.get_field(field_name)
       # Ignore controlled fields
       next if field.controlled?
-      values.split(SEPARATOR).each do |value|
+      split_values(values).each do |value|
         next if value.blank?
         value = value.strip.encode('utf-8', :invalid => :replace, :undef => :replace, :replace => '_') unless value.blank?
         value = unescape_csv(value)
@@ -292,62 +293,94 @@ class BulkOps::WorkProxy < ActiveRecord::Base
     return {}
   end
 
-  def interpret_relationship_fields raw_data
+  def interpret_relationship_fields raw_data, row_number=nil
     metadata = {}
-    raw_data.each do |field,value|
+    raw_data do |field,value|
       next if value.blank?  or field.blank?
       field = field.to_s              
       value = unescape_csv(value)
+      identifer_type = reference_identifier
 
       next if value == field
 
       if (split = field.split(":")).count == 2
-        ref_id = split.first
-        field = split.last.to_s
+        identifier_type = split.last
+        field = split.first.to_s
       end
 
-      normfield = field.downcase.parameterize.gsub(/[_\s-]/,'')
-      #      next unless RELATIONSHIP_FIELDS.include? normfield
-      
-      # If the field specifies the object's order among siblings (usually for multiple filesets)
-      update(order: value.to_f) if normfield == "order"
-      
-      # If the field specifies the name or ID of a collection,
-      # find or create the collection and update the metadata to match
-      if ["collection","collectiontitle","memberofcollection","collectionname", "collectionid"].include?(normfield)
+      relationship_type = normalize_relationship_field_name(field)
+      case relationship_type
+      when "order"
+         # If the field specifies the object's order among siblings (usually for multiple filesets)
+        update(order: value.to_f)
+      when "collection"
+        # If the field specifies the name or ID of a collection,
+        # find or create the collection and update the metadata to match
         col = find_or_create_collection(value)
         ( metadata[:member_of_collection_ids] ||= [] ) << col.id if col
+      when "parent", "child"
+        object_id = interpret_relationship_value(value,identifier_type)
       end
-
-      # All variations of field names that require BulkOps::Relationship objects
-      next unless ["parent","parentid","parentidentifier","parentwork","child","childid","childidentifier","childwork","next","nextfile","nextwork","nextid","nextfileidentifier","nextfileid","nextworkid"].include?(normfield)
       
-      # find which type of relationship
-      ["parent","child","next"].select{|type| normfield.include?(type)}.first
       # correctly interpret the notation "id:a78C2d81"
       if ((split = value.split(":")).count == 2)
-        ref_id = split.first
+        identifier_type = split.first
         value = split.last
       end
-      BulkOps::Relationship.create( { work_proxy_id: id,
-                                      identifier_type: ref_id || reference_identifier,
+
+      interpret_relationship_value(identifier_type, value)
+
+      relationship_parameters =  { work_proxy_id: id,
+                                      identifier_type: ref_type,
                                       relationship_type: normfield,
                                       object_identifier: value,
-                                      status: "new"} )
+                                      status: "new"}
+
+      #add previous sibling link if necessary
+      previous_value = op.metadata[row_number-1][field]
+      if previous_value.present? && (ref_type == "parent")
+        if value == interpret_relationship_value(ref_type, previous_value)
+          relationship_parameters[:previous_sibling] = operation.work_proxies.find_by(row_number: row_number-1).id 
+        end
+      end
+      BulkOps::Relationship.create(relationship_parameters)
     end
     return metadata
   end
 
-  def format_reference_id(value)
-    return value if value=="id"
-    # normalize the value string
-    value_method = value.titleize.gsub(/[-_\s]/,'').downcase_first_letter
-    # if this is a valid metadata property or solr parameter, return it as-is
-    return value_method if (schema.get_field?(value_method) || SolrDocument.new.respond_to?(value_method))
-    # if it is means to reference a row number, return the string "row"
-    case value.downcase.parameterize.gsub(/[_\s-]/,'')
-    when "row", "rownum","row number"
-      return "row"
+  def normalize_relationship_field_name field
+    normfield = field.downcase.parameterize.gsub(/[_\s-]/,'')
+    RELATIONSHIP_FIELDS.find{|field| normfield.include?(field) }
+  end
+
+  def find_previous_parent field
+    i = 0;
+    while (prev_row = operation.metadata[row_number - i])
+      return (row_number - i) if prev_row[field].blank?
+    end
+  end
+
+  def find_previous_sibling (field,ref_type,object)
+    previous_value = interpret_relationship_value(ref_type,op.metadata[row_number-1][field])
+    return nil unless previous_value == object
+    return 
+  end
+
+  def interpret_relationship_value id_type, value, field="parent"
+    #Handle "id:20kj4259" syntax if it hasn't already been handled
+    if (split = value.to_s.split(":")).count == 2
+      id_type = split.first
+      value = split.last
+    end
+    #Handle special shorthand syntax for refering to relative row numbers
+    if id_type == "row"
+      if value.to_i < 0
+        # if given a negative integer, count backwards from the current row 
+        return row_number - value
+      elsif value.to_s.downcase.include?("prev")
+        # if given any variation of the word "previous", get the first preceding row with no parent of its own
+        return find_previous_parent(field)
+      end
     end
   end
 
@@ -422,7 +455,13 @@ class BulkOps::WorkProxy < ActiveRecord::Base
     return metadata if metadata[:admin_set_id]
     asets = AdminSet.where({title: "Bulk Ingest Set"})
     asets = AdminSet.find('admin_set/default') if asets.blank?
-    metadata[:admin_set_id] = asets.first.id unless asets.blank?
+    metadata[:admin_set_id] = Array(asets).first.id unless asets.blank?
+    return metadata
+  end
+
+  def setMetadataInheritance metadata
+    return metadata if metadata[:metadataInheritance].present?
+    metadata[:metadataInheritance] = operation.options["metadataInheritance"] unless operation.options["metadataInheritance"].blank?
     return metadata
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulk_ops
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - Ned Henry, UCSC Library Digital Initiatives
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-16 00:00:00.000000000 Z
+date: 2019-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -106,6 +106,7 @@ files:
 - lib/bulk_ops/github_access.rb
 - lib/bulk_ops/github_credential.rb
 - lib/bulk_ops/operation.rb
+- lib/bulk_ops/queue_work_ingests_job.rb
 - lib/bulk_ops/relationship.rb
 - lib/bulk_ops/search_builder_behavior.rb
 - lib/bulk_ops/templates/configuration.yml