bulk_ops 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01207f13266c7653d3b4febb53b07ca3841fe8db72e8a7798fbd0686f8301816
-  data.tar.gz: '085b2596587d210d457e6dc0a7fe5f760ad8b28e5465da5266a2307133e83442'
+  metadata.gz: 5ca22890f1ad64017c266506d40ec4da8a457534bab1a129b8a19b2e49081bbc
+  data.tar.gz: dca608af70c35c1d1afb2def0c5afbdbf015d035f4ee15d4d7e58baaaf89f1a8
 SHA512:
-  metadata.gz: d423e774693a246d8e7dde5ec88fb58dead4f92e30d3601f74f285feeb370efd193475c5ba0a1a10b1013532e0fc3e0d80dc551bdd6407644838abd12317b4dc
-  data.tar.gz: 644fddf973a2c03e20c39b0bc55fa18ff290685ce41f4e1296c8cb0e1cf03a1b8e0a7a3597838a2cd858d87d549634de29af3c9c86e6a4dc00aefdd3d9fe5d68
+  metadata.gz: 7b3f8b03cd80cc8e24a1f135ebf8dadac1dc274d8c49d7fb0b1f968f41b5836aa2c92353d40e24a3a9d6dd48e32543ffe8ab83195c1cd8702cc481901bfd6d21
+  data.tar.gz: 9252201c835571f0a2c9d5a8027cd45c03acf95ffc865e5e5eebdbdab1c765d6c97c7515175874de2aede0a5950037a55b395fd47ed5ec590139501801f75751

data/app/controllers/bulk_ops/operations_controller.rb

@@ -316,7 +316,7 @@ module BulkOps
 
       @operation.stage = "verifying"
       @operation.save
-      BulkOps::VerificationJob.perform_later(@operation)
+      BulkOps::VerificationJob.perform_later(@operation.id)
       flash[:notice] = "We are now running the data from your spreadsheet through an automatic verification process to anticipate any problems before we begin the ingest. This may take a few minutes. You should recieve an email when the process completes."
       redirect_to action: "show"
     end

data/lib/bulk_ops/verification.rb

@@ -4,9 +4,13 @@ module BulkOps
     
     def verify
       @verification_errors ||= []
+      update(message: "verifying spreadsheet column headers")
       verify_column_headers
+      update(message: "verifying controlled vocab urls (starting)")
       verify_remote_urls
+      update(message: "verifying complex object structures")
       verify_internal_references
+      update(message: "verifying that all files exist")
       verify_files
       verify_works_to_update if operation_type.to_s == "update"
       unless @verification_errors.blank?
@@ -29,6 +33,7 @@ module BulkOps
 
     def is_file_field?(fieldname)
       return false if fieldname.blank?
+      return false if schema.get_field(fieldname)
       field_parts = fieldname.underscore.humanize.downcase.gsub(/[-_]/,' ').split(" ")
       return false unless field_parts.any?{ |field_type| BulkOps::WorkProxy::FILE_FIELDS.include?(field_type) }
       return "remove" if field_parts.any?{ |field_type| ['remove','delete'].include?(field_type) }
@@ -126,12 +131,18 @@ module BulkOps
     end
 
     def verify_remote_urls
-      get_spreadsheet.each do |row, row_num|
+      row_offset = BulkOps::GithubAccess::ROW_OFFSET.present? ? BulkOps::GithubAccess::ROW_OFFSET : 2
+      get_spreadsheet.each_with_index do |row, row_num|
+        update(message: "verifying controlled vocab urls (row number #{row_num})")
+        next if row_num.nil?
         schema.controlled_field_names.each do |controlled_field_name|
-          next unless (url = row[controlled_field_name])
-          label = ::WorkIndexer.fetch_remote_label(url)        
-          if !label || label.blank?
-            @verification_errors << BulkOps::Error.new({type: :cannot_retrieve_label, row: row_num + ROW_OFFSET, field: controlled_field_name, url: url})
+          next unless (urls = row[controlled_field_name])
+          urls.split(';').each do |url|
+            label = ::WorkIndexer.fetch_remote_label(url)
+            # if we can't get the label, and we aren't going to add a local label, throw an error
+            if (!label || label.blank?) && !schema.get_field(controlled_field_name).vocabularies.any?{|vocab| vocab["authority"].to_s.downcase == "local"}
+                @verification_errors << BulkOps::Error.new({type: :cannot_retrieve_label, row_number: row_num + row_offset, field: controlled_field_name, url: url})
+            end
           end
         end
       end
@@ -148,7 +159,6 @@ module BulkOps
       end
     end
 
-
     def verify_works_to_update
       return [] unless operation_type == "update"
       get_spreadsheet.each_with_index do |row, row_num|

data/lib/bulk_ops/verification_job.rb

@@ -7,17 +7,17 @@ class BulkOps::VerificationJob < ActiveJob::Base
 
   queue_as :default
 
-  def perform(operation)
-
-      if operation.verify
-        operation.set_stage "authorize"
-        if operation.create_pull_request
-          operation.notify(subject: "Bulk Operation Verification Successful", message: "Your bulk ingest has passed verification, and we have requested to start applying the operation. It may required one final approval from an administrator before the operation proceeds.")
-        else
-          operation.notify(subject: "Bulk Operation - Error creating Github pull request", message: "Your bulk ingest has passed verification, but we had a problem creating a pull request on Github in order to merge this operation with the master branch. Please check your github configuration.")
-        end
+  def perform(operation_id)
+    operation = BulkOps::Operation.find(operation_id)
+    if operation.verify
+      operation.set_stage "authorize"
+      if operation.create_pull_request
+        operation.notify(subject: "Bulk Operation Verification Successful", message: "Your bulk ingest has passed verification, and we have requested to start applying the operation. It may required one final approval from an administrator before the operation proceeds.")
       else
-        operation.set_stage "pending"
+        operation.notify(subject: "Bulk Operation - Error creating Github pull request", message: "Your bulk ingest has passed verification, but we had a problem creating a pull request on Github in order to merge this operation with the master branch. Please check your github configuration.")
       end
+    else
+      operation.set_stage "pending"
+    end
   end
 end

data/lib/bulk_ops/version.rb

@@ -1,3 +1,3 @@
 module BulkOps
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end

data/lib/bulk_ops/work_proxy.rb

@@ -111,7 +111,7 @@ class BulkOps::WorkProxy < ActiveRecord::Base
   def split_values value_string
     # Split values on all un-escaped separator character (escape character is '\')
     # Then replace all escaped separator charactors with un-escaped versions
-    value_string.split(/(?<!\\)#{SEPARATOR}/).map{|val| val.gsub("\\#{SEPARATOR}",SEPARATOR)}
+    value_string.split(/(?<!\\)#{SEPARATOR}/).map{|val| val.gsub("\\#{SEPARATOR}",SEPARATOR).strip}
   end
 
   def interpret_controlled_fields raw_data
@@ -154,11 +154,14 @@ class BulkOps::WorkProxy < ActiveRecord::Base
 
       remove = field_name.downcase.starts_with?("remove") || field_name.downcase.starts_with?("delete")
       
+
       # handle multiple values
-      split_values(value).each do |value|
-        
+      value_array = split_values(value)
+      controlled_data[field_name_norm] ||= [] unless value_array.blank?
+      value_array.each do |value|
         # Decide of we're dealing with a label or url
         # It's an ID if it's a URL and the name doesn't end in 'label'
+        value.strip!
         if value =~ /^#{URI::regexp}$/ and !field_name.downcase.ends_with?("label")
           id = value
           label = WorkIndexer.fetch_remote_label(value)
@@ -173,37 +176,21 @@ class BulkOps::WorkProxy < ActiveRecord::Base
                        url: value, 
                        row_number: row_number) unless id
         end
-        (controlled_data[field_name_norm] ||= []) << {id: id, label: label, remove: field_name.downcase.starts_with?("remove")}
+        controlled_data[field_name_norm] << {id: id, label: label, remove: field_name.downcase.starts_with?("remove")}
       end
     end
     
     #delete any duplicates (if someone listed a url and also its label, or the same url twice)
     controlled_data.each{|field_name, values| controlled_data[field_name] = values.uniq }
-    
-    if operation.options["compare_labels"]
-      controlled_data.each do |field_name,values|
-        unless labels['field'].count == values.count
-          report_error(:mismatched_auth_terms, 
-                       message: "Different numbers of labels and ids for #{field}",
-                       row_number: row_number) 
-        end
-      end
-      labels['field'].each do |label| 
-        next if controlled_data.any?{|dt| dt["label"] == label}
-        report_error(:mismatched_auth_terms, 
-                     message: "There are controlled vocab term labels that no provided URL resolves to, in the field #{field}.",
-                     row_number: row_number) 
-      end
-    end
-    
+        
     # Actually add all the data
     metadata = {}
     leftover_data = raw_data.dup.to_hash
     controlled_data.each do |property_name, data|
+      metadata["#{property_name}_attributes"] ||= [] unless data.blank?
       data.each do |datum| 
         atts = {"id" => datum[:id]}
         atts["_delete"] = true if datum[:remove]
-        metadata["#{property_name}_attributes"] ||= []
         metadata["#{property_name}_attributes"] << atts
         leftover_data.except! property_name
       end
@@ -382,7 +369,8 @@ class BulkOps::WorkProxy < ActiveRecord::Base
     end
   end
 
-  def mintLocalAuthUrl(auth_name, value) 
+  def mintLocalAuthUrl(auth_name, value)
+    value.strip!
     id = value.parameterize
     auth = Qa::LocalAuthority.find_or_create_by(name: auth_name)
     entry = Qa::LocalAuthorityEntry.create(local_authority: auth,
@@ -392,22 +380,25 @@ class BulkOps::WorkProxy < ActiveRecord::Base
   end
 
   def findAuthUrl(auth, value)
+    value.strip!
     return nil if auth.nil?
     return nil unless (entries = Qa::Authorities::Local.subauthority_for(auth).search(value))
     entries.each do |entry|
       #require exact match
-      if entry["label"] == value
-        url = entry["url"]
-        url ||= entry["id"]
-        url = localIdToUrl(url,auth) unless url =~ URI::regexp
-        return url
-      end
+      next unless entry["label"] == value
+      url = entry["url"]
+      url ||= entry["id"]
+      url = localIdToUrl(url,auth) unless url =~ URI::regexp
+      return url
     end
     return nil
   end
 
   def localIdToUrl(id,auth_name) 
-    return "https://digitalcollections.library.ucsc.edu/authorities/show/local/#{auth_name}/#{id}"
+    hostname = Socket.gethostname
+    hostname = "localhost" unless hostname.include?(',')
+    protocol = (Rails.env == "production") ? "https" : "http"
+    return "#{protocol}://#{hostname}/authorities/show/local/#{auth_name}/#{id}"
   end
 
   def getLocalAuth(field_name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulk_ops
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Ned Henry, UCSC Library Digital Initiatives
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-02 00:00:00.000000000 Z
+date: 2019-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails