bulkrax 4.2.1 → 4.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11a3549082d5c6493135ed66fe5729474b4303bb8dec30ee20a8e00fe66f563d
-  data.tar.gz: '08a021e73dc020a2791d1cf6e7f086a2cebdc5b511ffa2c83db004f31592a6af'
+  metadata.gz: d53012e0252f8033f5da334d5336c134deb3a6221040cfe4c1c6c01bb473d617
+  data.tar.gz: e986a3506c073aa533c4ea303a123e8866d2bf7ac6d8ab6df3949682f61b6c05
 SHA512:
-  metadata.gz: '049aaefb66b149b6e57857f75ded3408a910578cdbbd3c8094a85fedeffc2416124e0369e22cab136e2620cd56c9516f7fcf9bf906897682f09f8b32b2ef9dcc'
-  data.tar.gz: bf1a0ff60588f3b66295f6eb7941c80b4c2cb06cbbc68755145e5ec1688906103a8efd6e7ebb6fab5137460de9f1d3d6b5fe8191beac47cd8241ce0c35b250b2
+  metadata.gz: 8ea20593d1164f62fdfda2dbe84bd8cc4e199a65d7e85eea0b68b27e71aeff7579f1b0059042f55eec1c7ab36f8dec9db406ad58cc3a315b29b8e4900f7cb450
+  data.tar.gz: 89a03cd842d855d48f8c4952b6e9a6daea757e80cb8c0f9ee370b59f90e434b39dfb548deb1757f36463914bd5cccbcf1df6c5ffd9102bb5f0d54b5b7ea6476f

data/app/assets/javascripts/bulkrax/exporters.js

@@ -1,10 +1,12 @@
 function hideUnhide(field) {
   var allSources = $('body').find('.export-source-option')
+  removeRequired(allSources)
   hide(allSources)
 
   if (field.length > 0) {
     var selectedSource = $('.' + field)
     unhideSelected(selectedSource)
+    addRequired(selectedSource)
   }
 
   if (field === 'collection') {
@@ -12,6 +14,16 @@ function hideUnhide(field) {
   }
 };
 
+function addRequired(selectedSource) {  
+  selectedSource.addClass('required').attr('required', 'required');
+  selectedSource.parent().addClass('required');
+}
+
+function removeRequired(allSources) {
+  allSources.removeClass('required').removeAttr('required');
+  allSources.parent().removeClass('required').removeAttr('required')
+};
+
 // hide all export_source
 function hide(allSources) {
   allSources.addClass('hidden');

data/app/assets/javascripts/bulkrax/importers.js.erb

@@ -47,6 +47,7 @@ function prepBulkrax(event) {
     var config = { childList: true, attributes: true };
     var callback = function(mutationsList) {
       for(var mutation of mutationsList) {
+
         if (mutation.type == 'childList') {
           browseButton = document.getElementById('browse');
           var exp = /selected_files\[[0-9]*\]\[url\]/
@@ -119,10 +120,35 @@ function handleParserKlass() {
     $('.parser_fields').append(window[parser_klass.data('partial')])
   }
 
+  handleBrowseEverything()
   var file_path_value = $('#importer_parser_fields_import_file_path').val()
   handleFileToggle(file_path_value)
 }
 
+function handleBrowseEverything(){
+  var button = $("button[data-toggle='browse-everything']")
+  if(button.length == 0) { return; }
+  button.browseEverything({
+    route: button.data('route'),
+    target: button.data('target')
+  }).done(function(data) {
+    var evt = { isDefaultPrevented: function() { return false; } };
+    $('.ev-browser.show').removeClass('show')
+    if($('#fileupload').length > 0) {
+      var files = $.map(data, function(d) { return { name: d.file_name, size: d.file_size, id: d.url } });
+      $.blueimp.fileupload.prototype.options.done.call($('#fileupload').fileupload(), evt, { result: { files: files }});
+    }
+    return true
+    // User has submitted files; data contains an array of URLs and their options
+  }).cancel(function() {
+    $('.ev-browser.show').removeClass('show')
+    // User cancelled the browse operation
+  }).fail(function(status, error, text) {
+    $('.ev-browser.show').removeClass('show')
+    // URL retrieval experienced a technical failure
+  });
+}
+
 function handleSourceLoad(refresh_button, base_url, external_set_select) {
   if (base_url.val() == "") { // ignore empty base_url value
     return
@@ -163,4 +189,4 @@ function setError(selector, error) {
   selector.attr('disabled', true)
 }
 
-$(document).on({'ready': prepBulkrax, 'turbolinks:load': prepBulkrax})
+$(document).on({'ready': prepBulkrax, 'turbolinks:load': prepBulkrax})

data/app/controllers/bulkrax/exporters_controller.rb

@@ -22,7 +22,9 @@ module Bulkrax
       add_exporter_breadcrumbs
       add_breadcrumb @exporter.name
 
-      @work_entries = @exporter.entries.where(type: @exporter.parser.entry_class.to_s).page(params[:work_entries_page])
+      @work_entries = @exporter.entries.where(type: @exporter.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
+      @collection_entries = @exporter.entries.where(type: @exporter.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
+      @file_set_entries = @exporter.entries.where(type: @exporter.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
     end
 
     # GET /exporters/new

data/app/controllers/bulkrax/importers_controller.rb

@@ -276,7 +276,7 @@ module Bulkrax
     def setup_client(url)
       return false if url.nil?
       headers = { from: Bulkrax.server_name }
-      @client ||= OAI::Client.new(url, headers: headers, parser: 'libxml', metadata_prefix: 'oai_dc')
+      @client ||= OAI::Client.new(url, headers: headers, parser: 'libxml')
     end
 
     # Download methods

data/app/factories/bulkrax/object_factory.rb

@@ -159,28 +159,55 @@ module Bulkrax
       file_set_attrs = attrs.slice(*object.attributes.keys)
       object.assign_attributes(file_set_attrs)
 
-      attrs['uploaded_files'].each do |uploaded_file_id|
+      attrs['uploaded_files']&.each do |uploaded_file_id|
         uploaded_file = ::Hyrax::UploadedFile.find(uploaded_file_id)
         next if uploaded_file.file_set_uri.present?
 
-        actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
-        uploaded_file.update(file_set_uri: actor.file_set.uri)
-        actor.file_set.permissions_attributes = work_permissions
-        actor.create_metadata
-        actor.create_content(uploaded_file)
-        actor.attach_to_work(work)
+        create_file_set_actor(attrs, work, work_permissions, uploaded_file)
+      end
+      attrs['remote_files']&.each do |remote_file|
+        create_file_set_actor(attrs, work, work_permissions, nil, remote_file)
       end
 
       object.save!
     end
 
+    def create_file_set_actor(attrs, work, work_permissions, uploaded_file, remote_file = nil)
+      actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
+      uploaded_file&.update(file_set_uri: actor.file_set.uri)
+      actor.file_set.permissions_attributes = work_permissions
+      actor.create_metadata(attrs)
+      actor.create_content(uploaded_file) if uploaded_file
+      actor.attach_to_work(work, attrs)
+      handle_remote_file(remote_file: remote_file, actor: actor, update: false) if remote_file
+    end
+
     def update_file_set(attrs)
       file_set_attrs = attrs.slice(*object.attributes.keys)
       actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
-
+      attrs['remote_files']&.each do |remote_file|
+        handle_remote_file(remote_file: remote_file, actor: actor, update: true)
+      end
       actor.update_metadata(file_set_attrs)
     end
 
+    def handle_remote_file(remote_file:, actor:, update: false)
+      actor.file_set.label = remote_file['file_name']
+      actor.file_set.import_url = remote_file['url']
+
+      url = remote_file['url']
+      tmp_file = Tempfile.new(remote_file['file_name'].split('.').first)
+      tmp_file.binmode
+
+      URI.open(url) do |url_file|
+        tmp_file.write(url_file.read)
+      end
+
+      tmp_file.rewind
+      update == true ? actor.update_content(tmp_file) : actor.create_content(tmp_file, from_url: true)
+      tmp_file.close
+    end
+
     def clean_attrs(attrs)
       # avoid the "ArgumentError: Identifier must be a string of size > 0 in order to be treeified" error
       # when setting object.attributes

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -81,7 +81,7 @@ module Bulkrax
     # This is adding the reverse relationship, from the child to the parent
     def collection_parent_work_child
       child_work_ids = child_records[:works].map(&:id)
-      parent_record.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX
+      parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
 
       parent_record.add_member_objects(child_work_ids)
       ImporterRun.find(importer_run_id).increment!(:processed_relationships, child_work_ids.count) # rubocop:disable Rails/SkipsModelValidations

data/app/jobs/bulkrax/import_work_job.rb

@@ -5,25 +5,27 @@ module Bulkrax
     queue_as :import
 
     # rubocop:disable Rails/SkipsModelValidations
-    def perform(*args)
-      entry = Entry.find(args[0])
+    def perform(entry_id, run_id, *)
+      entry = Entry.find(entry_id)
+      importer_run = ImporterRun.find(run_id)
       entry.build
       if entry.status == "Complete"
-        ImporterRun.find(args[1]).increment!(:processed_records)
-        ImporterRun.find(args[1]).increment!(:processed_works)
-        ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
+        importer_run.increment!(:processed_records)
+        importer_run.increment!(:processed_works)
       else
         # do not retry here because whatever parse error kept you from creating a work will likely
         # keep preventing you from doing so.
-        ImporterRun.find(args[1]).increment!(:failed_records)
-        ImporterRun.find(args[1]).increment!(:failed_works)
-        ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
+        importer_run.increment!(:failed_records)
+        importer_run.increment!(:failed_works)
       end
+      # Regardless of completion or not, we want to decrement the enqueued records.
+      importer_run.decrement!(:enqueued_records) unless importer_run.enqueued_records <= 0
+
       entry.save!
-      entry.importer.current_run = ImporterRun.find(args[1])
+      entry.importer.current_run = importer_run
       entry.importer.record_status
     rescue Bulkrax::CollectionsCreatedError
-      reschedule(args[0], args[1])
+      reschedule(entry_id, run_id)
     end
     # rubocop:enable Rails/SkipsModelValidations
 

data/app/matchers/bulkrax/application_matcher.rb

@@ -30,7 +30,7 @@ module Bulkrax
 
     def process_split
       if self.split.is_a?(TrueClass)
-        @result = @result.split(/\s*[:;|]\s*/) # default split by : ; |
+        @result = @result.split(Bulkrax.multi_value_element_split_on)
       elsif self.split
         result = @result.split(Regexp.new(self.split))
         @result = result.map(&:strip)

data/app/models/bulkrax/csv_entry.rb

@@ -18,7 +18,7 @@ module Bulkrax
       raise StandardError, 'CSV path empty' if path.blank?
       CSV.read(path,
         headers: true,
-        header_converters: :symbol,
+        header_converters: ->(h) { h.to_sym },
         encoding: 'utf-8')
     end
 
@@ -81,7 +81,7 @@ module Bulkrax
     def add_file
       self.parsed_metadata['file'] ||= []
       if record['file']&.is_a?(String)
-        self.parsed_metadata['file'] = record['file'].split(/\s*[;|]\s*/)
+        self.parsed_metadata['file'] = record['file'].split(Bulkrax.multi_value_element_split_on)
       elsif record['file'].is_a?(Array)
         self.parsed_metadata['file'] = record['file']
       end
@@ -112,12 +112,16 @@ module Bulkrax
     end
 
     def build_files_metadata
-      file_mapping = key_for_export('file')
-      file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
-      filenames = map_file_sets(file_sets)
+      # attaching files to the FileSet row only so we don't have duplicates when importing to a new tenant
+      if hyrax_record.work?
+        build_thumbnail_files
+      else
+        file_mapping = key_for_export('file')
+        file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
+        filenames = map_file_sets(file_sets)
 
-      handle_join_on_export(file_mapping, filenames, mapping['file']&.[]('join')&.present?)
-      build_thumbnail_files if hyrax_record.work?
+        handle_join_on_export(file_mapping, filenames, mapping['file']&.[]('join')&.present?)
+      end
     end
 
     def build_relationship_metadata
@@ -172,7 +176,7 @@ module Bulkrax
       data = hyrax_record.send(key.to_s)
       if data.is_a?(ActiveTriples::Relation)
         if value['join']
-          self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join(' | ').to_s # TODO: make split char dynamic
+          self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join(Bulkrax.multi_value_element_join_on).to_s
         else
           data.each_with_index do |d, i|
             self.parsed_metadata["#{key_for_export(key)}_#{i + 1}"] = prepare_export_data(d)
@@ -232,7 +236,7 @@ module Bulkrax
 
     def handle_join_on_export(key, values, join)
       if join
-        parsed_metadata[key] = values.join(' | ') # TODO: make split char dynamic
+        parsed_metadata[key] = values.join(Bulkrax.multi_value_element_join_on)
       else
         values.each_with_index do |value, i|
           parsed_metadata["#{key}_#{i + 1}"] = value
@@ -256,7 +260,7 @@ module Bulkrax
       return [] unless parent_field_mapping.present? && record[parent_field_mapping].present?
 
       identifiers = []
-      split_references = record[parent_field_mapping].split(/\s*[;|]\s*/)
+      split_references = record[parent_field_mapping].split(Bulkrax.multi_value_element_split_on)
       split_references.each do |c_reference|
         matching_collection_entries = importerexporter.entries.select do |e|
           (e.raw_metadata&.[](source_identifier) == c_reference) &&

data/app/models/bulkrax/importer.rb

@@ -58,17 +58,26 @@ module Bulkrax
 
     # If field_mapping is empty, setup a default based on the export_properties
     def mapping
+      # rubocop:disable Style/IfUnlessModifier
       @mapping ||= if self.field_mapping.blank? || self.field_mapping == [{}]
                      if parser.import_fields.present? || self.field_mapping == [{}]
-                       ActiveSupport::HashWithIndifferentAccess.new(
-                         parser.import_fields.reject(&:nil?).map do |m|
-                           Bulkrax.default_field_mapping.call(m)
-                         end.inject(:merge)
-                       )
+                       default_field_mapping
                      end
                    else
-                     self.field_mapping
+                     default_field_mapping.merge(self.field_mapping)
                    end
+
+      # rubocop:enable Style/IfUnlessModifier
+    end
+
+    def default_field_mapping
+      return self.field_mapping if parser.import_fields.nil?
+
+      ActiveSupport::HashWithIndifferentAccess.new(
+        parser.import_fields.reject(&:nil?).map do |m|
+          Bulkrax.default_field_mapping.call(m)
+        end.inject(:merge)
+      )
     end
 
     def parser_fields
@@ -143,17 +152,13 @@ module Bulkrax
       import_objects(['relationship'])
     end
 
+    DEFAULT_OBJECT_TYPES = %w[collection work file_set relationship].freeze
+
     def import_objects(types_array = nil)
       self.only_updates ||= false
-      types = types_array || %w[collection work file_set relationship]
-      if parser.class == Bulkrax::CsvParser
-        parser.create_objects(types)
-      else
-        types.each do |object_type|
-          self.save if self.new_record? # Object needs to be saved for statuses
-          parser.send("create_#{object_type.pluralize}")
-        end
-      end
+      self.save if self.new_record? # Object needs to be saved for statuses
+      types = types_array || DEFAULT_OBJECT_TYPES
+      parser.create_objects(types)
     rescue StandardError => e
       status_info(e)
     end

data/app/models/bulkrax/oai_entry.rb

@@ -59,12 +59,11 @@ module Bulkrax
     def find_collection_ids
       return self.collection_ids if collections_created?
       if sets.blank? || parser.collection_name != 'all'
-        # c = Collection.where(Bulkrax.system_identifier_field => importerexporter.unique_collection_identifier(parser.collection_name)).first
         collection = find_collection(importerexporter.unique_collection_identifier(parser.collection_name))
         self.collection_ids << collection.id if collection.present? && !self.collection_ids.include?(collection.id)
       else # All - collections should exist for all sets
         sets.each do |set|
-          c = Collection.find_by(work_identifier => importerexporter.unique_collection_identifier(set.content))
+          c = find_collection(importerexporter.unique_collection_identifier(set.content))
           self.collection_ids << c.id if c.present? && !self.collection_ids.include?(c.id)
         end
       end

data/app/models/concerns/bulkrax/file_set_entry_behavior.rb

@@ -6,7 +6,14 @@ module Bulkrax
       ::FileSet
     end
 
+    def file_reference
+      return 'file' if parsed_metadata&.[]('file')&.map(&:present?)&.any?
+      return 'remote_files' if parsed_metadata&.[]('remote_files')&.map(&:present?)&.any?
+    end
+
     def add_path_to_file
+      return unless file_reference == 'file'
+
       parsed_metadata['file'].each_with_index do |filename, i|
         next if filename.blank?
 
@@ -22,7 +29,7 @@ module Bulkrax
     end
 
     def validate_presence_of_filename!
-      return if parsed_metadata&.[]('file')&.map(&:present?)&.any?
+      return if parsed_metadata&.[](file_reference)&.map(&:present?)&.any?
 
       raise StandardError, 'File set must have a filename'
     end

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -165,15 +165,16 @@ module Bulkrax
     end
 
     def factory
-      @factory ||= Bulkrax::ObjectFactory.new(attributes: self.parsed_metadata,
-                                              source_identifier_value: identifier,
-                                              work_identifier: parser.work_identifier,
-                                              related_parents_parsed_mapping: parser.related_parents_parsed_mapping,
-                                              replace_files: replace_files,
-                                              user: user,
-                                              klass: factory_class,
-                                              importer_run_id: importerexporter.last_run.id,
-                                              update_files: update_files)
+      of = Bulkrax.object_factory || Bulkrax::ObjectFactory
+      @factory ||= of.new(attributes: self.parsed_metadata,
+                          source_identifier_value: identifier,
+                          work_identifier: parser.work_identifier,
+                          related_parents_parsed_mapping: parser.related_parents_parsed_mapping,
+                          replace_files: replace_files,
+                          user: user,
+                          klass: factory_class,
+                          importer_run_id: importerexporter.last_run.id,
+                          update_files: update_files)
     end
 
     def factory_class