bulkrax 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23cd36b2c3dc13769f7d20da3c0d53f2d5c2cabef15d5beeeccee75aed6d679a
-  data.tar.gz: 868f0ec5c54862943e126af0ecf63d49771bc9a4ac88e566074205d7077e063c
+  metadata.gz: 49b7cfb404f1878429bbac77aad6d1167e97377fe28b54e2042cee7f6ca04e67
+  data.tar.gz: c3789c193dc1610c99117e5e2c450ed477a7390995916c4b6b419900f5cf539b
 SHA512:
-  metadata.gz: a4f9c359a539054dba54000ca30bd1209bb862254c75463c2598807e8fd91e7e562e3bd6203f5f347ffe17da22f58d7915c1090109e88aabccac774c0281b1ee
-  data.tar.gz: 11c0083e67c4add719f04c01d20fac3216f6e994407dfdc48ae5f6d82696241a2b81ef6156279568b92e4bd56d3598085e83bac3718d04892a4fc96bbefd1028
+  metadata.gz: a669e9b566770ad21e3a6242d24779f5c73847b2b97d4f7334f06209ac7de3634e7667550ba35ce6930f9ceaffda8efccb29cf3277b40f29d808ddbd9623a1b8
+  data.tar.gz: 794e6de65d4ebb5665ab73fdac3eba2d4157ff1b07f3da6cc63a6323b2127dc40021d69ea193da2a356e276f08dde8ccc51dc1d8c563e5ba3e8ecb592e4b4c88

data/README.md

@@ -1,7 +1,6 @@
 # Bulkrax
 Bulkrax is a batteries included importer for Samvera applications. It currently includes support for OAI-PMH (DC and Qualified DC) and CSV out of the box. It is also designed to be extensible, allowing you to easily add new importers in to your application or to include them with other gems. Bulkrax provides a full admin interface including creating, editing, scheduling and reviewing imports.
 
-
 ## Installation
 
 ### Install Generator
@@ -9,9 +8,9 @@ Bulkrax is a batteries included importer for Samvera applications. It currently
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'bulkrax', '1.0.0'
+gem 'bulkrax'
 # or if using from github
-gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git'
+gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git', branch: 'main'
 ```
 
 And then execute:
@@ -27,7 +26,7 @@ If using Sidekiq, set up queues for `import` and `export`.
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git'
+gem 'bulkrax'
 ```
 
 And then execute:

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -46,13 +46,9 @@ module Bulkrax
       end
 
       if (child_records[:collections].blank? && child_records[:works].blank?) || parent_record.blank?
-        reschedule(
-          parent_identifier: parent_identifier,
-          importer_run_id: importer_run_id
-        )
+        reschedule({ parent_identifier: parent_identifier, importer_run_id: importer_run_id })
         return false # stop current job from continuing to run after rescheduling
       end
-
       @parent_entry ||= Bulkrax::Entry.where(identifier: parent_identifier,
                                              importerexporter_id: ImporterRun.find(importer_run_id).importer_id,
                                              importerexporter_type: "Bulkrax::Importer").first

data/app/jobs/bulkrax/delete_collection_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteCollectionJob < DeleteJob; end
+end

data/app/jobs/bulkrax/delete_file_set_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteFileSetJob < DeleteJob; end
+end

data/app/jobs/bulkrax/delete_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteJob < ApplicationJob
+    queue_as :import
+
+    # rubocop:disable Rails/SkipsModelValidations
+    def perform(entry, importer_run)
+      obj = entry.factory.find
+      obj&.delete
+      ImporterRun.find(importer_run.id).increment!(:deleted_records)
+      ImporterRun.find(importer_run.id).decrement!(:enqueued_records)
+      entry.save!
+      entry.importer.current_run = ImporterRun.find(importer_run.id)
+      entry.importer.record_status
+      entry.status_info("Deleted", ImporterRun.find(importer_run.id))
+    end
+    # rubocop:enable Rails/SkipsModelValidations
+  end
+end

data/app/jobs/bulkrax/delete_work_job.rb

@@ -1,20 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class DeleteWorkJob < ApplicationJob
-    queue_as :import
-
-    # rubocop:disable Rails/SkipsModelValidations
-    def perform(entry, importer_run)
-      work = entry.factory.find
-      work&.delete
-      ImporterRun.find(importer_run.id).increment!(:deleted_records)
-      ImporterRun.find(importer_run.id).decrement!(:enqueued_records)
-      entry.save!
-      entry.importer.current_run = ImporterRun.find(importer_run.id)
-      entry.importer.record_status
-      entry.status_info("Deleted", ImporterRun.find(importer_run.id))
-    end
-    # rubocop:enable Rails/SkipsModelValidations
-  end
+  class DeleteWorkJob < DeleteJob; end
 end

data/app/jobs/bulkrax/import_collection_job.rb

@@ -9,16 +9,18 @@ module Bulkrax
       entry = Entry.find(args[0])
       begin
         entry.build
-        entry.save
+        entry.save!
         ImporterRun.find(args[1]).increment!(:processed_records)
         ImporterRun.find(args[1]).increment!(:processed_collections)
-        ImporterRun.find(args[1]).decrement!(:enqueued_records)
+        ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
       rescue => e
         ImporterRun.find(args[1]).increment!(:failed_records)
         ImporterRun.find(args[1]).increment!(:failed_collections)
-        ImporterRun.find(args[1]).decrement!(:enqueued_records)
+        ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
         raise e
       end
+      entry.importer.current_run = ImporterRun.find(args[1])
+      entry.importer.record_status
     end
     # rubocop:enable Rails/SkipsModelValidations
   end

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -26,17 +26,17 @@ module Bulkrax
         ImporterRun.find(importer_run_id).increment!(:failed_file_sets)
         # rubocop:enable Rails/SkipsModelValidations
       end
-      ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
+      ImporterRun.find(importer_run_id).decrement!(:enqueued_records) unless ImporterRun.find(importer_run_id).enqueued_records <= 0 # rubocop:disable Rails/SkipsModelValidations
       entry.save!
+      entry.importer.current_run = ImporterRun.find(importer_run_id)
+      entry.importer.record_status
 
     rescue MissingParentError => e
       # try waiting for the parent record to be created
       entry.import_attempts += 1
       entry.save!
       if entry.import_attempts < 5
-        ImportFileSetJob
-          .set(wait: (entry.import_attempts + 1).minutes)
-          .perform_later(entry_id, importer_run_id)
+        ImportFileSetJob.set(wait: (entry.import_attempts + 1).minutes).perform_later(entry_id, importer_run_id)
       else
         ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
         entry.status_info(e)

data/app/models/bulkrax/csv_entry.rb

@@ -93,17 +93,31 @@ module Bulkrax
     end
 
     def build_export_metadata
-      # make_round_trippable
       self.parsed_metadata = {}
-      self.parsed_metadata['id'] = hyrax_record.id
-      self.parsed_metadata[source_identifier] = hyrax_record.send(work_identifier)
-      self.parsed_metadata['model'] = hyrax_record.has_model.first
+
+      build_system_metadata
+      build_files_metadata unless hyrax_record.is_a?(Collection)
       build_relationship_metadata
       build_mapping_metadata
-      build_files unless hyrax_record.is_a?(Collection)
+
       self.parsed_metadata
     end
 
+    # Metadata required by Bulkrax for round-tripping
+    def build_system_metadata
+      self.parsed_metadata['id'] = hyrax_record.id
+      self.parsed_metadata[source_identifier] = hyrax_record.send(work_identifier)
+      self.parsed_metadata[key_for_export('model')] = hyrax_record.has_model.first
+    end
+
+    def build_files_metadata
+      file_mapping = key_for_export('file')
+      file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
+      filenames = file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact
+
+      handle_join_on_export(file_mapping, filenames, mapping['file']&.[]('join')&.present?)
+    end
+
     def build_relationship_metadata
       # Includes all relationship methods for all exportable record types (works, Collections, FileSets)
       relationship_methods = {
@@ -127,12 +141,10 @@ module Bulkrax
 
     def build_mapping_metadata
       mapping.each do |key, value|
-        next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
-        next if key == "model"
-        # relationships handled by #build_relationship_metadata
-        next if [related_parents_parsed_mapping, related_children_parsed_mapping].include?(key)
-        next if key == 'file' # handled by #build_files
+        # these keys are handled by other methods
+        next if ['model', 'file', related_parents_parsed_mapping, related_children_parsed_mapping].include?(key)
         next if value['excluded']
+        next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
 
         object_key = key if value.key?('object')
         next unless hyrax_record.respond_to?(key.to_s) || object_key.present?
@@ -205,14 +217,6 @@ module Bulkrax
       end
     end
 
-    def build_files
-      file_mapping = mapping['file']&.[]('from')&.first || 'file'
-      file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
-
-      filenames = file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact
-      handle_join_on_export(file_mapping, filenames, mapping['file']&.[]('join')&.present?)
-    end
-
     def handle_join_on_export(key, values, join)
       if join
         parsed_metadata[key] = values.join(' | ') # TODO: make split char dynamic
@@ -224,16 +228,6 @@ module Bulkrax
       end
     end
 
-    # In order for the existing exported hyrax_record, to be updated by a re-import
-    # we need a unique value in system_identifier
-    # add the existing hyrax_record id to system_identifier
-    def make_round_trippable
-      values = hyrax_record.send(work_identifier.to_s).to_a
-      values << hyrax_record.id
-      hyrax_record.send("#{work_identifier}=", values)
-      hyrax_record.save
-    end
-
     def record
       @record ||= raw_metadata
     end
@@ -263,7 +257,8 @@ module Bulkrax
     end
 
     def collections_created?
-      collection_identifiers.length == self.collection_ids.length
+      # TODO: look into if this method is still needed after new relationships code
+      true
     end
 
     def find_collection_ids

data/app/models/bulkrax/csv_file_set_entry.rb

@@ -2,31 +2,6 @@
 
 module Bulkrax
   class CsvFileSetEntry < CsvEntry
-    def factory_class
-      ::FileSet
-    end
-
-    def add_path_to_file
-      parsed_metadata['file'].each_with_index do |filename, i|
-        path_to_file = ::File.join(parser.path_to_files, filename)
-
-        parsed_metadata['file'][i] = path_to_file
-      end
-      raise ::StandardError, "one or more file paths are invalid: #{parsed_metadata['file'].join(', ')}" unless parsed_metadata['file'].map { |file_path| ::File.file?(file_path) }.all?
-
-      parsed_metadata['file']
-    end
-
-    def validate_presence_of_filename!
-      return if parsed_metadata&.[]('file')&.map(&:present?)&.any?
-
-      raise StandardError, 'File set must have a filename'
-    end
-
-    def validate_presence_of_parent!
-      return if parsed_metadata[related_parents_parsed_mapping]&.map(&:present?)&.any?
-
-      raise StandardError, 'File set must be related to at least one work'
-    end
+    include FileSetEntryBehavior
   end
 end

data/app/models/bulkrax/rdf_file_set_entry.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class RdfFileSetEntry < RdfEntry
+    include FileSetEntryBehavior
+  end
+end

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -12,7 +12,7 @@ module Bulkrax
       # check for our entry in our current importer first
       importer_id = ImporterRun.find(importer_run_id).importer_id
       default_scope = { identifier: identifier, importerexporter_type: 'Bulkrax::Importer' }
-      record = Entry.find_by(default_scope, importerexporter_id: importer_id) || Entry.find_by(default_scope)
+      record = Entry.find_by(default_scope.merge({ importerexporter_id: importer_id })) || Entry.find_by(default_scope)
 
       # TODO(alishaevn): discuss whether we are only looking for Collection models here
       # use ActiveFedora::Base.find(identifier) instead?

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -42,19 +42,22 @@ module Bulkrax
       end
     end
 
-    # Prepend the file_set id to ensure a unique filename
+    # Prepend the file_set id to ensure a unique filename and also one that is not longer than 255 characters
     def filename(file_set)
       return if file_set.original_file.blank?
       fn = file_set.original_file.file_name.first
       mime = Mime::Type.lookup(file_set.original_file.mime_type)
       ext_mime = MIME::Types.of(file_set.original_file.file_name).first
       if fn.include?(file_set.id) || importerexporter.metadata_only?
-        return fn if mime.to_s == ext_mime.to_s
-        return "#{fn}.#{mime.to_sym}"
+        filename = "#{fn}.#{mime.to_sym}"
+        filename = fn if mime.to_s == ext_mime.to_s
       else
-        return "#{file_set.id}_#{fn}" if mime.to_s == ext_mime.to_s
-        return "#{file_set.id}_#{fn}.#{mime.to_sym}"
+        filename = "#{file_set.id}_#{fn}.#{mime.to_sym}"
+        filename = "#{file_set.id}_#{fn}" if mime.to_s == ext_mime.to_s
       end
+      # Remove extention truncate and reattach
+      ext = File.extname(filename)
+      "#{File.basename(filename, ext)[0...(220 - ext.length)]}#{ext}"
     end
   end
 end

data/app/models/concerns/bulkrax/file_set_entry_behavior.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module FileSetEntryBehavior
+    def factory_class
+      ::FileSet
+    end
+
+    def add_path_to_file
+      parsed_metadata['file'].each_with_index do |filename, i|
+        path_to_file = ::File.join(parser.path_to_files, filename)
+
+        parsed_metadata['file'][i] = path_to_file
+      end
+      raise ::StandardError, "one or more file paths are invalid: #{parsed_metadata['file'].join(', ')}" unless parsed_metadata['file'].map { |file_path| ::File.file?(file_path) }.all?
+
+      parsed_metadata['file']
+    end
+
+    def validate_presence_of_filename!
+      return if parsed_metadata&.[]('file')&.map(&:present?)&.any?
+
+      raise StandardError, 'File set must have a filename'
+    end
+
+    def validate_presence_of_parent!
+      return if parsed_metadata[related_parents_parsed_mapping]&.map(&:present?)&.any?
+
+      raise StandardError, 'File set must be related to at least one work'
+    end
+  end
+end

data/app/parsers/bulkrax/bagit_parser.rb

@@ -23,6 +23,11 @@ module Bulkrax
       Entry
     end
 
+    def file_set_entry_class
+      csv_format = Bulkrax::Importer.last.parser_fields['metadata_format'] == "Bulkrax::CsvEntry"
+      csv_format ? CsvFileSetEntry : RdfFileSetEntry
+    end
+
     # Take a random sample of 10 metadata_paths and work out the import fields from that
     def import_fields
       raise StandardError, 'No metadata files were found' if metadata_paths.blank?

data/app/parsers/bulkrax/csv_parser.rb

@@ -11,12 +11,15 @@ module Bulkrax
     end
 
     def records(_opts = {})
+      return @records if @records.present?
+
       file_for_import = only_updates ? parser_fields['partial_import_file_path'] : import_file_path
       # data for entry does not need source_identifier for csv, because csvs are read sequentially and mapped after raw data is read.
       csv_data = entry_class.read_data(file_for_import)
       importer.parser_fields['total'] = csv_data.count
       importer.save
-      @records ||= csv_data.map { |record_data| entry_class.data_for_entry(record_data, nil, self) }
+
+      @records = csv_data.map { |record_data| entry_class.data_for_entry(record_data, nil, self) }
     end
 
     def build_records
@@ -145,7 +148,6 @@ module Bulkrax
                                        'Bulkrax::Importer',
                                        current_record.to_h)
       if current_record[:delete].present?
-        # TODO: create a "Delete" job for file_sets and collections
         "Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
       else
         "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
@@ -217,7 +219,7 @@ module Bulkrax
         instance_variable_set(instance_var, ActiveFedora::SolrService.post(
           extra_filters.to_s,
           fq: [
-            "#{::Solrizer.solr_name(work_identifier)}:(#{complete_entry_identifiers.join(' OR ')})",
+            %(#{::Solrizer.solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
             "has_model_ssim:(#{models_to_search.join(' OR ')})"
           ],
           fl: 'id',

data/app/views/bulkrax/importers/show.html.erb

@@ -178,7 +178,7 @@
                   <% elsif e.status == "Pending" %>
                     <td><span class="glyphicon glyphicon-option-horizontal" style="color: blue;"></span> <%= e.status %></td>
                   <% else %>
-                    <td><span class="glyphicon glyphicon-remove" style="color: red;"></span> <%= e.status %></td>
+                    <td><span class="glyphicon glyphicon-remove" style="color: <%= e.status == 'Deleted' ? 'green' : 'red' %>;"></span> <%= e.status %></td>
                   <% end %>
                   <% if e.last_error.present? %>
                     <td><%= link_to e.last_error.dig("error_class"), bulkrax.importer_entry_path(@importer.id, e.id) %></td>

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '3.1.0'
+  VERSION = '3.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Rob Kaufman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-19 00:00:00.000000000 Z
+date: 2022-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -260,6 +260,9 @@ files:
 - app/helpers/bulkrax/validation_helper.rb
 - app/jobs/bulkrax/application_job.rb
 - app/jobs/bulkrax/create_relationships_job.rb
+- app/jobs/bulkrax/delete_collection_job.rb
+- app/jobs/bulkrax/delete_file_set_job.rb
+- app/jobs/bulkrax/delete_job.rb
 - app/jobs/bulkrax/delete_work_job.rb
 - app/jobs/bulkrax/download_cloud_file_job.rb
 - app/jobs/bulkrax/export_work_job.rb
@@ -291,6 +294,7 @@ files:
 - app/models/bulkrax/pending_relationship.rb
 - app/models/bulkrax/rdf_collection_entry.rb
 - app/models/bulkrax/rdf_entry.rb
+- app/models/bulkrax/rdf_file_set_entry.rb
 - app/models/bulkrax/status.rb
 - app/models/bulkrax/xml_entry.rb
 - app/models/concerns/bulkrax/download_behavior.rb
@@ -298,6 +302,7 @@ files:
 - app/models/concerns/bulkrax/errored_entries.rb
 - app/models/concerns/bulkrax/export_behavior.rb
 - app/models/concerns/bulkrax/file_factory.rb
+- app/models/concerns/bulkrax/file_set_entry_behavior.rb
 - app/models/concerns/bulkrax/has_local_processing.rb
 - app/models/concerns/bulkrax/has_matchers.rb
 - app/models/concerns/bulkrax/import_behavior.rb