bulkrax 3.4.0 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0ee151bc10b7485eb716463b2c4895165d6df9d73c3dd60813d2eb4de8161d1
-  data.tar.gz: f4e5ddfb5ac602eb20a85850f8ec5a9286a9a668f8c83548dd049db4d91a2a0e
+  metadata.gz: 0a80674a9f582c3b8e83f442318908edb6ca9f0b615c970d09b17d941cc8027d
+  data.tar.gz: a2a53116ef49e03dde1aa1df14d8259a2b4abf06a82cff63a9d4ba622ba6600a
 SHA512:
-  metadata.gz: a5b029da7feaee11c8a3eb58e0c7150abcb06b6ca08f9d102134d5e1c9eef3049ae85b0084d74934b7504688da7d4958a0dc425c705b42101c1a3ce62a57d0c7
-  data.tar.gz: c807b68265c0d88b9e7faea4f6efdf94b3bcf90e965f5cc97cc259202bc55db976944eea8e4bf99876723f881d29d2fa7004dbd9985f7f8164648df517722133
+  metadata.gz: af3d75fb03105e37f7374f3a7f863c545d9cc9c95ab2f18bbbf7b4692024e09811f103a372327b4724c836568bad176ed0ad0b7be929ed556259aa9b0793fce6
+  data.tar.gz: 1117a185fbab2bae0746187f464bebea855759a5ecccf0d34f098ac55ad7a2952e663268372262ba8f97820c8c1f02bd29c74a388cfd8ea9cfed84a46dad94cf

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -42,10 +42,12 @@ module Bulkrax
       pending_relationships.each do |rel|
         raise ::StandardError, %("#{rel}" needs either a child or a parent to create a relationship) if rel.child_id.nil? || rel.parent_id.nil?
         @child_entry, child_record = find_record(rel.child_id, importer_run_id)
-        child_record.is_a?(::Collection) ? @child_records[:collections] << child_record : @child_records[:works] << child_record
+        if child_record
+          child_record.is_a?(::Collection) ? @child_records[:collections] << child_record : @child_records[:works] << child_record
+        end
       end
 
-      if (child_records[:collections].blank? && child_records[:works].blank?) || parent_record.blank?
+      if (child_records[:collections].blank? && child_records[:works].blank?) || parent_record.nil?
         reschedule({ parent_identifier: parent_identifier, importer_run_id: importer_run_id })
         return false # stop current job from continuing to run after rescheduling
       end

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -12,15 +12,14 @@ module Bulkrax
       # check for our entry in our current importer first
       importer_id = ImporterRun.find(importer_run_id).importer_id
       default_scope = { identifier: identifier, importerexporter_type: 'Bulkrax::Importer' }
-      record = Entry.find_by(default_scope.merge({ importerexporter_id: importer_id })) || Entry.find_by(default_scope)
 
-      # TODO(alishaevn): discuss whether we are only looking for Collection models here
-      # use ActiveFedora::Base.find(identifier) instead?
-      record ||= ::Collection.where(id: identifier).first # rubocop:disable Rails/FindBy
-      if record.blank?
-        available_work_types.each do |work_type|
-          record ||= work_type.where(id: identifier).first # rubocop:disable Rails/FindBy
-        end
+      begin
+        # the identifier parameter can be a :source_identifier or the id of an object
+        record = Entry.find_by(default_scope.merge({ importerexporter_id: importer_id })) || Entry.find_by(default_scope)
+        record ||= ActiveFedora::Base.find(identifier)
+      # NameError for if ActiveFedora isn't installed
+      rescue NameError, ActiveFedora::ObjectNotFoundError
+        record = nil
       end
 
       # return the found entry here instead of searching for it again in the CreateRelationshipsJob

data/app/models/concerns/bulkrax/file_set_entry_behavior.rb

@@ -8,10 +8,14 @@ module Bulkrax
 
     def add_path_to_file
       parsed_metadata['file'].each_with_index do |filename, i|
-        path_to_file = ::File.join(parser.path_to_files, filename)
+        next if filename.blank?
+
+        path_to_file = parser.path_to_files(filename: filename)
 
         parsed_metadata['file'][i] = path_to_file
       end
+      parsed_metadata['file'].delete('')
+
       raise ::StandardError, "one or more file paths are invalid: #{parsed_metadata['file'].join(', ')}" unless parsed_metadata['file'].map { |file_path| ::File.file?(file_path) }.all?
 
       parsed_metadata['file']

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -12,8 +12,8 @@ module Bulkrax
           raise CollectionsCreatedError unless collections_created?
           @item = factory.run!
           add_user_to_permission_templates! if self.class.to_s.include?("Collection")
-          parent_jobs if self.parsed_metadata[related_parents_parsed_mapping].present?
-          child_jobs if self.parsed_metadata[related_children_parsed_mapping].present?
+          parent_jobs if self.parsed_metadata[related_parents_parsed_mapping]&.join.present?
+          child_jobs if self.parsed_metadata[related_children_parsed_mapping]&.join.present?
         end
       rescue RSolr::Error::Http, CollectionsCreatedError => e
         raise e

data/app/parsers/bulkrax/bagit_parser.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class BagitParser < ApplicationParser # rubocop:disable Metrics/ClassLength
+  class BagitParser < CsvParser # rubocop:disable Metrics/ClassLength
     include ExportBehavior
 
     def self.export_supported?
@@ -20,12 +20,8 @@ module Bulkrax
       rdf_format ? RdfEntry : CsvEntry
     end
 
-    def collection_entry_class
-      CsvCollectionEntry
-    end
-
-    def file_set_entry_class
-      CsvFileSetEntry
+    def path_to_files(filename:)
+      @path_to_files ||= Dir.glob(File.join(import_file_path, '**/data', filename)).first
     end
 
     # Take a random sample of 10 metadata_paths and work out the import fields from that
@@ -36,39 +32,41 @@ module Bulkrax
       end.flatten.compact.uniq
     end
 
-    # Assume a single metadata record per path
-    # Create an Array of all metadata records, one per file
+    # Create an Array of all metadata records
     def records(_opts = {})
       raise StandardError, 'No BagIt records were found' if bags.blank?
       @records ||= bags.map do |bag|
         path = metadata_path(bag)
         raise StandardError, 'No metadata files were found' if path.blank?
         data = entry_class.read_data(path)
-        data = entry_class.data_for_entry(data, source_identifier, self)
-        data[:file] = bag.bag_files.join('|') unless importerexporter.metadata_only?
-        data
+        get_data(bag, data)
       end
+
+      @records = @records.flatten
     end
 
-    # Find or create collections referenced by works
-    # If the import data also contains records for these works, they will be updated
-    # during create works
-    def create_collections
-      collections.each_with_index do |collection, index|
-        next if collection.blank?
-        metadata = {
-          title: [collection],
-          work_identifier => [collection],
-          visibility: 'open',
-          collection_type_gid: Hyrax::CollectionType.find_or_create_default_collection_type.gid
-        }
-        new_entry = find_or_create_entry(collection_entry_class, collection, 'Bulkrax::Importer', metadata)
-        ImportCollectionJob.perform_now(new_entry.id, current_run.id)
-        increment_counters(index, collection: true)
+    def get_data(bag, data)
+      if entry_class == CsvEntry
+        data = data.map do |data_row|
+          record_data = entry_class.data_for_entry(data_row, source_identifier, self)
+          next record_data if importerexporter.metadata_only?
+
+          record_data[:file] = bag.bag_files.join('|') if ::Hyrax.config.curation_concerns.include? record_data[:model]&.constantize
+          record_data
+        end
+      else
+        data = entry_class.data_for_entry(data, source_identifier, self)
+        data[:file] = bag.bag_files.join('|') unless importerexporter.metadata_only?
       end
+
+      data
     end
 
     def create_works
+      entry_class == CsvEntry ? super : create_rdf_works
+    end
+
+    def create_rdf_works
       records.each_with_index do |record, index|
         next unless record_has_source_identifier(record, index)
         break if limit_reached?(limit, index)
@@ -87,19 +85,6 @@ module Bulkrax
       status_info(e)
     end
 
-    def collections
-      records.map { |r| r[related_parents_parsed_mapping].split(/\s*[;|]\s*/) if r[related_parents_parsed_mapping].present? }.flatten.compact.uniq
-    end
-
-    def collections_total
-      collections.size
-    end
-
-    # TODO: change to differentiate between collection and work records when adding ability to import collection metadata
-    def works_total
-      total
-    end
-
     def total
       @total = importer.parser_fields['total'] || 0 if importer?
 
@@ -112,18 +97,6 @@ module Bulkrax
       @total = 0
     end
 
-    def extra_filters
-      output = ""
-      if importerexporter.start_date.present?
-        start_dt = importerexporter.start_date.to_datetime.strftime('%FT%TZ')
-        finish_dt = importerexporter.finish_date.present? ? importerexporter.finish_date.to_datetime.end_of_day.strftime('%FT%TZ') : "NOW"
-        output += " AND system_modified_dtsi:[#{start_dt} TO #{finish_dt}]"
-      end
-      output += importerexporter.work_visibility.present? ? " AND visibility_ssi:#{importerexporter.work_visibility}" : ""
-      output += importerexporter.workflow_status.present? ? " AND workflow_state_name_ssim:#{importerexporter.workflow_status}" : ""
-      output
-    end
-
     def current_record_ids
       @work_ids = []
       @collection_ids = []
@@ -143,70 +116,19 @@ module Bulkrax
       @work_ids + @collection_ids + @file_set_ids
     end
 
-    # Set the following instance variables: @work_ids, @collection_ids, @file_set_ids
-    # @see #current_record_ids
-    def set_ids_for_exporting_from_importer
-      entry_ids = Importer.find(importerexporter.export_source).entries.pluck(:id)
-      complete_statuses = Status.latest_by_statusable
-                                .includes(:statusable)
-                                .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
-
-      complete_entry_identifiers = complete_statuses.map { |s| s.statusable&.identifier&.gsub(':', '\:') }
-      extra_filters = extra_filters.presence || '*:*'
-
-      { :@work_ids => ::Hyrax.config.curation_concerns, :@collection_ids => [::Collection], :@file_set_ids => [::FileSet] }.each do |instance_var, models_to_search|
-        instance_variable_set(instance_var, ActiveFedora::SolrService.post(
-          extra_filters.to_s,
-          fq: [
-            %(#{::Solrizer.solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
-            "has_model_ssim:(#{models_to_search.join(' OR ')})"
-          ],
-          fl: 'id',
-          rows: 2_000_000_000
-        )['response']['docs'].map { |obj| obj['id'] })
-      end
-    end
-
     # export methods
 
-    def create_new_entries
-      current_record_ids.each_with_index do |id, index|
-        break if limit_reached?(limit, index)
-
-        this_entry_class = if @collection_ids.include?(id)
-                             collection_entry_class
-                           elsif @file_set_ids.include?(id)
-                             file_set_entry_class
-                           else
-                             entry_class
-                           end
-        new_entry = find_or_create_entry(this_entry_class, id, 'Bulkrax::Exporter')
-
-        begin
-          entry = ExportWorkJob.perform_now(new_entry.id, current_run.id)
-        rescue => e
-          Rails.logger.info("#{e.message} was detected during export")
-        end
-
-        self.headers |= entry.parsed_metadata.keys if entry
-      end
-    end
-    alias create_from_collection create_new_entries
-    alias create_from_importer create_new_entries
-    alias create_from_worktype create_new_entries
-    alias create_from_all create_new_entries
-
     # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
     def write_files
       require 'open-uri'
       require 'socket'
       importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |entry|
-        work = ActiveFedora::Base.find(entry.identifier)
-        next unless Hyrax.config.curation_concerns.include?(work.class)
+        record = ActiveFedora::Base.find(entry.identifier)
+        next unless Hyrax.config.curation_concerns.include?(record.class)
         bag = BagIt::Bag.new setup_bagit_folder(entry.identifier)
         bag_entries = [entry]
 
-        work.file_sets.each do |fs|
+        record.file_sets.each do |fs|
           if @file_set_ids.present?
             file_set_entry = Bulkrax::CsvFileSetEntry.where("parsed_metadata LIKE '%#{fs.id}%'").first
             bag_entries << file_set_entry unless file_set_entry.nil?
@@ -245,42 +167,6 @@ module Bulkrax
         key != source_identifier.to_s
     end
 
-    # All possible column names
-    def export_headers
-      headers = sort_headers(self.headers)
-
-      # we don't want access_control_id exported and we want file at the end
-      headers.delete('access_control_id') if headers.include?('access_control_id')
-
-      # add the headers below at the beginning or end to maintain the preexisting export behavior
-      headers.prepend('model')
-      headers.prepend(source_identifier.to_s)
-      headers.prepend('id')
-
-      headers.uniq
-    end
-
-    def object_names
-      return @object_names if @object_names
-
-      @object_names = mapping.values.map { |value| value['object'] }
-      @object_names.uniq!.delete(nil)
-
-      @object_names
-    end
-
-    def sort_headers(headers)
-      # converting headers like creator_name_1 to creator_1_name so they get sorted by numerical order
-      # while keeping objects grouped together
-      headers.sort_by do |item|
-        number = item.match(/\d+/)&.[](0) || 0.to_s
-        sort_number = number.rjust(4, "0")
-        object_prefix = object_names.detect { |o| item.match(/^#{o}/) } || item
-        remainder = item.gsub(/^#{object_prefix}_/, '').gsub(/_#{number}/, '')
-        "#{object_prefix}_#{sort_number}_#{remainder}"
-      end
-    end
-
     def setup_triple_metadata_export_file(id)
       File.join(importerexporter.exporter_export_path, id, 'metadata.nt')
     end
@@ -300,11 +186,6 @@ module Bulkrax
       end
     end
 
-    def required_elements?(keys)
-      return if keys.blank?
-      !required_elements.map { |el| keys.map(&:to_s).include?(el) }.include?(false)
-    end
-
     # @todo - investigate getting directory structure
     # @todo - investigate using perform_later, and having the importer check for
     #   DownloadCloudFileJob before it starts
@@ -355,5 +236,11 @@ module Bulkrax
       return nil unless bag.valid?
       bag
     end
+
+    # use the version of this method from the application parser instead
+    def real_import_file_path
+      return importer_unzip_path if file? && zip?
+      parser_fields['import_file_path']
+    end
   end
 end

data/app/parsers/bulkrax/csv_parser.rb

@@ -272,8 +272,8 @@ module Bulkrax
       CsvFileSetEntry
     end
 
-    # See https://stackoverflow.com/questions/2650517/count-the-number-of-lines-in-a-file-without-reading-entire-file-into-memory
-    #   Changed to grep as wc -l counts blank lines, and ignores the final unescaped line (which may or may not contain data)
+    # TODO: figure out why using the version of this method that's in the bagit parser
+    # breaks specs for the "if importer?" line
     def total
       @total = importer.parser_fields['total'] || 0 if importer?
       @total = limit || current_record_ids.count if exporter?
@@ -382,10 +382,11 @@ module Bulkrax
     end
 
     # Retrieve the path where we expect to find the files
-    def path_to_files
+    def path_to_files(**args)
+      filename = args.fetch(:filename, '')
+
       @path_to_files ||= File.join(
-        zip? ? importer_unzip_path : File.dirname(import_file_path),
-        'files'
+        zip? ? importer_unzip_path : File.dirname(import_file_path), 'files', filename
       )
     end
 

data/app/views/bulkrax/exporters/_form.html.erb

@@ -29,6 +29,7 @@
 
   <%= form.input :export_source_importer,
     label: t('bulkrax.exporter.labels.importer'),
+    required: true,
     prompt: 'Select from the list',
     label_html: { class: 'importer export-source-option hidden' },
     input_html: { class: 'importer export-source-option hidden' },
@@ -37,6 +38,7 @@
   <%= form.input :export_source_collection,
     prompt: 'Start typing ...',
     label: t('bulkrax.exporter.labels.collection'),
+    required: true,
     placeholder: @collection&.title&.first,
     label_html: { class: 'collection export-source-option hidden' },
     input_html: {
@@ -50,6 +52,7 @@
 
   <%= form.input :export_source_worktype,
     label: t('bulkrax.exporter.labels.worktype'),
+    required: true,
     prompt: 'Select from the list',
     label_html: { class: 'worktype export-source-option hidden' },
     input_html: { class: 'worktype export-source-option hidden' },

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '3.4.0'
+  VERSION = '3.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.5.0
 platform: ruby
 authors:
 - Rob Kaufman
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-22 00:00:00.000000000 Z
+date: 2022-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -404,7 +404,7 @@ homepage: https://github.com/samvera-labs/bulkrax
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -419,8 +419,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: Import and export tool for Hyrax and Hyku
 test_files: []