bulkrax 4.0.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb56d86ee90ae9e1cf0628504694e1301ab8f2d6b24ffa8fd323f8953a8ee956
-  data.tar.gz: 71056b077e300f27eee3bcccd9d7e2bee2fc7bdf2fc6ba9248b69a29f3994f9c
+  metadata.gz: 61886547fff51b48446b9dfaf420f14deea3fc86e9c5d8b44c1886087ee8086a
+  data.tar.gz: d19557cd24341cbd6001e835fff081dca349eeeab818b6b8bcf4c4cb91e5b692
 SHA512:
-  metadata.gz: 05ea49e6f2c5e73cbddacf35dcaf9de499760d7093e3ae8f3ce4ea5ab28e25d065b7877607436fcbe02d21e17c2df940b0224f1ea7d638a602486ce807d99981
-  data.tar.gz: ecdda29924e09793e62684f16ebcd79cd90ab9e8204d011b89a577ca667a644709ff2df5b525ac6c32355426038c05d9fc3d74c6efb20ee7cfab653d9b89b67a
+  metadata.gz: d14c4c440b00f6fe0e689e506b77775b6bbfdf5e6a9c64c5310ccd84947a6f71880c52518575839534bb90be89cc7a198c9fd4d51831f6814f18380391abc8e2
+  data.tar.gz: 2c743fa2532fc672437240492faa2b6660194afff19c0ee27e6c6a9a8a402f7ae55ee0ecec84dad2ce474ecf86579218f47df578773e01d8409922f013cd9123

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -12,7 +12,8 @@ module Bulkrax
     def perform(entry_id, importer_run_id)
       @importer_run_id = importer_run_id
       entry = Entry.find(entry_id)
-      parent_identifier = entry.raw_metadata[entry.related_parents_raw_mapping]&.strip
+      # e.g. "parents" or "parents_1"
+      parent_identifier = (entry.raw_metadata[entry.related_parents_raw_mapping] || entry.raw_metadata["#{entry.related_parents_raw_mapping}_1"])&.strip
 
       validate_parent!(parent_identifier)
 
@@ -57,7 +58,7 @@ module Bulkrax
     end
 
     def check_parent_exists!(parent_identifier)
-      raise MissingParentError, %(Unable to find a record with the identifier "#{parent_identifier}") if parent_record.blank?
+      raise MissingParentError, %(Unable to find a record with the identifier "#{parent_identifier}") if parent_record.nil?
     end
 
     def check_parent_is_a_work!(parent_identifier)
@@ -66,8 +67,7 @@ module Bulkrax
     end
 
     def find_parent_record(parent_identifier)
-      @parent_record ||= find_record(parent_identifier, importer_run_id)
-      @parent_record = parent_record.last if parent_record.is_a? Array
+      _, @parent_record = find_record(parent_identifier, importer_run_id)
     end
   end
 end

data/app/models/bulkrax/exporter.rb

@@ -14,15 +14,13 @@ module Bulkrax
     validates :name, presence: true
     validates :parser_klass, presence: true
 
-    delegate :write, :create_from_collection, :create_from_collections_metadata, :create_from_importer, :create_from_worktype, :create_from_all, to: :parser
+    delegate :write, :create_from_collection, :create_from_importer, :create_from_worktype, :create_from_all, to: :parser
 
     def export
       current_run && setup_export_path
       case self.export_from
       when 'collection'
         create_from_collection
-      when 'collections metadata'
-        create_from_collections_metadata
       when 'importer'
         create_from_importer
       when 'worktype'
@@ -89,7 +87,6 @@ module Bulkrax
       [
         [I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
         [I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
-        [I18n.t('bulkrax.exporter.labels.collections_metadata'), 'collections metadata'],
         [I18n.t('bulkrax.exporter.labels.worktype'), 'worktype'],
         [I18n.t('bulkrax.exporter.labels.all'), 'all']
       ]

data/app/parsers/bulkrax/application_parser.rb

@@ -75,7 +75,7 @@ module Bulkrax
     def get_field_mapping_hash_for(key)
       return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
 
-      mapping = importerexporter.field_mapping == [{}] ? {} : importerexporter.field_mapping
+      mapping = importerexporter.field_mapping.is_a?(Hash) ? importerexporter.field_mapping : {}
       instance_variable_set(
         "@#{key}_hash",
         mapping&.with_indifferent_access&.select { |_, h| h.key?(key) }
@@ -264,6 +264,8 @@ module Bulkrax
 
       Dir["#{exporter_export_path}/**"].each do |folder|
         zip_path = "#{exporter_export_zip_path.split('/').last}_#{folder.split('/').last}.zip"
+        FileUtils.rm_rf("#{exporter_export_zip_path}/#{zip_path}")
+
         Zip::File.open(File.join("#{exporter_export_zip_path}/#{zip_path}"), create: true) do |zip_file|
           Dir["#{folder}/**/**"].each do |file|
             zip_file.add(file.sub("#{folder}/", ''), file)

data/app/parsers/bulkrax/bagit_parser.rb

@@ -97,28 +97,6 @@ module Bulkrax
       @total = 0
     end
 
-    def current_record_ids
-      @work_ids = []
-      @collection_ids = []
-      @file_set_ids = []
-
-      case importerexporter.export_from
-      when 'all'
-        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
-        @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
-      when 'collection'
-        @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
-      when 'worktype'
-        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
-      when 'importer'
-        set_ids_for_exporting_from_importer
-      end
-
-      find_child_file_sets(@work_ids) if importerexporter.export_from == 'collection' || importerexporter.export_from == 'worktype'
-
-      @work_ids + @collection_ids + @file_set_ids
-    end
-
     # export methods
 
     # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
@@ -128,14 +106,23 @@ module Bulkrax
 
       folder_count = 1
       records_in_folder = 0
+      work_entries = importerexporter.entries.where(identifier: @work_ids)
+      collection_entries = importerexporter.entries.where(identifier: @collection_ids)
+      file_set_entries = importerexporter.entries.where(identifier: @file_set_ids)
 
-      importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |entry|
+      work_entries[0..limit || total].each do |entry|
         record = ActiveFedora::Base.find(entry.identifier)
-        next unless Hyrax.config.curation_concerns.include?(record.class)
+        next unless record
 
         bag_entries = [entry]
-        file_set_entries = Bulkrax::CsvFileSetEntry.where(importerexporter_id: importerexporter.id).where("parsed_metadata LIKE '%#{record.id}%'")
-        file_set_entries.each { |fse| bag_entries << fse }
+
+        if record.member_of_collection_ids.present?
+          collection_entries.each { |ce| bag_entries << ce if ce.parsed_metadata.value?(record.id) }
+        end
+
+        if record.file_sets.present?
+          file_set_entries.each { |fse| bag_entries << fse if fse.parsed_metadata.value?(record.id) }
+        end
 
         records_in_folder += bag_entries.count
         if records_in_folder > records_split_count
@@ -153,7 +140,7 @@ module Bulkrax
           file.write(io.read)
           file.close
           begin
-            bag.add_file(file_name, file.path)
+            bag.add_file(file_name, file.path) if bag.bag_files.select { |b| b.include?(file_name) }.blank?
           rescue => e
             entry.status_info(e)
             status_info(e)

data/app/parsers/bulkrax/csv_parser.rb

@@ -184,6 +184,7 @@ module Bulkrax
       current_record_ids
     end
 
+    # rubocop:disable Metrics/AbcSize
     def current_record_ids
       @work_ids = []
       @collection_ids = []
@@ -195,18 +196,21 @@ module Bulkrax
         @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
         @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
       when 'collection'
-        @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
+        @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters} AND has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')})", method: :post, rows: 2_000_000_000).map(&:id)
+        # get the parent collection and child collections
         @collection_ids = ActiveFedora::SolrService.query("id:#{importerexporter.export_source} #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
-      when 'collections metadata'
-        @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
+        @collection_ids += ActiveFedora::SolrService.query("has_model_ssim:Collection AND member_of_collection_ids_ssim:#{importerexporter.export_source}", method: :post, rows: 2_147_483_647).map(&:id)
       when 'worktype'
         @work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
       when 'importer'
         set_ids_for_exporting_from_importer
       end
 
+      find_child_file_sets(@work_ids) if importerexporter.export_from == 'collection'
+
       @work_ids + @collection_ids + @file_set_ids
     end
+    # rubocop:enable Metrics/AbcSize
 
     # find the related file set ids so entries can be made for export
     def find_child_file_sets(work_ids)
@@ -262,7 +266,6 @@ module Bulkrax
       end
     end
     alias create_from_collection create_new_entries
-    alias create_from_collections_metadata create_new_entries
     alias create_from_importer create_new_entries
     alias create_from_worktype create_new_entries
     alias create_from_all create_new_entries
@@ -280,6 +283,10 @@ module Bulkrax
       CsvFileSetEntry
     end
 
+    def valid_entry_types
+      ['Bulkrax::CsvCollectionEntry', 'Bulkrax::CsvFileSetEntry', 'Bulkrax::CsvEntry']
+    end
+
     # TODO: figure out why using the version of this method that's in the bagit parser
     # breaks specs for the "if importer?" line
     def total
@@ -321,8 +328,10 @@ module Bulkrax
     def write_files
       require 'open-uri'
       folder_count = 0
+      sorted_entries = sort_entries(importerexporter.entries.uniq(&:identifier))
+                       .select { |e| valid_entry_types.include?(e.type) }
 
-      importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].in_groups_of(records_split_count, false) do |group|
+      sorted_entries[0..limit || total].in_groups_of(records_split_count, false) do |group|
         folder_count += 1
 
         CSV.open(setup_export_file(folder_count), "w", headers: export_headers, write_headers: true) do |csv|
@@ -338,6 +347,8 @@ module Bulkrax
 
     def store_files(identifier, folder_count)
       record = ActiveFedora::Base.find(identifier)
+      return unless record
+
       file_sets = record.file_set? ? Array.wrap(record) : record.file_sets
       file_sets << record.thumbnail if exporter.include_thumbnails && record.thumbnail.present? && record.work?
       file_sets.each do |fs|
@@ -383,6 +394,20 @@ module Bulkrax
       @object_names
     end
 
+    def sort_entries(entries)
+      # always export models in the same order: work, collection, file set
+      entries.sort_by do |entry|
+        case entry.type
+        when 'Bulkrax::CsvCollectionEntry'
+          '1'
+        when 'Bulkrax::CsvFileSetEntry'
+          '2'
+        else
+          '0'
+        end
+      end
+    end
+
     def sort_headers(headers)
       # converting headers like creator_name_1 to creator_1_name so they get sorted by numerical order
       # while keeping objects grouped together

data/app/views/bulkrax/exporters/show.html.erb

@@ -41,11 +41,6 @@
       <% when 'collection' %>
         <% collection = Collection.find(@exporter.export_source) %>
         <%= link_to collection&.title&.first, hyrax.dashboard_collection_path(collection.id) %>
-      <% when 'collections metadata' %>
-        <% collections = Collection.all %>
-        <% collections.each_with_index do |c, i| %>
-          <%= link_to c&.title&.first, hyrax.dashboard_collection_path(c.id) %><%= ',' if i != collections.count - 1 %>
-        <% end %>
       <% when 'importer' %>
         <% importer = Bulkrax::Importer.find(@exporter.export_source) %>
         <%= link_to importer.name, bulkrax.importer_path(importer.id) %>

data/config/locales/bulkrax.en.yml

@@ -8,7 +8,6 @@ en:
       labels:
         all: All
         collection: Collection
-        collections_metadata: All Collections' Metadata (only)
         export_format: Export Format
         export_from: Export From
         export_source: Export Source

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '4.0.0'
+  VERSION = '4.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Rob Kaufman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-15 00:00:00.000000000 Z
+date: 2022-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails