bulkrax 3.3.2 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b136a8742a7f9b953b4c3ef86d700540b931b6fbf22798d719e9aa693ea61fa9
-  data.tar.gz: e1cc32eda55a606285cf6e080340db608a8867f0bd4de8ab1361ca4a3d21adf9
+  metadata.gz: f0ee151bc10b7485eb716463b2c4895165d6df9d73c3dd60813d2eb4de8161d1
+  data.tar.gz: f4e5ddfb5ac602eb20a85850f8ec5a9286a9a668f8c83548dd049db4d91a2a0e
 SHA512:
-  metadata.gz: 87e35f340faa583a9ae6ac156f95fa4958fcaf9d3ed09a9963a8dda39649a66307a8c494375a989e2103c2d1edef62343f5b69878eae2249d63cc4b8f65240a8
-  data.tar.gz: 21ddafc671eda822144b6b73abfb89892ffac954f0692952985299f40f7d0e1aba472077aac72a1f681da315ba35de795f43d95df323c2178576716d02eafc75
+  metadata.gz: a5b029da7feaee11c8a3eb58e0c7150abcb06b6ca08f9d102134d5e1c9eef3049ae85b0084d74934b7504688da7d4958a0dc425c705b42101c1a3ce62a57d0c7
+  data.tar.gz: c807b68265c0d88b9e7faea4f6efdf94b3bcf90e965f5cc97cc259202bc55db976944eea8e4bf99876723f881d29d2fa7004dbd9985f7f8164648df517722133

data/app/jobs/bulkrax/export_work_job.rb

@@ -6,26 +6,26 @@ module Bulkrax
 
     def perform(*args)
       entry = Entry.find(args[0])
+      exporter_run = ExporterRun.find(args[1])
       begin
         entry.build
         entry.save
       rescue StandardError
         # rubocop:disable Rails/SkipsModelValidations
-        ExporterRun.find(args[1]).increment!(:failed_records)
-        ExporterRun.find(args[1]).decrement!(:enqueued_records)
+        exporter_run.increment!(:failed_records)
+        exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0
         raise
       else
         if entry.failed?
-          ExporterRun.find(args[1]).increment!(:failed_records)
-          ExporterRun.find(args[1]).decrement!(:enqueued_records)
+          exporter_run.increment!(:failed_records)
+          exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0
           raise entry.reload.current_status.error_class.constantize
         else
-          ExporterRun.find(args[1]).increment!(:processed_records)
-          ExporterRun.find(args[1]).decrement!(:enqueued_records)
+          exporter_run.increment!(:processed_records)
+          exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0
         end
         # rubocop:enable Rails/SkipsModelValidations
       end
-      exporter_run = ExporterRun.find(args[1])
       return entry if exporter_run.enqueued_records.positive?
 
       if exporter_run.failed_records.positive?

data/app/models/bulkrax/exporter.rb

@@ -14,13 +14,15 @@ module Bulkrax
     validates :name, presence: true
     validates :parser_klass, presence: true
 
-    delegate :write, :create_from_collection, :create_from_importer, :create_from_worktype, :create_from_all, to: :parser
+    delegate :write, :create_from_collection, :create_from_collections_metadata, :create_from_importer, :create_from_worktype, :create_from_all, to: :parser
 
     def export
       current_run && setup_export_path
       case self.export_from
       when 'collection'
         create_from_collection
+      when 'collections metadata'
+        create_from_collections_metadata
       when 'importer'
         create_from_importer
       when 'worktype'
@@ -87,6 +89,7 @@ module Bulkrax
       [
         [I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
         [I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
+        [I18n.t('bulkrax.exporter.labels.collections_metadata'), 'collections metadata'],
         [I18n.t('bulkrax.exporter.labels.worktype'), 'worktype'],
         [I18n.t('bulkrax.exporter.labels.all'), 'all']
       ]

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -51,7 +51,7 @@ module Bulkrax
       fn = file_set.original_file.file_name.first
       mime = Mime::Type.lookup(file_set.original_file.mime_type)
       ext_mime = MIME::Types.of(file_set.original_file.file_name).first
-      if fn.include?(file_set.id) || importerexporter.metadata_only? || importerexporter.parser_klass.include?('Bagit')
+      if fn.include?(file_set.id) || importerexporter.metadata_only?
         filename = "#{fn}.#{mime.to_sym}"
         filename = fn if mime.to_s == ext_mime.to_s
       else

data/app/parsers/bulkrax/application_parser.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'zip'
 
 module Bulkrax
   class ApplicationParser # rubocop:disable Metrics/ClassLength
@@ -261,7 +262,6 @@ module Bulkrax
     end
 
     def zip
-      require 'zip'
       FileUtils.rm_rf(exporter_export_zip_path)
       Zip::File.open(exporter_export_zip_path, create: true) do |zip_file|
         Dir["#{exporter_export_path}/**/**"].each do |file|

data/app/parsers/bulkrax/bagit_parser.rb

@@ -19,8 +19,14 @@ module Bulkrax
       rdf_format = parser_fields&.[]('metadata_format') == "Bulkrax::RdfEntry"
       rdf_format ? RdfEntry : CsvEntry
     end
-    alias collection_entry_class entry_class
-    alias file_set_entry_class entry_class
+
+    def collection_entry_class
+      CsvCollectionEntry
+    end
+
+    def file_set_entry_class
+      CsvFileSetEntry
+    end
 
     # Take a random sample of 10 metadata_paths and work out the import fields from that
     def import_fields
@@ -95,7 +101,15 @@ module Bulkrax
     end
 
     def total
-      importerexporter.entries.count
+      @total = importer.parser_fields['total'] || 0 if importer?
+
+      @total = if exporter?
+                 limit.nil? || limit.zero? ? current_record_ids.count : limit
+               end
+
+      return @total || 0
+    rescue StandardError
+      @total = 0
     end
 
     def extra_filters
@@ -118,7 +132,6 @@ module Bulkrax
       case importerexporter.export_from
       when 'all'
         @work_ids = ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
-        @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
         @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
       when 'collection'
         @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
@@ -127,7 +140,6 @@ module Bulkrax
       when 'importer'
         set_ids_for_exporting_from_importer
       end
-
       @work_ids + @collection_ids + @file_set_ids
     end
 
@@ -155,6 +167,8 @@ module Bulkrax
       end
     end
 
+    # export methods
+
     def create_new_entries
       current_record_ids.each_with_index do |id, index|
         break if limit_reached?(limit, index)
@@ -182,34 +196,44 @@ module Bulkrax
     alias create_from_worktype create_new_entries
     alias create_from_all create_new_entries
 
-    # export methods
-
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
     def write_files
       require 'open-uri'
       require 'socket'
-      importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |e|
-        bag = BagIt::Bag.new setup_bagit_folder(e.identifier)
-        w = ActiveFedora::Base.find(e.identifier)
-        next unless Hyrax.config.curation_concerns.include?(w.class)
+      importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |entry|
+        work = ActiveFedora::Base.find(entry.identifier)
+        next unless Hyrax.config.curation_concerns.include?(work.class)
+        bag = BagIt::Bag.new setup_bagit_folder(entry.identifier)
+        bag_entries = [entry]
+
+        work.file_sets.each do |fs|
+          if @file_set_ids.present?
+            file_set_entry = Bulkrax::CsvFileSetEntry.where("parsed_metadata LIKE '%#{fs.id}%'").first
+            bag_entries << file_set_entry unless file_set_entry.nil?
+          end
 
-        w.file_sets.each do |fs|
           file_name = filename(fs)
           next if file_name.blank?
           io = open(fs.original_file.uri)
           file = Tempfile.new([file_name, File.extname(file_name)], binmode: true)
           file.write(io.read)
           file.close
-          bag.add_file(file_name, file.path)
+          begin
+            bag.add_file(file_name, file.path)
+          rescue => e
+            entry.status_info(e)
+            status_info(e)
+          end
         end
-        CSV.open(setup_csv_metadata_export_file(e.identifier), "w", headers: export_headers, write_headers: true) do |csv|
-          csv << e.parsed_metadata
+
+        CSV.open(setup_csv_metadata_export_file(entry.identifier), "w", headers: export_headers, write_headers: true) do |csv|
+          bag_entries.each { |csv_entry| csv << csv_entry.parsed_metadata }
         end
-        write_triples(e)
+        write_triples(entry)
         bag.manifest!(algo: 'sha256')
       end
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
     def setup_csv_metadata_export_file(id)
       File.join(importerexporter.exporter_export_path, id, 'metadata.csv')

data/app/parsers/bulkrax/csv_parser.rb

@@ -195,6 +195,9 @@ module Bulkrax
         @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
       when 'collection'
         @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
+        @collection_ids = ActiveFedora::SolrService.query("id:#{importerexporter.export_source} #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
+      when 'collections metadata'
+        @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
       when 'worktype'
         @work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
       when 'importer'
@@ -251,6 +254,7 @@ module Bulkrax
       end
     end
     alias create_from_collection create_new_entries
+    alias create_from_collections_metadata create_new_entries
     alias create_from_importer create_new_entries
     alias create_from_worktype create_new_entries
     alias create_from_all create_new_entries

data/app/views/bulkrax/exporters/_form.html.erb

@@ -15,15 +15,15 @@
 
   <%= form.hidden_field :user_id, value: current_user.id %>
 
-  <%= form.input :export_type, 
-    collection:  form.object.export_type_list, 
-    label: t('bulkrax.exporter.labels.export_type'), 
+  <%= form.input :export_type,
+    collection:  form.object.export_type_list,
+    label: t('bulkrax.exporter.labels.export_type'),
     required: true,
     prompt: 'Please select an export type' %>
 
-  <%= form.input :export_from, 
-    collection: form.object.export_from_list, 
-    label: t('bulkrax.exporter.labels.export_from'), 
+  <%= form.input :export_from,
+    collection: form.object.export_from_list,
+    label: t('bulkrax.exporter.labels.export_from'),
     required: true,
     prompt: 'Please select an export source' %>
 
@@ -55,8 +55,8 @@
     input_html: { class: 'worktype export-source-option hidden' },
     collection: Hyrax.config.curation_concerns.map {|cc| [cc.to_s, cc.to_s] } %>
 
-  <%= form.input :limit, 
-    as: :integer, 
+  <%= form.input :limit,
+    as: :integer,
     hint: 'leave blank or 0 for all records',
     label: t('bulkrax.exporter.labels.limit') %>
 
@@ -90,8 +90,8 @@
                  collection: form.object.workflow_status_list,
                  label: t('bulkrax.exporter.labels.status') %>
 
-  <%= form.input :parser_klass, 
-    collection: Bulkrax.parsers.map {|p| [p[:name], p[:class_name], {'data-partial' => p[:partial]}] if p[:class_name].constantize.export_supported? }.compact, 
+  <%= form.input :parser_klass,
+    collection: Bulkrax.parsers.map {|p| [p[:name], p[:class_name], {'data-partial' => p[:partial]}] if p[:class_name].constantize.export_supported? }.compact,
     label: t('bulkrax.exporter.labels.export_format') %>
 </div>
 

data/app/views/bulkrax/exporters/show.html.erb

@@ -40,6 +40,11 @@
       <% when 'collection' %>
         <% collection = Collection.find(@exporter.export_source) %>
         <%= link_to collection&.title&.first, hyrax.dashboard_collection_path(collection.id) %>
+      <% when 'collections metadata' %>
+        <% collections = Collection.all %>
+        <% collections.each_with_index do |c, i| %>
+          <%= link_to c&.title&.first, hyrax.dashboard_collection_path(c.id) %><%= ',' if i != collections.count - 1 %>
+        <% end %>
       <% when 'importer' %>
         <% importer = Bulkrax::Importer.find(@exporter.export_source) %>
         <%= link_to importer.name, bulkrax.importer_path(importer.id) %>

data/config/locales/bulkrax.en.yml

@@ -8,6 +8,7 @@ en:
       labels:
         all: All
         collection: Collection
+        collections_metadata: All Collections' Metadata (only)
         export_format: Export Format
         export_from: Export From
         export_source: Export Source

data/db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb

@@ -1,7 +1,17 @@
 class RenameBulkraxImporterRunToImporterRun < ActiveRecord::Migration[5.2]
-  def change
+  def up
     if column_exists?(:bulkrax_pending_relationships, :bulkrax_importer_run_id)
+      remove_foreign_key :bulkrax_pending_relationships, :bulkrax_importer_runs
+      remove_index :bulkrax_pending_relationships, column: :bulkrax_importer_run_id
+
       rename_column :bulkrax_pending_relationships, :bulkrax_importer_run_id, :importer_run_id
+
+      add_foreign_key :bulkrax_pending_relationships, :bulkrax_importer_runs, column: :importer_run_id
+      add_index :bulkrax_pending_relationships, :importer_run_id, name: 'index_bulkrax_pending_relationships_on_importer_run_id'
     end
   end
+
+  def down
+    rename_column :bulkrax_pending_relationships, :importer_run_id, :bulkrax_importer_run_id
+  end
 end

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '3.3.2'
+  VERSION = '3.4.0'
 end

data/lib/generators/bulkrax/install_generator.rb

@@ -10,7 +10,7 @@ class Bulkrax::InstallGenerator < Rails::Generators::Base
   end
 
   def add_to_gemfile
-    gem 'willow_sword', github: 'notch8/willow_sword'
+    gem 'bulkrax'
 
     Bundler.with_clean_env do
       run "bundle install"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 3.3.2
+  version: 3.4.0
 platform: ruby
 authors:
 - Rob Kaufman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-10 00:00:00.000000000 Z
+date: 2022-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -25,81 +25,81 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.1.6
 - !ruby/object:Gem::Dependency
-  name: loofah
+  name: bagit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.3
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.3
+        version: '0.4'
 - !ruby/object:Gem::Dependency
-  name: rack
+  name: coderay
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: simple_form
+  name: iso8601
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
-  name: iso8601
+  name: kaminari
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: oai
+  name: language_list
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
-    - - "<"
+        version: '1.2'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.x
+        version: 1.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
-    - - "<"
+        version: '1.2'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.x
+        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: libxml-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -115,61 +115,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
-  name: language_list
+  name: loofah
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 2.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 2.2.3
 - !ruby/object:Gem::Dependency
-  name: rdf
+  name: oai
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: '0.4'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: 2.x
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: '0.4'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: 2.x
 - !ruby/object:Gem::Dependency
-  name: bagit
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 2.0.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 2.0.6
 - !ruby/object:Gem::Dependency
-  name: coderay
+  name: rdf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -183,7 +197,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: kaminari
+  name: simple_form
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="