bulkrax 3.0.0 → 3.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3976f63546f72c7a1fcc47ba595aa3e6486f2f6bcf7fb39a65f9cceb61513dc2
-  data.tar.gz: 267eb1e87602fdc7a510eff6ac61a459c11a1bbc3691b0b07e9aebd9d84de743
+  metadata.gz: 41dde3161532c80ff433be4697fa6c42f0f453dfea3547181c44914a68e3b466
+  data.tar.gz: 67473e62b537f71aa77aa664c361ba1e86c60e77451e4813a44597a9f465010b
 SHA512:
-  metadata.gz: 6512d5c1169d1024b9b5e16847772cb8a662b98638127204e198e09463a54bd3c93f36eac12460e20c43156148ff546a85fbea3d01dd9a0a55df179c56fc8b5d
-  data.tar.gz: f80a6017b2247235a10b7d23fbbd5199464ac018cd7a9d5609ad8a6c97898e7b20b718c8cb2d9821231de9654f68caf6f39359eb82270c4832d15385ad1f7d12
+  metadata.gz: 3859148384111048a2a2c096cee9bde5a9c78cb8e5a73b5ddb1d31c3ddefe7a221d1ebff97e3b1f96f09e746ac60b3ec1c14e615ef162fe1a5e16f883a3a7ecf
+  data.tar.gz: bbc83265d59d5026546e6e92ccab3b93d6d4bf35ae08145ab6b27112ed587449572d763fdf3d876ec391dfce43d8b5fa96154e8507053ff2fac7b81bc2409580

data/README.md

@@ -1,7 +1,6 @@
 # Bulkrax
 Bulkrax is a batteries included importer for Samvera applications. It currently includes support for OAI-PMH (DC and Qualified DC) and CSV out of the box. It is also designed to be extensible, allowing you to easily add new importers in to your application or to include them with other gems. Bulkrax provides a full admin interface including creating, editing, scheduling and reviewing imports.
 
-
 ## Installation
 
 ### Install Generator
@@ -9,9 +8,9 @@ Bulkrax is a batteries included importer for Samvera applications. It currently
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'bulkrax', '1.0.0'
+gem 'bulkrax'
 # or if using from github
-gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git'
+gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git', branch: 'main'
 ```
 
 And then execute:
@@ -27,7 +26,7 @@ If using Sidekiq, set up queues for `import` and `export`.
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git'
+gem 'bulkrax'
 ```
 
 And then execute:

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -46,18 +46,16 @@ module Bulkrax
       end
 
       if (child_records[:collections].blank? && child_records[:works].blank?) || parent_record.blank?
-        reschedule(
-          parent_identifier: parent_identifier,
-          importer_run_id: importer_run_id
-        )
+        reschedule({ parent_identifier: parent_identifier, importer_run_id: importer_run_id })
         return false # stop current job from continuing to run after rescheduling
       end
-
+      importer_id = ImporterRun.find(importer_run_id).importer_id
       @parent_entry ||= Bulkrax::Entry.where(identifier: parent_identifier,
-                                             importerexporter_id: ImporterRun.find(importer_run_id).importer_id,
+                                             importerexporter_id: importer_id,
                                              importerexporter_type: "Bulkrax::Importer").first
       create_relationships
       pending_relationships.each(&:destroy)
+      Bulkrax::Importer.find(importer_id).record_status
     rescue ::StandardError => e
       parent_entry ? parent_entry.status_info(e) : child_entry.status_info(e)
       Bulkrax::ImporterRun.find(importer_run_id).increment!(:failed_relationships) # rubocop:disable Rails/SkipsModelValidations
@@ -82,18 +80,18 @@ module Bulkrax
     # Work-Collection membership is added to the child as member_of_collection_ids
     # This is adding the reverse relationship, from the child to the parent
     def collection_parent_work_child
-      child_records[:works].each do |child_record|
-        ::Hyrax::Collections::NestedCollectionPersistenceService.persist_nested_collection_for(parent: parent_record, child: child_record)
-        # TODO: add counters for :processed_parents and :failed_parents
-        Bulkrax::ImporterRun.find(importer_run_id).increment!(:processed_relationships) # rubocop:disable Rails/SkipsModelValidations
-      end
+      child_work_ids = child_records[:works].map(&:id)
+      parent_record.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX
+
+      parent_record.add_member_objects(child_work_ids)
+      ImporterRun.find(importer_run_id).increment!(:processed_relationships, child_work_ids.count) # rubocop:disable Rails/SkipsModelValidations
     end
 
     # Collection-Collection membership is added to the as member_ids
     def collection_parent_collection_child
       child_records[:collections].each do |child_record|
         ::Hyrax::Collections::NestedCollectionPersistenceService.persist_nested_collection_for(parent: parent_record, child: child_record)
-        Bulkrax::ImporterRun.find(importer_run_id).increment!(:processed_relationships) # rubocop:disable Rails/SkipsModelValidations
+        ImporterRun.find(importer_run_id).increment!(:processed_relationships) # rubocop:disable Rails/SkipsModelValidations
       end
     end
 
@@ -103,14 +101,12 @@ module Bulkrax
       child_records[:works].each_with_index do |child_record, i|
         records_hash[i] = { id: child_record.id }
       end
-      attrs = {
-        work_members_attributes: records_hash
-      }
-      parent_record.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if parent_record.respond_to?(:reindex_extent)
+      attrs = { work_members_attributes: records_hash }
+      parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
       env = Hyrax::Actors::Environment.new(parent_record, Ability.new(user), attrs)
+
       Hyrax::CurationConcern.actor.update(env)
-      # TODO: add counters for :processed_parents and :failed_parents
-      Bulkrax::ImporterRun.find(importer_run_id).increment!(:processed_relationships) # rubocop:disable Rails/SkipsModelValidations
+      ImporterRun.find(importer_run_id).increment!(:processed_relationships, child_records[:works].count) # rubocop:disable Rails/SkipsModelValidations
     end
 
     def reschedule(parent_identifier:, importer_run_id:)

data/app/jobs/bulkrax/export_work_job.rb

@@ -27,6 +27,7 @@ module Bulkrax
       end
       exporter_run = ExporterRun.find(args[1])
       return entry if exporter_run.enqueued_records.positive?
+
       if exporter_run.failed_records.positive?
         exporter_run.exporter.status_info('Complete (with failures)')
       else

data/app/jobs/bulkrax/import_collection_job.rb

@@ -9,16 +9,18 @@ module Bulkrax
       entry = Entry.find(args[0])
       begin
         entry.build
-        entry.save
+        entry.save!
         ImporterRun.find(args[1]).increment!(:processed_records)
         ImporterRun.find(args[1]).increment!(:processed_collections)
-        ImporterRun.find(args[1]).decrement!(:enqueued_records)
+        ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
       rescue => e
         ImporterRun.find(args[1]).increment!(:failed_records)
         ImporterRun.find(args[1]).increment!(:failed_collections)
-        ImporterRun.find(args[1]).decrement!(:enqueued_records)
+        ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
         raise e
       end
+      entry.importer.current_run = ImporterRun.find(args[1])
+      entry.importer.record_status
     end
     # rubocop:enable Rails/SkipsModelValidations
   end

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -26,17 +26,17 @@ module Bulkrax
         ImporterRun.find(importer_run_id).increment!(:failed_file_sets)
         # rubocop:enable Rails/SkipsModelValidations
       end
-      ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
+      ImporterRun.find(importer_run_id).decrement!(:enqueued_records) unless ImporterRun.find(importer_run_id).enqueued_records <= 0 # rubocop:disable Rails/SkipsModelValidations
       entry.save!
+      entry.importer.current_run = ImporterRun.find(importer_run_id)
+      entry.importer.record_status
 
     rescue MissingParentError => e
       # try waiting for the parent record to be created
       entry.import_attempts += 1
       entry.save!
       if entry.import_attempts < 5
-        ImportFileSetJob
-          .set(wait: (entry.import_attempts + 1).minutes)
-          .perform_later(entry_id, importer_run_id)
+        ImportFileSetJob.set(wait: (entry.import_attempts + 1).minutes).perform_later(entry_id, importer_run_id)
       else
         ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
         entry.status_info(e)

data/app/models/bulkrax/csv_entry.rb

@@ -46,6 +46,7 @@ module Bulkrax
       add_visibility
       add_metadata_for_model
       add_rights_statement
+      sanitize_controlled_uri_values!
       add_local
 
       self.parsed_metadata

data/app/models/bulkrax/csv_file_set_entry.rb

@@ -2,31 +2,6 @@
 
 module Bulkrax
   class CsvFileSetEntry < CsvEntry
-    def factory_class
-      ::FileSet
-    end
-
-    def add_path_to_file
-      parsed_metadata['file'].each_with_index do |filename, i|
-        path_to_file = ::File.join(parser.path_to_files, filename)
-
-        parsed_metadata['file'][i] = path_to_file
-      end
-      raise ::StandardError, "one or more file paths are invalid: #{parsed_metadata['file'].join(', ')}" unless parsed_metadata['file'].map { |file_path| ::File.file?(file_path) }.all?
-
-      parsed_metadata['file']
-    end
-
-    def validate_presence_of_filename!
-      return if parsed_metadata&.[]('file')&.map(&:present?)&.any?
-
-      raise StandardError, 'File set must have a filename'
-    end
-
-    def validate_presence_of_parent!
-      return if parsed_metadata[related_parents_parsed_mapping]&.map(&:present?)&.any?
-
-      raise StandardError, 'File set must be related to at least one work'
-    end
+    include FileSetEntryBehavior
   end
 end

data/app/models/bulkrax/exporter.rb

@@ -96,7 +96,8 @@ module Bulkrax
     end
 
     def current_run
-      @current_run ||= self.exporter_runs.create!(total_work_entries: self.limit || parser.total)
+      total = self.limit || parser.total
+      @current_run ||= self.exporter_runs.create!(total_work_entries: total, enqueued_records: total)
     end
 
     def last_run

data/app/models/bulkrax/importer.rb

@@ -142,7 +142,7 @@ module Bulkrax
 
     def import_objects(types_array = nil)
       self.only_updates ||= false
-      types = types_array || %w[work collection file_set relationship]
+      types = types_array || %w[collection work file_set relationship]
       if parser.class == Bulkrax::CsvParser
         parser.create_objects(types)
       else

data/app/models/bulkrax/rdf_file_set_entry.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class RdfFileSetEntry < RdfEntry
+    include FileSetEntryBehavior
+  end
+end

data/app/models/concerns/bulkrax/file_set_entry_behavior.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module FileSetEntryBehavior
+    def factory_class
+      ::FileSet
+    end
+
+    def add_path_to_file
+      parsed_metadata['file'].each_with_index do |filename, i|
+        path_to_file = ::File.join(parser.path_to_files, filename)
+
+        parsed_metadata['file'][i] = path_to_file
+      end
+      raise ::StandardError, "one or more file paths are invalid: #{parsed_metadata['file'].join(', ')}" unless parsed_metadata['file'].map { |file_path| ::File.file?(file_path) }.all?
+
+      parsed_metadata['file']
+    end
+
+    def validate_presence_of_filename!
+      return if parsed_metadata&.[]('file')&.map(&:present?)&.any?
+
+      raise StandardError, 'File set must have a filename'
+    end
+
+    def validate_presence_of_parent!
+      return if parsed_metadata[related_parents_parsed_mapping]&.map(&:present?)&.any?
+
+      raise StandardError, 'File set must be related to at least one work'
+    end
+  end
+end

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   # Import Behavior for Entry classes
-  module ImportBehavior
+  module ImportBehavior # rubocop:disable Metrics/ModuleLength
     extend ActiveSupport::Concern
 
     def build_for_importer
@@ -21,6 +21,8 @@ module Bulkrax
         status_info(e)
       else
         status_info
+      ensure
+        self.save!
       end
       return @item
     end
@@ -103,6 +105,65 @@ module Bulkrax
       end
     end
 
+    # Attempt to sanitize Questioning Authority URI values for configured controlled fields of common
+    # data entry mistakes. Controlled URI values are only valid if they are an exact match.
+    # Example:
+    #   Valid value:     http://rightsstatements.org/vocab/InC/1.0/
+    #   Provided value:  https://rightsstatements.org/vocab/InC/1.0
+    #   Sanitized value: http://rightsstatements.org/vocab/InC/1.0/ ("s" from "https" removed, trailing "/" added)
+    #
+    # @return [Boolean] true if all controlled URI values are sanitized successfully
+    def sanitize_controlled_uri_values!
+      Bulkrax.qa_controlled_properties.each do |field|
+        next if parsed_metadata[field].blank?
+
+        parsed_metadata[field].each_with_index do |value, i|
+          next if value.blank?
+
+          if (validated_uri_value = validate_value(value, field))
+            parsed_metadata[field][i] = validated_uri_value
+          else
+            debug_msg = %(Unable to locate active authority ID "#{value}" in config/authorities/#{field.pluralize}.yml)
+            Rails.logger.debug(debug_msg)
+            error_msg = %("#{value}" is not a valid and/or active authority ID for the :#{field} field)
+            raise ::StandardError, error_msg
+          end
+        end
+      end
+
+      true
+    end
+
+    # @param value [String] value to validate
+    # @param field [String] name of the controlled property
+    # @return [String, nil] validated URI value or nil
+    def validate_value(value, field)
+      if value.match?(::URI::DEFAULT_PARSER.make_regexp)
+        value = value.strip.chomp
+        # add trailing forward slash unless one is already present
+        value << '/' unless value.match?(%r{/$})
+      end
+
+      valid = if active_id_for_authority?(value, field)
+                true
+              else
+                value.include?('https') ? value.sub!('https', 'http') : value.sub!('http', 'https')
+                active_id_for_authority?(value, field)
+              end
+
+      valid ? value : nil
+    end
+
+    # @param value [String] value to check
+    # @param field [String] name of the controlled property
+    # @return [Boolean] provided value is a present, active authority ID for the provided field
+    def active_id_for_authority?(value, field)
+      field_service = ('Hyrax::' + "#{field}_service".camelcase).constantize
+      active_authority_ids = field_service.new.active_elements.map { |ae| ae['id'] }
+
+      active_authority_ids.include?(value)
+    end
+
     def factory
       @factory ||= Bulkrax::ObjectFactory.new(attributes: self.parsed_metadata,
                                               source_identifier_value: identifier,

data/app/parsers/bulkrax/bagit_parser.rb

@@ -23,6 +23,11 @@ module Bulkrax
       Entry
     end
 
+    def file_set_entry_class
+      csv_format = Bulkrax::Importer.last.parser_fields['metadata_format'] == "Bulkrax::CsvEntry"
+      csv_format ? CsvFileSetEntry : RdfFileSetEntry
+    end
+
     # Take a random sample of 10 metadata_paths and work out the import fields from that
     def import_fields
       raise StandardError, 'No metadata files were found' if metadata_paths.blank?

data/app/parsers/bulkrax/csv_parser.rb

@@ -217,7 +217,7 @@ module Bulkrax
         instance_variable_set(instance_var, ActiveFedora::SolrService.post(
           extra_filters.to_s,
           fq: [
-            "#{::Solrizer.solr_name(work_identifier)}:(#{complete_entry_identifiers.join(' OR ')})",
+            %(#{::Solrizer.solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
             "has_model_ssim:(#{models_to_search.join(' OR ')})"
           ],
           fl: 'id',
@@ -270,7 +270,7 @@ module Bulkrax
     #   Changed to grep as wc -l counts blank lines, and ignores the final unescaped line (which may or may not contain data)
     def total
       @total = importer.parser_fields['total'] || 0 if importer?
-      @total = importerexporter.entries.count if exporter?
+      @total = limit || current_record_ids.count if exporter?
 
       return @total || 0
     rescue StandardError

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '3.0.0'
+  VERSION = '3.1.1'
 end

data/lib/bulkrax.rb

@@ -12,13 +12,13 @@ module Bulkrax
                    :related_children_field_mapping,
                    :related_parents_field_mapping,
                    :reserved_properties,
+                   :qa_controlled_properties,
                    :field_mappings,
                    :import_path,
                    :export_path,
                    :removed_image_path,
                    :server_name,
-                   :api_definition,
-                   :removed_image_path
+                   :api_definition
 
     self.parsers = [
       { name: "OAI - Dublin Core", class_name: "Bulkrax::OaiDcParser", partial: "oai_fields" },
@@ -119,6 +119,11 @@ module Bulkrax
       original_url
       relative_path
     ]
+
+    # List of Questioning Authority properties that are controlled via YAML files in
+    # the config/authorities/ directory. For example, the :rights_statement property
+    # is controlled by the active terms in config/authorities/rights_statements.yml
+    self.qa_controlled_properties = %w[rights_statement license]
   end
 
   def self.api_definition
@@ -131,8 +136,6 @@ module Bulkrax
     )
   end
 
-  self.removed_image_path = 'app/assets/images/bulkrax/removed.png'
-
   # this function maps the vars from your app into your engine
   def self.setup
     yield self

data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb

@@ -61,6 +61,12 @@ Bulkrax.setup do |config|
 
   # Properties that should not be used in imports/exports. They are reserved for use by Hyrax.
   # config.reserved_properties += ['my_field']
+
+  # List of Questioning Authority properties that are controlled via YAML files in
+  # the config/authorities/ directory. For example, the :rights_statement property
+  # is controlled by the active terms in config/authorities/rights_statements.yml
+  # Defaults: 'rights_statement' and 'license'
+  # config.qa_controlled_properties += ['my_field']
 end
 
 # Sidebar for hyrax 3+ support

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.1.1
 platform: ruby
 authors:
 - Rob Kaufman
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-18 00:00:00.000000000 Z
+date: 2022-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -291,6 +291,7 @@ files:
 - app/models/bulkrax/pending_relationship.rb
 - app/models/bulkrax/rdf_collection_entry.rb
 - app/models/bulkrax/rdf_entry.rb
+- app/models/bulkrax/rdf_file_set_entry.rb
 - app/models/bulkrax/status.rb
 - app/models/bulkrax/xml_entry.rb
 - app/models/concerns/bulkrax/download_behavior.rb
@@ -298,6 +299,7 @@ files:
 - app/models/concerns/bulkrax/errored_entries.rb
 - app/models/concerns/bulkrax/export_behavior.rb
 - app/models/concerns/bulkrax/file_factory.rb
+- app/models/concerns/bulkrax/file_set_entry_behavior.rb
 - app/models/concerns/bulkrax/has_local_processing.rb
 - app/models/concerns/bulkrax/has_matchers.rb
 - app/models/concerns/bulkrax/import_behavior.rb
@@ -382,7 +384,7 @@ homepage: https://github.com/samvera-labs/bulkrax
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -397,8 +399,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: Import and export tool for Hyrax and Hyku
 test_files: []