bulkrax 7.0.0 → 8.1.0

data/app/jobs/bulkrax/importer_job.rb

@@ -6,6 +6,7 @@ module Bulkrax
 
     def perform(importer_id, only_updates_since_last_import = false)
       importer = Importer.find(importer_id)
+      return schedule(importer, Time.zone.now + 3.minutes, 'Rescheduling: cloud files are not ready yet') unless all_files_completed?(importer)
 
       importer.current_run
       unzip_imported_file(importer.parser)
@@ -16,6 +17,8 @@ module Bulkrax
       importer.set_status_info(e)
     end
 
+    private
+
     def import(importer, only_updates_since_last_import)
       importer.only_updates = only_updates_since_last_import || false
       return unless importer.valid_import?
@@ -36,8 +39,21 @@ module Bulkrax
       importer.current_run.save!
     end
 
-    def schedule(importer)
-      ImporterJob.set(wait_until: importer.next_import_at).perform_later(importer.id, true)
+    def schedule(importer, wait_until = importer.next_import_at, message = nil)
+      Rails.logger.info message if message
+      ImporterJob.set(wait_until: wait_until).perform_later(importer.id, true)
+    end
+
+    # checks the file sizes of the download files to match the original files
+    def all_files_completed?(importer)
+      cloud_files = importer.parser_fields['cloud_file_paths']
+      original_files = importer.parser_fields['original_file_paths']
+      return true unless cloud_files.present? && original_files.present?
+
+      imported_file_sizes = cloud_files.map { |_, v| v['file_size'].to_i }
+      original_file_sizes = original_files.map { |imported_file| File.size(imported_file) }
+
+      original_file_sizes == imported_file_sizes
     end
   end
 end

data/app/matchers/bulkrax/application_matcher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'language_list'
-
 module Bulkrax
   class ApplicationMatcher
     attr_accessor :to, :from, :parsed, :if, :split, :excluded, :nested_type

data/app/models/bulkrax/csv_collection_entry.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class CsvCollectionEntry < CsvEntry
-    self.default_work_type = "Collection"
+    self.default_work_type = Bulkrax.collection_model_class.to_s
 
     # Use identifier set by CsvParser#unique_collection_identifier, which falls back
     # on the Collection's first title if record[source_identifier] is not present

data/app/models/bulkrax/csv_entry.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'csv'
-
 module Bulkrax
   # TODO: We need to rework this class some to address the Metrics/ClassLength rubocop offense.
   # We do too much in these entry classes. We need to extract the common logic from the various
@@ -106,7 +104,7 @@ module Bulkrax
     end
 
     def add_metadata_for_model
-      if defined?(::Collection) && factory_class == ::Collection
+      if factory_class.present? && factory_class == Bulkrax.collection_model_class
         add_collection_type_gid if defined?(::Hyrax)
         # add any additional collection metadata methods here
       elsif factory_class == Bulkrax.file_model_class
@@ -146,7 +144,7 @@ module Bulkrax
       self.parsed_metadata = {}
 
       build_system_metadata
-      build_files_metadata if defined?(Collection) && !hyrax_record.is_a?(Collection)
+      build_files_metadata if Bulkrax.collection_model_class.present? && !hyrax_record.is_a?(Bulkrax.collection_model_class)
       build_relationship_metadata
       build_mapping_metadata
       self.save!
@@ -158,9 +156,12 @@ module Bulkrax
     def build_system_metadata
       self.parsed_metadata['id'] = hyrax_record.id
       source_id = hyrax_record.send(work_identifier)
-      source_id = source_id.to_a.first if source_id.is_a?(ActiveTriples::Relation)
+      # Because ActiveTriples::Relation does not respond to #to_ary we can't rely on Array.wrap universally
+      source_id = source_id.to_a if source_id.is_a?(ActiveTriples::Relation)
+      source_id = Array.wrap(source_id).first
       self.parsed_metadata[source_identifier] = source_id
-      self.parsed_metadata[key_for_export('model')] = hyrax_record.has_model.first
+      model_name = hyrax_record.respond_to?(:to_rdf_representation) ? hyrax_record.to_rdf_representation : hyrax_record.has_model.first
+      self.parsed_metadata[key_for_export('model')] = model_name
     end
 
     def build_files_metadata

data/app/models/bulkrax/entry.rb

@@ -103,22 +103,18 @@ module Bulkrax
       self.importerexporter_type == 'Bulkrax::Exporter'
     end
 
-    def valid_system_id(model_class)
-      return true if model_class.properties.keys.include?(work_identifier)
-      raise(
-        "#{model_class} does not implement the system_identifier_field: #{work_identifier}"
-      )
-    end
-
     def last_run
       self.importerexporter&.last_run
     end
 
     def find_collection(collection_identifier)
-      return unless Collection.properties.keys.include?(work_identifier)
-      Collection.where(
-        work_identifier => collection_identifier
-      ).detect { |m| m.send(work_identifier).include?(collection_identifier) }
+      Bulkrax.object_factory.search_by_property(
+        klass: Bulkrax.collection_model_class,
+        value: collection_identifier,
+        search_field: work_identifier,
+        name_field: work_identifier,
+        verify_property: true
+      )
     end
   end
 end

data/app/models/bulkrax/exporter.rb

@@ -137,8 +137,8 @@ module Bulkrax
     end
 
     def export_properties
-      properties = Bulkrax.curation_concerns.map { |work| work.properties.keys }.flatten.uniq.sort
-      properties.reject { |prop| Bulkrax.reserved_properties.include?(prop) }
+      # TODO: Does this work for Valkyrie?
+      Bulkrax.object_factory.export_properties
     end
 
     def metadata_only?

data/app/models/bulkrax/importer.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'iso8601'
-
 module Bulkrax
   class Importer < ApplicationRecord # rubocop:disable Metrics/ClassLength
     include Bulkrax::ImporterExporterBehavior
@@ -18,7 +16,7 @@ module Bulkrax
     validates :admin_set_id, presence: true if defined?(::Hyrax)
     validates :parser_klass, presence: true
 
-    delegate :valid_import?, :write_errored_entries_file, :visibility, to: :parser
+    delegate :create_parent_child_relationships, :valid_import?, :write_errored_entries_file, :visibility, to: :parser
 
     attr_accessor :only_updates, :file_style, :file
     attr_writer :current_run

data/app/models/bulkrax/oai_entry.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'erb'
-require 'ostruct'
-
 module Bulkrax
   class OaiEntry < Entry
     serialize :raw_metadata, Bulkrax::NormalizedJson

data/app/models/bulkrax/oai_set_entry.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class OaiSetEntry < OaiEntry
-    self.default_work_type = "Collection"
+    self.default_work_type = Bulkrax.collection_model_class.to_s
 
     def build_metadata
       self.parsed_metadata = self.raw_metadata

data/app/models/bulkrax/rdf_collection_entry.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class RdfCollectionEntry < RdfEntry
-    self.default_work_type = "Collection"
+    self.default_work_type = Bulkrax.collection_model_class.to_s
     def record
       @record ||= self.raw_metadata
     end

data/app/models/bulkrax/rdf_entry.rb

@@ -1,91 +1,92 @@
 # frozen_string_literal: true
 
-require 'rdf'
-module Bulkrax
-  class RdfEntry < Entry
-    serialize :raw_metadata, Bulkrax::NormalizedJson
+unless ENV.fetch('BULKRAX_NO_RDF', 'false').to_s == 'true'
+  module Bulkrax
+    class RdfEntry < Entry
+      serialize :raw_metadata, Bulkrax::NormalizedJson
 
-    def self.read_data(path)
-      RDF::Reader.open(path)
-    end
+      def self.read_data(path)
+        RDF::Reader.open(path)
+      end
 
-    def self.fields_from_data(data)
-      data.predicates.map(&:to_s)
-    end
+      def self.fields_from_data(data)
+        data.predicates.map(&:to_s)
+      end
 
-    def self.data_for_entry(data, source_id, parser)
-      reader = data
-      format = reader.class.format.to_sym
-      collections = []
-      children = []
-      delete = nil
-      data = RDF::Writer.for(format).buffer do |writer|
-        reader.each_statement do |statement|
-          collections << statement.object.to_s if parent_field(parser).present? && parent_field(parser) == statement.predicate.to_s
-          children << statement.object.to_s if related_children_parsed_mapping.present? && related_children_parsed_mapping == statement.predicate.to_s
-          delete = statement.object.to_s if /deleted/.match?(statement.predicate.to_s)
-          writer << statement
+      def self.data_for_entry(data, source_id, parser)
+        reader = data
+        format = reader.class.format.to_sym
+        collections = []
+        children = []
+        delete = nil
+        data = RDF::Writer.for(format).buffer do |writer|
+          reader.each_statement do |statement|
+            collections << statement.object.to_s if parent_field(parser).present? && parent_field(parser) == statement.predicate.to_s
+            children << statement.object.to_s if related_children_parsed_mapping.present? && related_children_parsed_mapping == statement.predicate.to_s
+            delete = statement.object.to_s if /deleted/.match?(statement.predicate.to_s)
+            writer << statement
+          end
         end
+        return {
+          source_id => reader.subjects.first.to_s,
+          delete: delete,
+          format: format,
+          data: data,
+          collection: collections,
+          children: children
+        }
       end
-      return {
-        source_id => reader.subjects.first.to_s,
-        delete: delete,
-        format: format,
-        data: data,
-        collection: collections,
-        children: children
-      }
-    end
 
-    def self.related_children_parsed_mapping
-      return @related_children_parsed_mapping if @related_children_parsed_mapping.present?
+      def self.related_children_parsed_mapping
+        return @related_children_parsed_mapping if @related_children_parsed_mapping.present?
 
-      rdf_related_children_field_mapping = Bulkrax.field_mappings['Bulkrax::RdfParser']&.select { |_, h| h.key?('related_children_field_mapping') }
-      return if rdf_related_children_field_mapping.blank?
+        rdf_related_children_field_mapping = Bulkrax.field_mappings['Bulkrax::RdfParser']&.select { |_, h| h.key?('related_children_field_mapping') }
+        return if rdf_related_children_field_mapping.blank?
 
-      @related_children_parsed_mapping = rdf_related_children_field_mapping&.keys&.first
-    end
+        @related_children_parsed_mapping = rdf_related_children_field_mapping&.keys&.first
+      end
 
-    def record
-      @record ||= RDF::Reader.for(self.raw_metadata['format'].to_sym).new(self.raw_metadata['data'])
-    end
+      def record
+        @record ||= RDF::Reader.for(self.raw_metadata['format'].to_sym).new(self.raw_metadata['data'])
+      end
 
-    def build_metadata
-      raise StandardError, 'Record not found' if record.nil?
-      raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
+      def build_metadata
+        raise StandardError, 'Record not found' if record.nil?
+        raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
 
-      self.parsed_metadata = {}
-      self.parsed_metadata[work_identifier] = [self.raw_metadata[source_identifier]]
+        self.parsed_metadata = {}
+        self.parsed_metadata[work_identifier] = [self.raw_metadata[source_identifier]]
 
-      record.each_statement do |statement|
-        # Only process the subject for our record (in case other data is in the file)
-        next unless statement.subject.to_s == self.raw_metadata[source_identifier]
-        add_metadata(statement.predicate.to_s, statement.object.to_s)
-      end
-      add_visibility
-      add_rights_statement
-      add_admin_set_id
-      add_collections
-      add_local
-      self.parsed_metadata['file'] = self.raw_metadata['file']
+        record.each_statement do |statement|
+          # Only process the subject for our record (in case other data is in the file)
+          next unless statement.subject.to_s == self.raw_metadata[source_identifier]
+          add_metadata(statement.predicate.to_s, statement.object.to_s)
+        end
+        add_visibility
+        add_rights_statement
+        add_admin_set_id
+        add_collections
+        add_local
+        self.parsed_metadata['file'] = self.raw_metadata['file']
 
-      self.parsed_metadata
-    end
+        self.parsed_metadata
+      end
 
-    def collections_created?
-      return true if self.raw_metadata['collection'].blank?
-      self.raw_metadata['collection'].length == self.collection_ids.length
-    end
+      def collections_created?
+        return true if self.raw_metadata['collection'].blank?
+        self.raw_metadata['collection'].length == self.collection_ids.length
+      end
 
-    def find_collection_ids
-      return self.collection_ids if collections_created?
-      if self.raw_metadata['collection'].present?
-        self.raw_metadata['collection'].each do |collection|
-          c = find_collection(collection)
-          self.collection_ids << c.id unless c.blank? || self.collection_ids.include?(c.id)
+      def find_collection_ids
+        return self.collection_ids if collections_created?
+        if self.raw_metadata['collection'].present?
+          self.raw_metadata['collection'].each do |collection|
+            c = find_collection(collection)
+            self.collection_ids << c.id unless c.blank? || self.collection_ids.include?(c.id)
+          end
         end
+        return self.collection_ids
       end
-      return self.collection_ids
     end
   end
 end

data/app/models/bulkrax/xml_entry.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'nokogiri'
 module Bulkrax
   # Generic XML Entry
   class XmlEntry < Entry

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -18,9 +18,9 @@ module Bulkrax
       begin
         # the identifier parameter can be a :source_identifier or the id of an object
         record = Entry.find_by(default_scope.merge({ importerexporter_id: importer_id })) || Entry.find_by(default_scope)
-        record ||= ActiveFedora::Base.find(identifier)
+        record ||= Bulkrax.object_factory.find(identifier)
       # NameError for if ActiveFedora isn't installed
-      rescue NameError, ActiveFedora::ObjectNotFoundError
+      rescue NameError, ActiveFedora::ObjectNotFoundError, Bulkrax::ObjectFactoryInterface::ObjectNotFoundError
         record = nil
       end
 
@@ -28,22 +28,5 @@ module Bulkrax
       # also accounts for when the found entry isn't a part of this importer
       record.is_a?(Entry) ? [record, record.factory.find] : [nil, record]
     end
-
-    # Check if the record is a Work
-    def curation_concern?(record)
-      available_work_types.include?(record.class)
-    end
-
-    private
-
-    # @return [Array<Class>] list of work type classes
-    def available_work_types
-      # If running in a Hyku app, do not include disabled work types
-      @available_work_types ||= if defined?(::Hyku)
-                                  ::Site.instance.available_works.map(&:constantize)
-                                else
-                                  Bulkrax.curation_concerns
-                                end
-    end
   end
 end

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'marcel'
 
 module Bulkrax
   module ExportBehavior
@@ -22,11 +21,12 @@ module Bulkrax
     end
 
     def hyrax_record
-      @hyrax_record ||= ActiveFedora::Base.find(self.identifier)
+      @hyrax_record ||= Bulkrax.object_factory.find(self.identifier)
     end
 
     # Prepend the file_set id to ensure a unique filename and also one that is not longer than 255 characters
     def filename(file_set)
+      # NOTE: Will this work with Valkyrie?
       return if file_set.original_file.blank?
       fn = file_set.original_file.file_name.first
       mime = ::Marcel::MimeType.for(file_set.original_file.mime_type)