bulkrax 6.0.1 → 8.0.0

data/app/parsers/bulkrax/csv_parser.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'csv'
 module Bulkrax
   class CsvParser < ApplicationParser # rubocop:disable Metrics/ClassLength
     include ErroredEntries
@@ -23,6 +22,7 @@ module Bulkrax
       @records = csv_data.map { |record_data| entry_class.data_for_entry(record_data, nil, self) }
     end
 
+    # rubocop:disable Metrics/AbcSize
     def build_records
       @collections = []
       @works = []
@@ -34,7 +34,9 @@ module Bulkrax
             next unless r.key?(model_mapping)
 
             model = r[model_mapping].nil? ? "" : r[model_mapping].strip
-            if model.casecmp('collection').zero?
+            # TODO: Eventually this should be refactored to us Hyrax.config.collection_model
+            #       We aren't right now because so many Bulkrax users are in between Fedora and Valkyrie
+            if model.casecmp('collection').zero? || model.casecmp('collectionresource').zero?
               @collections << r
             elsif model.casecmp('fileset').zero?
               @file_sets << r
@@ -52,6 +54,7 @@ module Bulkrax
 
       true
     end
+    # rubocop:enabled Metrics/AbcSize
 
     def collections
       build_records if @collections.nil?
@@ -113,57 +116,6 @@ module Bulkrax
       false
     end
 
-    def create_collections
-      create_objects(['collection'])
-    end
-
-    def create_works
-      create_objects(['work'])
-    end
-
-    def create_file_sets
-      create_objects(['file_set'])
-    end
-
-    def create_relationships
-      create_objects(['relationship'])
-    end
-
-    def create_objects(types_array = nil)
-      index = 0
-      (types_array || %w[collection work file_set relationship]).each do |type|
-        if type.eql?('relationship')
-          ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
-          next
-        end
-        send(type.pluralize).each do |current_record|
-          next unless record_has_source_identifier(current_record, index)
-          break if limit_reached?(limit, index)
-
-          seen[current_record[source_identifier]] = true
-          create_entry_and_job(current_record, type)
-          increment_counters(index, "#{type}": true)
-          index += 1
-        end
-        importer.record_status
-      end
-      true
-    rescue StandardError => e
-      set_status_info(e)
-    end
-
-    def create_entry_and_job(current_record, type)
-      new_entry = find_or_create_entry(send("#{type}_entry_class"),
-                                       current_record[source_identifier],
-                                       'Bulkrax::Importer',
-                                       current_record.to_h)
-      if current_record[:delete].present?
-        "Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
-      else
-        "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
-      end
-    end
-
     def write_partial_import_file(file)
       import_filename = import_file_path.split('/').last
       partial_import_filename = "#{File.basename(import_filename, '.csv')}_corrected_entries.csv"
@@ -204,7 +156,6 @@ module Bulkrax
     def entry_class
       CsvEntry
     end
-    alias work_entry_class entry_class
 
     def collection_entry_class
       CsvCollectionEntry
@@ -242,9 +193,10 @@ module Bulkrax
     # @todo - investigate getting directory structure
     # @todo - investigate using perform_later, and having the importer check for
     #   DownloadCloudFileJob before it starts
-    def retrieve_cloud_files(files)
+    def retrieve_cloud_files(files, importer)
       files_path = File.join(path_for_import, 'files')
       FileUtils.mkdir_p(files_path) unless File.exist?(files_path)
+      target_files = []
       files.each_pair do |_key, file|
         # fixes bug where auth headers do not get attached properly
         if file['auth_header'].present?
@@ -253,10 +205,12 @@ module Bulkrax
         end
         # this only works for uniquely named files
         target_file = File.join(files_path, file['file_name'].tr(' ', '_'))
+        target_files << target_file
         # Now because we want the files in place before the importer runs
         # Problematic for a large upload
-        Bulkrax::DownloadCloudFileJob.perform_now(file, target_file)
+        Bulkrax::DownloadCloudFileJob.perform_later(file, target_file)
       end
+      importer[:parser_fields]['original_file_paths'] = target_files
       return nil
     end
 
@@ -277,6 +231,7 @@ module Bulkrax
         CSV.open(setup_export_file(folder_count), "w", headers: export_headers, write_headers: true) do |csv|
           group.each do |entry|
             csv << entry.parsed_metadata
+            # TODO: This is precarious when we have descendents of Bulkrax::CsvCollectionEntry
             next if importerexporter.metadata_only? || entry.type == 'Bulkrax::CsvCollectionEntry'
 
             store_files(entry.identifier, folder_count.to_s)
@@ -286,7 +241,7 @@ module Bulkrax
     end
 
     def store_files(identifier, folder_count)
-      record = ActiveFedora::Base.find(identifier)
+      record = Bulkrax.object_factory.find(identifier)
       return unless record
 
       file_sets = record.file_set? ? Array.wrap(record) : record.file_sets
@@ -338,6 +293,9 @@ module Bulkrax
 
     def sort_entries(entries)
       # always export models in the same order: work, collection, file set
+      #
+      # TODO: This is a problem in that only these classes are compared.  Instead
+      #       We should add a comparison operator to the classes.
       entries.sort_by do |entry|
         case entry.type
         when 'Bulkrax::CsvCollectionEntry'

data/app/parsers/bulkrax/oai_dc_parser.rb

@@ -63,6 +63,12 @@ module Bulkrax
 
     delegate :list_sets, to: :client
 
+    def create_objects(types = [])
+      types.each do |object_type|
+        send("create_#{object_type.pluralize}")
+      end
+    end
+
     def create_collections
       metadata = {
         visibility: 'open'
@@ -86,27 +92,49 @@ module Bulkrax
       results = self.records(quick: true)
       return if results.blank?
       results.full.each_with_index do |record, index|
-        identifier = record.send(source_identifier)
-        if identifier.blank?
-          if Bulkrax.fill_in_blank_source_identifiers.present?
-            identifier = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
-          else
-            invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
-            next
-          end
-        end
-
+        identifier = record_has_source_identifier(record, index)
+        next unless identifier
         break if limit_reached?(limit, index)
+
         seen[identifier] = true
-        new_entry = entry_class.where(importerexporter: self.importerexporter, identifier: identifier).first_or_create!
-        if record.deleted?
-          DeleteWorkJob.send(perform_method, new_entry, importerexporter.current_run)
-        else
-          ImportWorkJob.send(perform_method, new_entry.id, importerexporter.current_run.id)
-        end
+        create_entry_and_job(record, 'work', identifier)
         increment_counters(index, work: true)
       end
       importer.record_status
+    rescue StandardError => e
+      set_status_info(e)
+    end
+
+    def create_file_sets; end
+
+    def create_relationships
+      ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
+    end
+
+    def record_raw_metadata(_record)
+      nil
+    end
+
+    def record_deleted?(_record)
+      false
+    end
+
+    def record_remove_and_rerun?(_record)
+      false
+    end
+
+    # oai records so not let us set the source identifier easily
+    def record_has_source_identifier(record, index)
+      identifier = record.send(source_identifier)
+      if identifier.blank?
+        if Bulkrax.fill_in_blank_source_identifiers.present?
+          identifier = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
+        else
+          invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
+          return false
+        end
+      end
+      identifier
     end
 
     def collections

data/app/parsers/bulkrax/parser_export_record_set.rb

@@ -113,14 +113,14 @@ module Bulkrax
       #
       # @see #file_sets
       def candidate_file_set_ids
-        @candidate_file_set_ids ||= works.flat_map { |work| work.fetch("#{Bulkrax.file_model_class.to_s.underscore}_ids_ssim", []) }
+        @candidate_file_set_ids ||= works.flat_map { |work| work.fetch(Bulkrax.solr_key_for_member_file_ids, []) }
       end
 
       # @note Specifically not memoizing this so we can merge values without changing the object.
       #
       # No sense attempting to query for more than the limit.
       def query_kwargs
-        { fl: "id,#{Bulkrax.file_model_class.to_s.underscore}_ids_ssim", method: :post, rows: row_limit }
+        { fl: "id,#{Bulkrax.solr_key_for_member_file_ids}", method: :post, rows: row_limit }
       end
 
       # If we have a limit, we need not query beyond that limit
@@ -149,12 +149,12 @@ module Bulkrax
       end
 
       def works
-        @works ||= ActiveFedora::SolrService.query(works_query, **works_query_kwargs)
+        @works ||= Bulkrax.object_factory.query(works_query, **works_query_kwargs)
       end
 
       def collections
         @collections ||= if collections_query
-                           ActiveFedora::SolrService.query(collections_query, **collections_query_kwargs)
+                           Bulkrax.object_factory.query(collections_query, **collections_query_kwargs)
                          else
                            []
                          end
@@ -173,43 +173,39 @@ module Bulkrax
       # @see https://github.com/samvera/hyrax/blob/64c0bbf0dc0d3e1b49f040b50ea70d177cc9d8f6/app/indexers/hyrax/work_indexer.rb#L15-L18
       def file_sets
         @file_sets ||= ParserExportRecordSet.in_batches(candidate_file_set_ids) do |batch_of_ids|
-          fsq = "has_model_ssim:#{Bulkrax.file_model_class} AND id:(\"" + batch_of_ids.join('" OR "') + "\")"
+          fsq = "has_model_ssim:#{Bulkrax.file_model_internal_resource} AND id:(\"" + batch_of_ids.join('" OR "') + "\")"
           fsq += extra_filters if extra_filters.present?
-          ActiveFedora::SolrService.query(
+          Bulkrax.object_factory.query(
             fsq,
-            { fl: "id", method: :post, rows: batch_of_ids.size }
+            fl: "id", method: :post, rows: batch_of_ids.size
           )
         end
       end
 
       def solr_name(base_name)
-        if Module.const_defined?(:Solrizer)
-          ::Solrizer.solr_name(base_name)
-        else
-          ::ActiveFedora.index_field_mapper.solr_name(base_name)
-        end
+        Bulkrax.object_factory.solr_name(base_name)
       end
     end
 
     class All < Base
       def works_query
-        "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')}) #{extra_filters}"
+        "has_model_ssim:(#{Bulkrax.curation_concern_internal_resources.join(' OR ')}) #{extra_filters}"
       end
 
       def collections_query
-        "has_model_ssim:Collection #{extra_filters}"
+        "has_model_ssim:#{Bulkrax.collection_model_internal_resource} #{extra_filters}"
       end
     end
 
     class Collection < Base
       def works_query
         "member_of_collection_ids_ssim:#{importerexporter.export_source} #{extra_filters} AND " \
-        "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')})"
+        "has_model_ssim:(#{Bulkrax.curation_concern_internal_resources.join(' OR ')})"
       end
 
       def collections_query
         "(id:#{importerexporter.export_source} #{extra_filters}) OR " \
-        "(has_model_ssim:Collection AND member_of_collection_ids_ssim:#{importerexporter.export_source})"
+        "(has_model_ssim:#{Bulkrax.collection_model_internal_resource} AND member_of_collection_ids_ssim:#{importerexporter.export_source})"
       end
     end
 
@@ -247,12 +243,12 @@ module Bulkrax
 
       def works
         @works ||= ParserExportRecordSet.in_batches(complete_entry_identifiers) do |ids|
-          ActiveFedora::SolrService.query(
+          Bulkrax.object_factory.query(
             extra_filters.to_s,
             **query_kwargs.merge(
               fq: [
                 %(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
-                "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')})"
+                "has_model_ssim:(#{Bulkrax.curation_concern_internal_resources.join(' OR ')})"
               ],
               fl: 'id'
             )
@@ -262,12 +258,12 @@ module Bulkrax
 
       def collections
         @collections ||= ParserExportRecordSet.in_batches(complete_entry_identifiers) do |ids|
-          ActiveFedora::SolrService.query(
-            "has_model_ssim:Collection #{extra_filters}",
+          Bulkrax.object_factory.query(
+            "has_model_ssim:#{Bulkrax.collection_model_internal_resource} #{extra_filters}",
             **query_kwargs.merge(
               fq: [
                 %(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
-                "has_model_ssim:Collection"
+                "has_model_ssim:#{Bulkrax.collection_model_internal_resource}"
               ],
               fl: "id"
             )
@@ -281,12 +277,12 @@ module Bulkrax
       # @see Bulkrax::ParserExportRecordSet::Base#file_sets
       def file_sets
         @file_sets ||= ParserExportRecordSet.in_batches(complete_entry_identifiers) do |ids|
-          ActiveFedora::SolrService.query(
+          Bulkrax.object_factory.query(
             extra_filters,
-            query_kwargs.merge(
+            **query_kwargs.merge(
               fq: [
                 %(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
-                "has_model_ssim:#{Bulkrax.file_model_class}"
+                "has_model_ssim:#{Bulkrax.file_model_internal_resource}"
               ],
               fl: 'id'
             )

data/app/parsers/bulkrax/xml_parser.rb

@@ -1,6 +1,4 @@
 # frozen_string_literal: true
-require 'marcel'
-
 module Bulkrax
   class XmlParser < ApplicationParser
     def entry_class
@@ -11,13 +9,29 @@ module Bulkrax
     def collection_entry_class; end
 
     # @todo not yet supported
-    def create_collections; end
+    def create_collections
+      raise NotImplementedError
+    end
 
     # @todo not yet supported
     def file_set_entry_class; end
 
     # @todo not yet supported
-    def create_file_sets; end
+    def create_file_sets
+      raise NotImplementedError
+    end
+
+    def file_sets
+      raise NotImplementedError
+    end
+
+    def collections
+      raise NotImplementedError
+    end
+
+    def works
+      records
+    end
 
     # TODO: change to differentiate between collection and work records when adding ability to import collection metadata
     def works_total
@@ -92,25 +106,6 @@ module Bulkrax
       %w[.xml .xls .xsd].include?(File.extname(path)) || ::Marcel::MimeType.for(path).include?('application/xml')
     end
 
-    def create_works
-      records.each_with_index do |record, index|
-        next unless record_has_source_identifier(record, index)
-        break if !limit.nil? && index >= limit
-
-        seen[record[source_identifier]] = true
-        new_entry = find_or_create_entry(entry_class, record[source_identifier], 'Bulkrax::Importer', record)
-        if record[:delete].present?
-          DeleteWorkJob.send(perform_method, new_entry, current_run)
-        else
-          ImportWorkJob.send(perform_method, new_entry.id, current_run.id)
-        end
-        increment_counters(index, work: true)
-      end
-      importer.record_status
-    rescue StandardError => e
-      set_status_info(e)
-    end
-
     def total
       records.size
     end

data/app/services/bulkrax/factory_class_finder.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class FactoryClassFinder
+    ##
+    # The v6.0.0 default coercer.  Responsible for converting a factory class name to a constant.
+    module DefaultCoercer
+      ##
+      # @param name [String]
+      # @return [Class] when the name is a coercible constant.
+      # @raise [NameError] when the name is not coercible to a constant.
+      def self.call(name)
+        name.constantize
+      end
+    end
+
+    ##
+    # A name coercer that favors classes that end with "Resource" but will attempt to fallback to
+    # those that don't.
+    module ValkyrieMigrationCoercer
+      SUFFIX = "Resource"
+
+      ##
+      # @param name [String]
+      # @param suffix [String] the suffix we use for a naming convention.
+      #
+      # @return [Class] when the name is a coercible constant.
+      # @raise [NameError] when the name is not coercible to a constant.
+      def self.call(name, suffix: SUFFIX)
+        if name.end_with?(suffix)
+          name.constantize
+        elsif name == "FileSet"
+          Bulkrax.file_model_class
+        else
+          begin
+            "#{name}#{suffix}".constantize
+          rescue NameError
+            name.constantize
+          end
+        end
+      end
+    end
+
+    ##
+    # @param entry [Bulkrax::Entry]
+    # @return [Class]
+    def self.find(entry:, coercer: Bulkrax.factory_class_name_coercer || DefaultCoercer)
+      new(entry: entry, coercer: coercer).find
+    end
+
+    def initialize(entry:, coercer:)
+      @entry = entry
+      @coercer = coercer
+    end
+    attr_reader :entry, :coercer
+
+    ##
+    # @return [Class] when we are able to derive the class based on the {#name}.
+    # @return [Nil] when we encounter errors with constantizing the {#name}.
+    # @see #name
+    def find
+      coercer.call(name)
+    rescue NameError
+      nil
+    rescue
+      entry.default_work_type.constantize
+    end
+
+    ##
+    # @api private
+    # @return [String]
+    def name
+      fc = if entry.parsed_metadata&.[]('model').present?
+             Array.wrap(entry.parsed_metadata['model']).first
+           elsif entry.importerexporter&.mapping&.[]('work_type').present?
+             # Because of delegation's nil guard, we're reaching rather far into the implementation
+             # details.
+             Array.wrap(entry.parsed_metadata['work_type']).first
+           else
+             entry.default_work_type
+           end
+
+      # Let's coerce this into the right shape; we're not mutating the string because it might well
+      # be frozen.
+      fc = fc.tr(' ', '_')
+      fc = fc.downcase if fc.match?(/[-_]/)
+      fc.camelcase
+    rescue
+      entry.default_work_type
+    end
+  end
+end

data/app/services/bulkrax/remove_relationships_for_importer.rb

@@ -57,7 +57,7 @@ module Bulkrax
         obj = entry.factory.find
         next if obj.is_a?(Bulkrax.file_model_class) # FileSets must be attached to a Work
 
-        if obj.is_a?(Collection)
+        if obj.is_a?(Bulkrax.collection_model_class)
           remove_relationships_from_collection(obj)
         else
           remove_relationships_from_work(obj)
@@ -78,12 +78,14 @@ module Bulkrax
 
       return if defined?(Hyrax)
 
+      # NOTE: This should not need to be migrated to the object factory.
       # Remove parent collection relationships
       collection.member_of_collections.each do |parent_col|
         Hyrax::Collections::NestedCollectionPersistenceService
           .remove_nested_relationship_for(parent: parent_col, child: collection)
       end
 
+      # NOTE: This should not need to be migrated to the object factory.
       # Remove child collection relationships
       collection.member_collections.each do |child_col|
         Hyrax::Collections::NestedCollectionPersistenceService

data/app/services/hyrax/custom_queries/find_by_source_identifier.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Hyrax
+  module CustomQueries
+    ##
+    # @see https://github.com/samvera/valkyrie/wiki/Queries#custom-queries
+    class FindBySourceIdentifier
+      def self.queries
+        [:find_by_model_and_property_value]
+      end
+
+      def initialize(query_service:)
+        @query_service = query_service
+      end
+
+      attr_reader :query_service
+      delegate :resource_factory, to: :query_service
+      delegate :orm_class, to: :resource_factory
+
+      ##
+      # @param model [Class, #internal_resource]
+      # @param property [#to_s] the name of the property we're attempting to
+      #        query.
+      # @param value [#to_s] the propety's value that we're trying to match.
+      #
+      # @return [NilClass] when no record was found
+      # @return [Valkyrie::Resource] when a record was found
+      #
+      # @note This is not a real estate transaction nor a Zillow lookup.
+      def find_by_model_and_property_value(model:, property:, value:)
+        sql_query = sql_for_find_by_model_and_property_value
+        # NOTE: Do we need to ask the model for it's internal_resource?
+        # TODO: no => undefined method `internal_resource' for Image:Class
+        query_service.run_query(sql_query, model, property, value).first
+      end
+
+      private
+
+      def sql_for_find_by_model_and_property_value
+        # NOTE: This is querying the first element of the property, but we might
+        # want to check all of the elements.
+        <<-SQL
+          SELECT * FROM orm_resources
+          WHERE internal_resource = ? AND metadata -> ? ->> 0 = ?
+          LIMIT 1;
+        SQL
+      end
+    end
+  end
+end

data/app/services/wings/custom_queries/find_by_source_identifier.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Wings
+  module CustomQueries
+    class FindBySourceIdentifier
+      # Custom query override specific to Wings
+
+      def self.queries
+        [:find_by_model_and_property_value]
+      end
+
+      attr_reader :query_service
+      delegate :resource_factory, to: :query_service
+
+      def initialize(query_service:)
+        @query_service = query_service
+      end
+
+      def find_by_model_and_property_value(model:, property:, value:, use_valkyrie: Hyrax.config.use_valkyrie?)
+        # NOTE: This is using the Bulkrax::ObjectFactory (e.g. the one
+        # envisioned for ActiveFedora).  In doing this, we avoid the situation
+        # where Bulkrax::ValkyrieObjectFactory calls this custom query.
+        af_object = Bulkrax::ObjectFactory.search_by_property(value: value, klass: model, field: property)
+
+        return if af_object.blank?
+        return af_object unless use_valkyrie
+
+        resource_factory.to_resource(object: af_object)
+      end
+    end
+  end
+end

data/app/views/bulkrax/entries/_parsed_metadata.html.erb

@@ -5,7 +5,7 @@
         Parsed Metadata:
       </a>
       <a role="button" data-toggle="collapse" data-target="#parsed-metadata-show" aria-expanded="true" aria-controls="parsed-metadata-show">
-        <div class="accordion-icon glyphicon glyphicon-remove-circle" aria-hidden="true"></div>
+        <div class="accordion-icon fa fa-times-circle" aria-hidden="true"></div>
       </a>
     </div>
     <div id="parsed-metadata-show" class="accordion-collapse collapse" role="tabpanel" aria-labelledby="parsed-metadata-show">
@@ -16,4 +16,4 @@
       </div>
     </div>
   </div>
-<% end %>
+<% end %>

data/app/views/bulkrax/entries/_raw_metadata.html.erb

@@ -5,7 +5,7 @@
         Raw Metadata:
       </a>
       <a role="button" data-toggle="collapse" data-target="#raw-metadata-show" aria-expanded="true" aria-controls="raw-metadata-show">
-        <div class="accordion-icon glyphicon glyphicon-remove-circle" aria-hidden="true"></div>
+        <div class="accordion-icon fa fa-times-circle" aria-hidden="true"></div>
       </a>
     </div>
     <div id="raw-metadata-show" class="accordion-collapse collapse" role="tabpanel" aria-labelledby="raw-metadata-show">
@@ -16,4 +16,4 @@
       </div>
     </div>
   </div>
-<% end %>
+<% end %>