bulkrax 5.0.0 → 5.2.0

data/app/parsers/bulkrax/csv_parser.rb

@@ -84,23 +84,31 @@ module Bulkrax
       @import_fields ||= records.inject(:merge).keys.compact.uniq
     end
 
-    def required_elements?(keys)
-      return if keys.blank?
-      missing_elements(keys).blank?
-    end
-
-    def missing_elements(keys)
-      required_elements.map(&:to_s) - keys.map(&:to_s)
+    def required_elements?(record)
+      missing_elements(record).blank?
+    end
+
+    def missing_elements(record)
+      keys_from_record = keys_without_numbers(record.reject { |_, v| v.blank? }.keys.compact.uniq.map(&:to_s))
+      keys = []
+      # Because we're persisting the mapping in the database, these are likely string keys.
+      # However, there's no guarantee.  So, we need to ensure that by running stringify.
+      importerexporter.mapping.stringify_keys.map do |k, v|
+        Array.wrap(v['from']).each do |vf|
+          keys << k if keys_from_record.include?(vf)
+        end
+      end
+      required_elements.map(&:to_s) - keys.uniq.map(&:to_s)
     end
 
     def valid_import?
-      import_strings = keys_without_numbers(import_fields.map(&:to_s))
-      error_alert = "Missing at least one required element, missing element(s) are: #{missing_elements(import_strings).join(', ')}"
-      raise StandardError, error_alert unless required_elements?(import_strings)
+      compressed_record = records.flat_map(&:to_a).partition { |_, v| !v }.flatten(1).to_h
+      error_alert = "Missing at least one required element, missing element(s) are: #{missing_elements(compressed_record).join(', ')}"
+      raise StandardError, error_alert unless required_elements?(compressed_record)
 
       file_paths.is_a?(Array)
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
       false
     end
 
@@ -140,7 +148,7 @@ module Bulkrax
       end
       true
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
     end
 
     def create_entry_and_job(current_record, type)
@@ -167,102 +175,17 @@ module Bulkrax
       path
     end
 
-    def extra_filters
-      output = ""
-      if importerexporter.start_date.present?
-        start_dt = importerexporter.start_date.to_datetime.strftime('%FT%TZ')
-        finish_dt = importerexporter.finish_date.present? ? importerexporter.finish_date.to_datetime.end_of_day.strftime('%FT%TZ') : "NOW"
-        output += " AND system_modified_dtsi:[#{start_dt} TO #{finish_dt}]"
-      end
-      output += importerexporter.work_visibility.present? ? " AND visibility_ssi:#{importerexporter.work_visibility}" : ""
-      output += importerexporter.workflow_status.present? ? " AND workflow_state_name_ssim:#{importerexporter.workflow_status}" : ""
-      output
-    end
-
-    def current_work_ids
-      ActiveSupport::Deprecation.warn('Bulkrax::CsvParser#current_work_ids will be replaced with #current_record_ids in version 3.0')
-      current_record_ids
-    end
-
-    # rubocop:disable Metrics/AbcSize
-    def current_record_ids
-      @work_ids = []
-      @collection_ids = []
-      @file_set_ids = []
-
-      case importerexporter.export_from
-      when 'all'
-        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
-        @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
-        @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
-      when 'collection'
-        @work_ids = ActiveFedora::SolrService.query(
-"member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters} AND has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')})", method: :post, rows: 2_000_000_000
-).map(&:id)
-        # get the parent collection and child collections
-        @collection_ids = ActiveFedora::SolrService.query("id:#{importerexporter.export_source} #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
-        @collection_ids += ActiveFedora::SolrService.query("has_model_ssim:Collection AND member_of_collection_ids_ssim:#{importerexporter.export_source}", method: :post,
-                                                                                                                                                            rows: 2_147_483_647).map(&:id)
-        find_child_file_sets(@work_ids)
-      when 'worktype'
-        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
-        find_child_file_sets(@work_ids)
-      when 'importer'
-        set_ids_for_exporting_from_importer
-      end
-
-      @work_ids + @collection_ids + @file_set_ids
-    end
-    # rubocop:enable Metrics/AbcSize
-
-    # find the related file set ids so entries can be made for export
-    def find_child_file_sets(work_ids)
-      work_ids.each do |id|
-        ActiveFedora::Base.find(id).file_set_ids.each { |fs_id| @file_set_ids << fs_id }
-      end
-    end
-
-    # Set the following instance variables: @work_ids, @collection_ids, @file_set_ids
-    # @see #current_record_ids
-    def set_ids_for_exporting_from_importer
-      entry_ids = Importer.find(importerexporter.export_source).entries.pluck(:id)
-      complete_statuses = Status.latest_by_statusable
-                                .includes(:statusable)
-                                .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
-
-      complete_entry_identifiers = complete_statuses.map { |s| s.statusable&.identifier&.gsub(':', '\:') }
-      extra_filters = extra_filters.presence || '*:*'
-
-      { :@work_ids => ::Hyrax.config.curation_concerns, :@collection_ids => [::Collection], :@file_set_ids => [::FileSet] }.each do |instance_var, models_to_search|
-        instance_variable_set(instance_var, ActiveFedora::SolrService.post(
-          extra_filters.to_s,
-          fq: [
-            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
-            "has_model_ssim:(#{models_to_search.join(' OR ')})"
-          ],
-          fl: 'id',
-          rows: 2_000_000_000
-        )['response']['docs'].map { |obj| obj['id'] })
-      end
-    end
-
-    def solr_name(base_name)
-      Module.const_defined?(:Solrizer) ? ::Solrizer.solr_name(base_name) : ::ActiveFedora.index_field_mapper.solr_name(base_name)
+    def current_records_for_export
+      @current_records_for_export ||= Bulkrax::ParserExportRecordSet.for(
+        parser: self,
+        export_from: importerexporter.export_from
+      )
     end
 
     def create_new_entries
-      current_record_ids.each_with_index do |id, index|
-        break if limit_reached?(limit, index)
-
-        this_entry_class = if @collection_ids.include?(id)
-                             collection_entry_class
-                           elsif @file_set_ids.include?(id)
-                             file_set_entry_class
-                           else
-                             entry_class
-                           end
-        new_entry = find_or_create_entry(this_entry_class, id, 'Bulkrax::Exporter')
-
+      # NOTE: The each method enforces the limit, as it can best optimize the underlying queries.
+      current_records_for_export.each do |id, entry_class|
+        new_entry = find_or_create_entry(entry_class, id, 'Bulkrax::Exporter')
         begin
           entry = ExportWorkJob.perform_now(new_entry.id, current_run.id)
         rescue => e
@@ -291,16 +214,22 @@ module Bulkrax
     end
 
     def valid_entry_types
-      ['Bulkrax::CsvCollectionEntry', 'Bulkrax::CsvFileSetEntry', 'Bulkrax::CsvEntry']
+      [collection_entry_class.to_s, file_set_entry_class.to_s, entry_class.to_s]
     end
 
     # TODO: figure out why using the version of this method that's in the bagit parser
     # breaks specs for the "if importer?" line
     def total
-      @total = importer.parser_fields['total'] || 0 if importer?
-      @total = limit || current_record_ids.count if exporter?
+      @total =
+        if importer?
+          importer.parser_fields['total'] || 0
+        elsif exporter?
+          limit.to_i.zero? ? current_records_for_export.count : limit.to_i
+        else
+          0
+        end
 
-      return @total || 0
+      return @total
     rescue StandardError
       @total = 0
     end
@@ -335,10 +264,13 @@ module Bulkrax
     def write_files
       require 'open-uri'
       folder_count = 0
+      # TODO: This is not performant as well; unclear how to address, but lower priority as of
+      #       <2023-02-21 Tue>.
       sorted_entries = sort_entries(importerexporter.entries.uniq(&:identifier))
                        .select { |e| valid_entry_types.include?(e.type) }
 
-      sorted_entries[0..limit || total].in_groups_of(records_split_count, false) do |group|
+      group_size = limit.to_i.zero? ? total : limit.to_i
+      sorted_entries[0..group_size].in_groups_of(records_split_count, false) do |group|
         folder_count += 1
 
         CSV.open(setup_export_file(folder_count), "w", headers: export_headers, write_headers: true) do |csv|
@@ -398,7 +330,7 @@ module Bulkrax
       return @object_names if @object_names
 
       @object_names = mapping.values.map { |value| value['object'] }
-      @object_names.uniq!.delete(nil)
+      @object_names.uniq!&.delete(nil)
 
       @object_names
     end

data/app/parsers/bulkrax/oai_dc_parser.rb

@@ -63,9 +63,9 @@ module Bulkrax
 
     def create_collections
       metadata = {
-        visibility: 'open',
-        collection_type_gid: Hyrax::CollectionType.find_or_create_default_collection_type.gid
+        visibility: 'open'
       }
+      metadata[:collection_type_gid] = Hyrax::CollectionType.find_or_create_default_collection_type.gid if defined?(::Hyrax)
 
       collections.each_with_index do |set, index|
         next unless collection_name == 'all' || collection_name == set.spec

data/app/parsers/bulkrax/parser_export_record_set.rb

@@ -0,0 +1,281 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # This module is responsible for providing the means of querying Solr for the appropriate works,
+  # collections, and file sets for an export of entries.
+  #
+  # @see .for
+  module ParserExportRecordSet
+    # @api public
+    #
+    # A factory method for returning an object that can yield each id and associated entry_class as
+    # well as return the count of objects in the record set.
+    #
+    # @param parser [Bulkrax::ApplicationParser]
+    # @param export_from [String]
+    #
+    # @return [#each, #count] An object, likely a descendant of
+    #         {Bulkrax::CurrentParserRecordSet::Base} that responds to {Base#count} and
+    #         {Base#each}.
+    def self.for(parser:, export_from:)
+      "Bulkrax::ParserExportRecordSet::#{export_from.classify}".constantize.new(parser: parser)
+    end
+
+    # @abstract
+    #
+    # @note This has {#each} and {#count} but is not an Enumerable.  But because it has these two
+    #       methods that echo {Array}, we can do some lovely mocking and stubbing in those classes
+    #       dependent on this file.  :)
+    class Base
+      def initialize(parser:)
+        @parser = parser
+      end
+      attr_reader :parser
+      private :parser
+
+      delegate :limit_reached?, :work_entry_class, :collection_entry_class, :file_set_entry_class, :importerexporter, to: :parser
+      private :limit_reached?, :work_entry_class, :collection_entry_class, :file_set_entry_class, :importerexporter
+
+      # @return [Integer]
+      def count
+        sum = works.count + collections.count + file_sets.count
+        return sum if limit.zero?
+        return limit if sum > limit
+        return sum
+      end
+
+      # Yield first the works, then collections, then file sets.  Once we've yielded as many times
+      # as the parser's limit, we break the iteration and return.
+      #
+      # @yieldparam id [String] The ID of the work/collection/file_set
+      # @yieldparam entry_class [Class] The parser associated entry class for the
+      #             work/collection/file_set.
+      #
+      # @note The order of what we yield has been previously determined.
+      def each
+        counter = 0
+
+        works.each do |work|
+          break if limit_reached?(limit, counter)
+          yield(work.fetch('id'), work_entry_class)
+          counter += 1
+        end
+
+        return if limit_reached?(limit, counter)
+
+        collections.each do |collection|
+          break if limit_reached?(limit, counter)
+          yield(collection.fetch('id'), collection_entry_class)
+          counter += 1
+        end
+
+        return if limit_reached?(limit, counter)
+
+        file_sets.each do |file_set|
+          break if limit_reached?(limit, counter)
+          yield(file_set.fetch('id'), file_set_entry_class)
+          counter += 1
+        end
+      end
+
+      private
+
+      # Why call these candidates and not the actual file_set_ids?  Because of implementation
+      # details of Hyrax.  What are those details?  The upstream application (as of v2.9.x) puts
+      # child works into the `file_set_ids_ssim` field.  So we have a mix of file sets and works in
+      # that property.
+      #
+      # @see #file_sets
+      def candidate_file_set_ids
+        @candidate_file_set_ids ||= works.flat_map { |work| work.fetch("#{Bulkrax.file_model_class.to_s.underscore}_ids_ssim", []) }
+      end
+
+      # @note Specifically not memoizing this so we can merge values without changing the object.
+      #
+      # No sense attempting to query for more than the limit.
+      def query_kwargs
+        { fl: "id,#{Bulkrax.file_model_class.to_s.underscore}_ids_ssim", method: :post, rows: row_limit }
+      end
+
+      # If we have a limit, we need not query beyond that limit
+      def row_limit
+        return 2_147_483_647 if limit.zero?
+        limit
+      end
+
+      def limit
+        parser.limit.to_i
+      end
+
+      alias works_query_kwargs query_kwargs
+      alias collections_query_kwargs query_kwargs
+
+      def extra_filters
+        output = ""
+        if importerexporter.start_date.present?
+          start_dt = importerexporter.start_date.to_datetime.strftime('%FT%TZ')
+          finish_dt = importerexporter.finish_date.present? ? importerexporter.finish_date.to_datetime.end_of_day.strftime('%FT%TZ') : "NOW"
+          output += " AND system_modified_dtsi:[#{start_dt} TO #{finish_dt}]"
+        end
+        output += importerexporter.work_visibility.present? ? " AND visibility_ssi:#{importerexporter.work_visibility}" : ""
+        output += importerexporter.workflow_status.present? ? " AND workflow_state_name_ssim:#{importerexporter.workflow_status}" : ""
+        output
+      end
+
+      def works
+        @works ||= ActiveFedora::SolrService.query(works_query, **works_query_kwargs)
+      end
+
+      def collections
+        @collections ||= if collections_query
+                           ActiveFedora::SolrService.query(collections_query, **collections_query_kwargs)
+                         else
+                           []
+                         end
+      end
+
+      SOLR_QUERY_PAGE_SIZE = 512
+
+      # @note In most cases, when we don't have any candidate file sets, there is no need to query SOLR.
+      #
+      # @see Bulkrax::ParserExportRecordSet::Importer#file_sets
+      #
+      # Why can't we just use the candidate_file_set_ids?  Because Hyrax is pushing child works into the
+      # `file_set_ids_ssim` field.
+      #
+      # For v2.9.x of Hryax; perhaps this is resolved.
+      #
+      # @see https://github.com/scientist-softserv/britishlibrary/issues/289
+      # @see https://github.com/samvera/hyrax/blob/64c0bbf0dc0d3e1b49f040b50ea70d177cc9d8f6/app/indexers/hyrax/work_indexer.rb#L15-L18
+      def file_sets
+        @file_sets ||= if candidate_file_set_ids.empty?
+                         []
+                       else
+                         results = []
+                         candidate_file_set_ids.each_slice(SOLR_QUERY_PAGE_SIZE) do |ids|
+                           fsq = "has_model_ssim:#{Bulkrax.file_model_class} AND id:(\"" + ids.join('" OR "') + "\")"
+                           fsq += extra_filters if extra_filters.present?
+                           results += ActiveFedora::SolrService.query(
+                             fsq,
+                             { fl: "id", method: :post, rows: ids.size }
+                           )
+                         end
+                         results
+                       end
+      end
+
+      def solr_name(base_name)
+        if Module.const_defined?(:Solrizer)
+          ::Solrizer.solr_name(base_name)
+        else
+          ::ActiveFedora.index_field_mapper.solr_name(base_name)
+        end
+      end
+    end
+
+    class All < Base
+      def works_query
+        "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')}) #{extra_filters}"
+      end
+
+      def collections_query
+        "has_model_ssim:Collection #{extra_filters}"
+      end
+    end
+
+    class Collection < Base
+      def works_query
+        "member_of_collection_ids_ssim:#{importerexporter.export_source} #{extra_filters} AND " \
+        "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')})"
+      end
+
+      def collections_query
+        "(id:#{importerexporter.export_source} #{extra_filters}) OR " \
+        "(has_model_ssim:Collection AND member_of_collection_ids_ssim:#{importerexporter.export_source})"
+      end
+    end
+
+    class Worktype < Base
+      def works_query
+        "has_model_ssim:#{importerexporter.export_source} #{extra_filters}"
+      end
+
+      def collections_query
+        nil
+      end
+    end
+
+    class Importer < Base
+      private
+
+      delegate :work_identifier, to: :parser
+      private :work_identifier
+
+      def extra_filters
+        '*:*' + super
+      end
+
+      def complete_entry_identifiers
+        @complete_entry_identifiers ||=
+          begin
+            entry_ids ||= Bulkrax::Importer.find(importerexporter.export_source).entries.pluck(:id)
+            complete_statuses ||= Bulkrax::Status.latest_by_statusable
+                                                 .includes(:statusable)
+                                                 .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
+
+            complete_statuses.map { |s| s.statusable&.identifier&.gsub(':', '\:') }
+          end
+      end
+
+      def works_query_kwargs
+        query_kwargs.merge(
+          fq: [
+            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
+            "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')})"
+          ],
+          fl: 'id'
+        )
+      end
+
+      def works_query
+        extra_filters.to_s
+      end
+
+      def collections_query_kwargs
+        query_kwargs.merge(
+          fq: [
+            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
+            "has_model_ssim:Collection"
+          ],
+          fl: 'id'
+        )
+      end
+
+      def collections_query
+        "has_model_ssim:Collection #{extra_filters}"
+      end
+
+      # This is an exception; we don't know how many candidate file sets there might be.  So we will instead
+      # make the query (assuming that there are {#complete_entry_identifiers}).
+      #
+      # @see Bulkrax::ParserExportRecordSet::Base#file_sets
+      def file_sets
+        @file_sets ||= ActiveFedora::SolrService.query(file_sets_query, **file_sets_query_kwargs)
+      end
+
+      def file_sets_query_kwargs
+        query_kwargs.merge(
+          fq: [
+            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
+            "has_model_ssim:#{Bulkrax.file_model_class}"
+          ],
+          fl: 'id'
+        )
+      end
+
+      def file_sets_query
+        extra_filters
+      end
+    end
+  end
+end

data/app/parsers/bulkrax/xml_parser.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'marcel'
 
 module Bulkrax
   class XmlParser < ApplicationParser
@@ -31,7 +32,7 @@ module Bulkrax
       raise StandardError, 'No records found' if records.blank?
       true
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
       false
     end
 
@@ -78,16 +79,19 @@ module Bulkrax
     # Otherwise return all xml files in the given folder
     def metadata_paths
       @metadata_paths ||=
-        if file? && MIME::Types.type_for(import_file_path).include?('application/xml')
+        if file? && good_file_type?(import_file_path)
           [import_file_path]
         else
           file_paths.select do |f|
-            MIME::Types.type_for(f).include?('application/xml') &&
-              f.include?("import_#{importerexporter.id}")
+            good_file_type?(f) && f.include?("import_#{importerexporter.id}")
           end
         end
     end
 
+    def good_file_type?(path)
+      %w[.xml .xls .xsd].include?(File.extname(path)) || ::Marcel::MimeType.for(path).include?('application/xml')
+    end
+
     def create_works
       records.each_with_index do |record, index|
         next unless record_has_source_identifier(record, index)
@@ -104,7 +108,7 @@ module Bulkrax
       end
       importer.record_status
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
     end
 
     def total

data/app/services/bulkrax/remove_relationships_for_importer.rb

@@ -55,7 +55,7 @@ module Bulkrax
         progress_bar.increment
 
         obj = entry.factory.find
-        next if obj.is_a?(FileSet) # FileSets must be attached to a Work
+        next if obj.is_a?(Bulkrax.file_model_class) # FileSets must be attached to a Work
 
         if obj.is_a?(Collection)
           remove_relationships_from_collection(obj)
@@ -63,7 +63,7 @@ module Bulkrax
           remove_relationships_from_work(obj)
         end
 
-        obj.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
+        obj.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) if defined?(Hyrax)
         obj.save!
       end
     end
@@ -75,6 +75,8 @@ module Bulkrax
         work.save! if change.present?
       end
 
+      return if defined?(Hyrax)
+
       # Remove parent collection relationships
       collection.member_of_collections.each do |parent_col|
         Hyrax::Collections::NestedCollectionPersistenceService

data/app/views/bulkrax/entries/show.html.erb

@@ -48,7 +48,7 @@
         <% record = @entry&.hyrax_record %>
         <% if record.present? && @entry.factory_class %>
           <strong><%= record.class.to_s %> Link:</strong>
-          <% if record.is_a?(Collection) %>
+          <% if defined?(Collection) && record.is_a?(Collection) %>
             <%= link_to record.class.to_s, hyrax.polymorphic_path(record) %>
           <% else %>
             <%= link_to record.class.to_s, main_app.polymorphic_path(record) %>