bulkrax 6.0.1 → 8.0.0

data/app/parsers/bulkrax/application_parser.rb

@@ -1,6 +1,4 @@
 # frozen_string_literal: true
-require 'zip'
-require 'marcel'
 
 module Bulkrax
   # An abstract class that establishes the API for Bulkrax's import and export parsing.
@@ -14,7 +12,7 @@ module Bulkrax
              :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
              :key_without_numbers, :status, :set_status_info, :status_info, :status_at,
              :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
-             :zip?, :file?,
+             :zip?, :file?, :remove_and_rerun,
              to: :importerexporter
 
     # @todo Convert to `class_attribute :parser_fiels, default: {}`
@@ -47,6 +45,10 @@ module Bulkrax
       raise NotImplementedError, 'must be defined'
     end
 
+    def work_entry_class
+      entry_class
+    end
+
     # @api public
     # @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
     def collection_entry_class
@@ -157,6 +159,22 @@ module Bulkrax
       @visibility ||= self.parser_fields['visibility'] || 'open'
     end
 
+    def create_collections
+      create_objects(['collection'])
+    end
+
+    def create_works
+      create_objects(['work'])
+    end
+
+    def create_file_sets
+      create_objects(['file_set'])
+    end
+
+    def create_relationships
+      create_objects(['relationship'])
+    end
+
     # @api public
     #
     # @param types [Array<Symbol>] the types of objects that we'll create.
@@ -166,34 +184,95 @@ module Bulkrax
     # @see #create_works
     # @see #create_file_sets
     # @see #create_relationships
-    def create_objects(types = [])
-      types.each do |object_type|
-        send("create_#{object_type.pluralize}")
+    def create_objects(types_array = nil)
+      index = 0
+      (types_array || %w[collection work file_set relationship]).each do |type|
+        if type.eql?('relationship')
+          ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
+          next
+        end
+        send(type.pluralize).each do |current_record|
+          next unless record_has_source_identifier(current_record, index)
+          break if limit_reached?(limit, index)
+          seen[current_record[source_identifier]] = true
+          create_entry_and_job(current_record, type)
+          increment_counters(index, "#{type}": true)
+          index += 1
+        end
+        importer.record_status
+      end
+      true
+    rescue StandardError => e
+      set_status_info(e)
+    end
+
+    def rebuild_entries(types_array = nil)
+      index = 0
+      (types_array || %w[collection work file_set relationship]).each do |type|
+        # works are not gurneteed to have Work in the type
+
+        importer.entries.where(rebuild_entry_query(type, parser_fields['entry_statuses'])).find_each do |e|
+          seen[e.identifier] = true
+          e.status_info('Pending', importer.current_run)
+          if remove_and_rerun
+            delay = calculate_type_delay(type)
+            "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, e, current_run)
+          else
+            "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, e.id, current_run.id)
+          end
+          increment_counters(index)
+          index += 1
+        end
       end
     end
 
-    # @abstract Subclass and override {#create_collections} to implement behavior for the parser.
-    def create_collections
-      raise NotImplementedError, 'must be defined' if importer?
+    def rebuild_entry_query(type, statuses)
+      type_col = Bulkrax::Entry.arel_table['type']
+      status_col = Bulkrax::Entry.arel_table['status_message']
+
+      query = (type == 'work' ? type_col.does_not_match_all(%w[collection file_set]) : type_col.matches(type.camelize))
+      query.and(status_col.in(statuses))
     end
 
-    # @abstract Subclass and override {#create_works} to implement behavior for the parser.
-    def create_works
-      raise NotImplementedError, 'must be defined' if importer?
+    def calculate_type_delay(type)
+      return 2.minutes if type == 'file_set'
+      return 1.minute if type == 'work'
+      return 0
     end
 
-    # @abstract Subclass and override {#create_file_sets} to implement behavior for the parser.
-    def create_file_sets
-      raise NotImplementedError, 'must be defined' if importer?
+    def record_raw_metadata(record)
+      record.to_h
     end
 
-    # @abstract Subclass and override {#create_relationships} to implement behavior for the parser.
-    def create_relationships
-      raise NotImplementedError, 'must be defined' if importer?
+    def record_deleted?(record)
+      return false unless record.key?(:delete)
+      ActiveModel::Type::Boolean.new.cast(record[:delete])
+    end
+
+    def record_remove_and_rerun?(record)
+      return false unless record.key?(:remove_and_rerun)
+      ActiveModel::Type::Boolean.new.cast(record[:remove_and_rerun])
+    end
+
+    def create_entry_and_job(current_record, type, identifier = nil)
+      identifier ||= current_record[source_identifier]
+      new_entry = find_or_create_entry(send("#{type}_entry_class"),
+                                       identifier,
+                                       'Bulkrax::Importer',
+                                       record_raw_metadata(current_record))
+      new_entry.status_info('Pending', importer.current_run)
+      if record_deleted?(current_record)
+        "Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
+      elsif record_remove_and_rerun?(current_record) || remove_and_rerun
+        delay = calculate_type_delay(type)
+        "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, new_entry, current_run)
+      else
+        "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
+      end
     end
 
     # Optional, define if using browse everything for file upload
-    def retrieve_cloud_files(files); end
+    def retrieve_cloud_files(_files, _importer); end
 
     # @param file [#path, #original_filename] the file object that with the relevant data for the
     #        import.
@@ -305,12 +384,19 @@ module Bulkrax
     end
 
     def find_or_create_entry(entryclass, identifier, type, raw_metadata = nil)
-      entry = entryclass.where(
+      # limit entry search to just this importer or exporter. Don't go moving them
+      entry = importerexporter.entries.where(
+        identifier: identifier
+      ).first
+      entry ||= entryclass.new(
         importerexporter_id: importerexporter.id,
         importerexporter_type: type,
         identifier: identifier
-      ).first_or_create!
+      )
       entry.raw_metadata = raw_metadata
+      # Setting parsed_metadata specifically for the id so we can find the object via the
+      # id in a delete.  This is likely to get clobbered in a regular import, which is fine.
+      entry.parsed_metadata = { id: raw_metadata['id'] } if raw_metadata&.key?('id')
       entry.save!
       entry
     end
@@ -342,6 +428,8 @@ module Bulkrax
     end
 
     def unzip(file_to_unzip)
+      return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
+
       Zip::File.open(file_to_unzip) do |zip_file|
         zip_file.each do |entry|
           entry_path = File.join(importer_unzip_path, entry.name)
@@ -351,6 +439,13 @@ module Bulkrax
       end
     end
 
+    def untar(file_to_untar)
+      Dir.mkdir(importer_unzip_path) unless File.directory?(importer_unzip_path)
+      command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"
+      result = system(command)
+      raise "Failed to extract #{file_to_untar}" unless result
+    end
+
     def zip
       FileUtils.mkdir_p(exporter_export_zip_path)
 

data/app/parsers/bulkrax/bagit_parser.rb

@@ -1,246 +1,224 @@
 # frozen_string_literal: true
-require 'bagit'
 
-module Bulkrax
-  class BagitParser < CsvParser # rubocop:disable Metrics/ClassLength
-    include ExportBehavior
+unless ENV.fetch('BULKRAX_NO_BAGIT', 'false').to_s == 'true'
+  module Bulkrax
+    class BagitParser < CsvParser # rubocop:disable Metrics/ClassLength
+      include ExportBehavior
 
-    def self.export_supported?
-      true
-    end
-
-    def valid_import?
-      return true if import_fields.present?
-    rescue => e
-      set_status_info(e)
-      false
-    end
-
-    def entry_class
-      rdf_format = parser_fields&.[]('metadata_format') == "Bulkrax::RdfEntry"
-      rdf_format ? RdfEntry : CsvEntry
-    end
-
-    def path_to_files(filename:)
-      @path_to_files ||= Dir.glob(File.join(import_file_path, '**/data', filename)).first
-    end
-
-    # Take a random sample of 10 metadata_paths and work out the import fields from that
-    def import_fields
-      raise StandardError, 'No metadata files were found' if metadata_paths.blank?
-      @import_fields ||= metadata_paths.sample(10).map do |path|
-        entry_class.fields_from_data(entry_class.read_data(path))
-      end.flatten.compact.uniq
-    end
+      def self.export_supported?
+        true
+      end
 
-    # Create an Array of all metadata records
-    def records(_opts = {})
-      raise StandardError, 'No BagIt records were found' if bags.blank?
-      @records ||= bags.map do |bag|
-        path = metadata_path(bag)
-        raise StandardError, 'No metadata files were found' if path.blank?
-        data = entry_class.read_data(path)
-        get_data(bag, data)
+      def valid_import?
+        return true if import_fields.present?
+      rescue => e
+        set_status_info(e)
+        false
       end
 
-      @records = @records.flatten
-    end
+      def entry_class
+        rdf_format = parser_fields&.[]('metadata_format') == "Bulkrax::RdfEntry"
+        rdf_format ? RdfEntry : CsvEntry
+      end
 
-    def get_data(bag, data)
-      if entry_class == CsvEntry
-        data = data.map do |data_row|
-          record_data = entry_class.data_for_entry(data_row, source_identifier, self)
-          next record_data if importerexporter.metadata_only?
+      def path_to_files(filename:)
+        @path_to_files ||= Dir.glob(File.join(import_file_path, '**/data', filename)).first
+      end
 
-          record_data[:file] = bag.bag_files.join('|') if Bulkrax.curation_concerns.include? record_data[:model]&.constantize
-          record_data
-        end
-      else
-        data = entry_class.data_for_entry(data, source_identifier, self)
-        data[:file] = bag.bag_files.join('|') unless importerexporter.metadata_only?
+      # Take a random sample of 10 metadata_paths and work out the import fields from that
+      def import_fields
+        raise StandardError, 'No metadata files were found' if metadata_paths.blank?
+        @import_fields ||= metadata_paths.sample(10).map do |path|
+          entry_class.fields_from_data(entry_class.read_data(path))
+        end.flatten.compact.uniq
       end
 
-      data
-    end
+      # Create an Array of all metadata records
+      def records(_opts = {})
+        raise StandardError, 'No BagIt records were found' if bags.blank?
+        @records ||= bags.map do |bag|
+          path = metadata_path(bag)
+          raise StandardError, 'No metadata files were found' if path.blank?
+          data = entry_class.read_data(path)
+          get_data(bag, data)
+        end
 
-    def create_works
-      entry_class == CsvEntry ? super : create_rdf_works
-    end
+        @records = @records.flatten
+      end
 
-    def create_rdf_works
-      records.each_with_index do |record, index|
-        next unless record_has_source_identifier(record, index)
-        break if limit_reached?(limit, index)
+      def get_data(bag, data)
+        if entry_class == CsvEntry
+          data = data.map do |data_row|
+            record_data = entry_class.data_for_entry(data_row, source_identifier, self)
+            next record_data if importerexporter.metadata_only?
 
-        seen[record[source_identifier]] = true
-        new_entry = find_or_create_entry(entry_class, record[source_identifier], 'Bulkrax::Importer', record)
-        if record[:delete].present?
-          DeleteWorkJob.send(perform_method, new_entry, current_run)
+            record_data[:file] = bag.bag_files.join('|') if Bulkrax.curation_concerns.include? record_data[:model]&.constantize
+            record_data
+          end
         else
-          ImportWorkJob.send(perform_method, new_entry.id, current_run.id)
+          data = entry_class.data_for_entry(data, source_identifier, self)
+          data[:file] = bag.bag_files.join('|') unless importerexporter.metadata_only?
         end
-        increment_counters(index, work: true)
+
+        data
       end
-      importer.record_status
-    rescue StandardError => e
-      set_status_info(e)
-    end
 
-    # export methods
+      # export methods
 
-    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-    def write_files
-      require 'open-uri'
-      require 'socket'
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def write_files
+        require 'open-uri'
+        require 'socket'
 
-      folder_count = 1
-      records_in_folder = 0
-      work_entries = importerexporter.entries.where(type: work_entry_class.to_s)
-      collection_entries = importerexporter.entries.where(type: collection_entry_class.to_s)
-      file_set_entries = importerexporter.entries.where(type: file_set_entry_class.to_s)
+        folder_count = 1
+        records_in_folder = 0
+        work_entries = importerexporter.entries.where(type: work_entry_class.to_s)
+        collection_entries = importerexporter.entries.where(type: collection_entry_class.to_s)
+        file_set_entries = importerexporter.entries.where(type: file_set_entry_class.to_s)
 
-      work_entries[0..limit || total].each do |entry|
-        record = ActiveFedora::Base.find(entry.identifier)
-        next unless record
+        work_entries[0..limit || total].each do |entry|
+          record = Bulkrax.object_factory.find(entry.identifier)
+          next unless record
 
-        bag_entries = [entry]
+          bag_entries = [entry]
 
-        if record.member_of_collection_ids.present?
-          collection_entries.each { |ce| bag_entries << ce if ce.parsed_metadata.value?(record.id) }
-        end
+          if record.member_of_collection_ids.present?
+            collection_entries.each { |ce| bag_entries << ce if ce.parsed_metadata.value?(record.id) }
+          end
 
-        if record.file_sets.present?
-          file_set_entries.each { |fse| bag_entries << fse if fse.parsed_metadata.value?(record.id) }
-        end
+          if record.file_sets.present?
+            file_set_entries.each { |fse| bag_entries << fse if fse.parsed_metadata.value?(record.id) }
+          end
 
-        records_in_folder += bag_entries.count
-        if records_in_folder > records_split_count
-          folder_count += 1
-          records_in_folder = bag_entries.count
-        end
+          records_in_folder += bag_entries.count
+          if records_in_folder > records_split_count
+            folder_count += 1
+            records_in_folder = bag_entries.count
+          end
 
-        bag ||= BagIt::Bag.new setup_bagit_folder(folder_count, entry.identifier)
-
-        record.file_sets.each do |fs|
-          file_name = filename(fs)
-          next if file_name.blank? || fs.original_file.blank?
-
-          io = open(fs.original_file.uri)
-          file = Tempfile.new([file_name, File.extname(file_name)], binmode: true)
-          file.write(io.read)
-          file.close
-          begin
-            bag.add_file(file_name, file.path) if bag.bag_files.select { |b| b.include?(file_name) }.blank?
-          rescue => e
-            entry.set_status_info(e)
-            set_status_info(e)
+          bag ||= BagIt::Bag.new setup_bagit_folder(folder_count, entry.identifier)
+
+          record.file_sets.each do |fs|
+            file_name = filename(fs)
+            next if file_name.blank? || fs.original_file.blank?
+
+            io = open(fs.original_file.uri)
+            file = Tempfile.new([file_name, File.extname(file_name)], binmode: true)
+            file.write(io.read)
+            file.close
+            begin
+              bag.add_file(file_name, file.path) if bag.bag_files.select { |b| b.include?(file_name) }.blank?
+            rescue => e
+              entry.set_status_info(e)
+              set_status_info(e)
+            end
           end
-        end
 
-        CSV.open(setup_csv_metadata_export_file(folder_count, entry.identifier), "w", headers: export_headers, write_headers: true) do |csv|
-          bag_entries.each { |csv_entry| csv << csv_entry.parsed_metadata }
-        end
+          CSV.open(setup_csv_metadata_export_file(folder_count, entry.identifier), "w", headers: export_headers, write_headers: true) do |csv|
+            bag_entries.each { |csv_entry| csv << csv_entry.parsed_metadata }
+          end
 
-        write_triples(folder_count, entry)
-        bag.manifest!(algo: 'sha256')
+          write_triples(folder_count, entry)
+          bag.manifest!(algo: 'sha256')
+        end
       end
-    end
-    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
-    def setup_csv_metadata_export_file(folder_count, id)
-      path = File.join(importerexporter.exporter_export_path, folder_count.to_s)
-      FileUtils.mkdir_p(path) unless File.exist?(path)
+      def setup_csv_metadata_export_file(folder_count, id)
+        path = File.join(importerexporter.exporter_export_path, folder_count.to_s)
+        FileUtils.mkdir_p(path) unless File.exist?(path)
 
-      File.join(path, id, 'metadata.csv')
-    end
+        File.join(path, id, 'metadata.csv')
+      end
 
-    def key_allowed(key)
-      !Bulkrax.reserved_properties.include?(key) &&
-        new_entry(entry_class, 'Bulkrax::Exporter').field_supported?(key) &&
-        key != source_identifier.to_s
-    end
+      def key_allowed(key)
+        !Bulkrax.reserved_properties.include?(key) &&
+          new_entry(entry_class, 'Bulkrax::Exporter').field_supported?(key) &&
+          key != source_identifier.to_s
+      end
 
-    def setup_triple_metadata_export_file(folder_count, id)
-      path = File.join(importerexporter.exporter_export_path, folder_count.to_s)
-      FileUtils.mkdir_p(path) unless File.exist?(path)
+      def setup_triple_metadata_export_file(folder_count, id)
+        path = File.join(importerexporter.exporter_export_path, folder_count.to_s)
+        FileUtils.mkdir_p(path) unless File.exist?(path)
 
-      File.join(path, id, 'metadata.nt')
-    end
+        File.join(path, id, 'metadata.nt')
+      end
 
-    def setup_bagit_folder(folder_count, id)
-      path = File.join(importerexporter.exporter_export_path, folder_count.to_s)
-      FileUtils.mkdir_p(path) unless File.exist?(path)
+      def setup_bagit_folder(folder_count, id)
+        path = File.join(importerexporter.exporter_export_path, folder_count.to_s)
+        FileUtils.mkdir_p(path) unless File.exist?(path)
 
-      File.join(path, id)
-    end
+        File.join(path, id)
+      end
 
-    # @todo(bjustice) - remove hyrax reference
-    def write_triples(folder_count, e)
-      sd = SolrDocument.find(e.identifier)
-      return if sd.nil?
+      # @todo(bjustice) - remove hyrax reference
+      def write_triples(folder_count, e)
+        sd = SolrDocument.find(e.identifier)
+        return if sd.nil?
 
-      req = ActionDispatch::Request.new({ 'HTTP_HOST' => Socket.gethostname })
-      rdf = Hyrax::GraphExporter.new(sd, req).fetch.dump(:ntriples)
-      File.open(setup_triple_metadata_export_file(folder_count, e.identifier), "w") do |triples|
-        triples.write(rdf)
+        req = ActionDispatch::Request.new({ 'HTTP_HOST' => Socket.gethostname })
+        rdf = Hyrax::GraphExporter.new(sd, req).fetch.dump(:ntriples)
+        File.open(setup_triple_metadata_export_file(folder_count, e.identifier), "w") do |triples|
+          triples.write(rdf)
+        end
       end
-    end
 
-    # @todo - investigate getting directory structure
-    # @todo - investigate using perform_later, and having the importer check for
-    #   DownloadCloudFileJob before it starts
-    def retrieve_cloud_files(files)
-      # There should only be one zip file for Bagit, take the first
-      return if files['0'].blank?
-      target_file = File.join(path_for_import, files['0']['file_name'].tr(' ', '_'))
-      # Now because we want the files in place before the importer runs
-      Bulkrax::DownloadCloudFileJob.perform_now(files['0'], target_file)
-      return target_file
-    end
+      # @todo - investigate getting directory structure
+      # @todo - investigate using perform_later, and having the importer check for
+      #   DownloadCloudFileJob before it starts
+      def retrieve_cloud_files(files, _importer)
+        # There should only be one zip file for Bagit, take the first
+        return if files['0'].blank?
+        target_file = File.join(path_for_import, files['0']['file_name'].tr(' ', '_'))
+        # Now because we want the files in place before the importer runs
+        Bulkrax::DownloadCloudFileJob.perform_now(files['0'], target_file)
+        return target_file
+      end
 
-    private
+      private
 
-    def bags
-      return @bags if @bags.present?
-      new_bag = bag(import_file_path)
-      @bags = new_bag ? [new_bag] : Dir.glob("#{import_file_path}/**/*").map { |d| bag(d) }
-      @bags.delete(nil)
-      raise StandardError, 'No valid bags found' if @bags.blank?
-      return @bags
-    end
+      def bags
+        return @bags if @bags.present?
+        new_bag = bag(import_file_path)
+        @bags = new_bag ? [new_bag] : Dir.glob("#{import_file_path}/**/*").map { |d| bag(d) }
+        @bags.delete(nil)
+        raise StandardError, 'No valid bags found' if @bags.blank?
+        return @bags
+      end
 
-    # Gather the paths to all bags; skip any stray files
-    def bag_paths
-      bags.map(&:bag_dir)
-    end
+      # Gather the paths to all bags; skip any stray files
+      def bag_paths
+        bags.map(&:bag_dir)
+      end
 
-    def metadata_file_name
-      raise StandardError, 'The metadata file name must be specified' if parser_fields['metadata_file_name'].blank?
-      parser_fields['metadata_file_name']
-    end
+      def metadata_file_name
+        raise StandardError, 'The metadata file name must be specified' if parser_fields['metadata_file_name'].blank?
+        parser_fields['metadata_file_name']
+      end
 
-    # Gather the paths to all metadata files matching the metadata_file_name
-    def metadata_paths
-      @metadata_paths ||= bag_paths.map do |b|
-        Dir.glob("#{b}/**/*").select { |f| File.file?(f) && f.ends_with?(metadata_file_name) }
-      end.flatten.compact
-    end
+      # Gather the paths to all metadata files matching the metadata_file_name
+      def metadata_paths
+        @metadata_paths ||= bag_paths.map do |b|
+          Dir.glob("#{b}/**/*").select { |f| File.file?(f) && f.ends_with?(metadata_file_name) }
+        end.flatten.compact
+      end
 
-    def metadata_path(bag)
-      Dir.glob("#{bag.bag_dir}/**/*").detect { |f| File.file?(f) && f.ends_with?(metadata_file_name) }
-    end
+      def metadata_path(bag)
+        Dir.glob("#{bag.bag_dir}/**/*").detect { |f| File.file?(f) && f.ends_with?(metadata_file_name) }
+      end
 
-    def bag(path)
-      return nil unless path && File.exist?(File.join(path, 'bagit.txt'))
-      bag = BagIt::Bag.new(path)
-      return nil unless bag.valid?
-      bag
-    end
+      def bag(path)
+        return nil unless path && File.exist?(File.join(path, 'bagit.txt'))
+        bag = BagIt::Bag.new(path)
+        return nil unless bag.valid?
+        bag
+      end
 
-    # use the version of this method from the application parser instead
-    def real_import_file_path
-      return importer_unzip_path if file? && zip?
-      parser_fields['import_file_path']
+      # use the version of this method from the application parser instead
+      def real_import_file_path
+        return importer_unzip_path if file? && zip?
+        parser_fields['import_file_path']
+      end
     end
   end
 end