bulkrax 1.0.0

data/app/models/concerns/bulkrax/download_behavior.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module DownloadBehavior
+    # The following download code is based on
+    # https://github.com/samvera/hydra-head/blob/main/hydra-core/app/controllers/concerns/hydra/controller/download_behavior.rb
+
+    def file
+      @file ||= File.open(file_path, 'r')
+    end
+
+    # Override this if you'd like a different filename
+    # @return [String] the filename
+    def file_name
+      file_path.split('/').last
+    end
+
+    def download_content_type
+      'application/zip'
+    end
+
+    def send_content
+      response.headers['Accept-Ranges'] = 'bytes'
+      if request.head?
+        content_head
+      else
+        send_file_contents
+      end
+    end
+
+    # Create some headers for the datastream
+    def content_options
+      { disposition: 'inline', type: download_content_type, filename: file_name }
+    end
+
+    # render an HTTP HEAD response
+    def content_head
+      response.headers['Content-Length'] = file.size
+      head :ok, content_type: download_content_type
+    end
+
+    def send_file_contents
+      self.status = 200
+      prepare_file_headers
+      stream_body file.read
+    end
+
+    def prepare_file_headers
+      send_file_headers! content_options
+      response.headers['Content-Type'] = download_content_type
+      response.headers['Content-Length'] ||= file.size.to_s
+      # Prevent Rack::ETag from calculating a digest over body
+      response.headers['Last-Modified'] = File.mtime(file_path).utc.strftime("%a, %d %b %Y %T GMT")
+      self.content_type = download_content_type
+    end
+
+    def stream_body(iostream)
+      self.response_body = iostream
+    end
+  end
+end

data/app/models/concerns/bulkrax/errored_entries.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module ErroredEntries
+    extend ActiveSupport::Concern
+
+    def write_errored_entries_file
+      if @errored_entries.blank?
+        entry_ids = importerexporter.entries.pluck(:id)
+        error_statuses = Bulkrax::Status.latest_by_statusable
+                                        .includes(:statusable)
+                                        .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Failed')
+        @errored_entries = error_statuses.map(&:statusable)
+      end
+      return if @errored_entries.blank?
+
+      file = setup_errored_entries_file
+      headers = import_fields
+      file.puts(headers.to_csv)
+      @errored_entries.each do |ee|
+        row = build_errored_entry_row(headers, ee)
+        file.puts(row)
+      end
+      file.close
+      true
+    end
+
+    def build_errored_entry_row(headers, errored_entry)
+      row = {}
+      # Ensure each header has a value, even if it's just an empty string
+      headers.each do |h|
+        row.merge!("#{h}": nil)
+      end
+      # Match each value to its corresponding header
+      row.merge!(errored_entry.raw_metadata.symbolize_keys)
+
+      row.values.to_csv
+    end
+
+    def setup_errored_entries_file
+      FileUtils.mkdir_p(File.dirname(importerexporter.errored_entries_csv_path))
+      File.open(importerexporter.errored_entries_csv_path, 'w')
+    end
+  end
+end

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+module Bulkrax
+  module ExportBehavior
+    extend ActiveSupport::Concern
+
+    delegate :export_type, :exporter_export_path, to: :importerexporter
+
+    def build_for_exporter
+      build_export_metadata
+      write_files if export_type == 'full' && !importerexporter.parser_klass.include?('Bagit')
+    rescue RSolr::Error::Http, CollectionsCreatedError => e
+      raise e
+    rescue StandardError => e
+      status_info(e)
+    else
+      status_info
+    end
+
+    def build_export_metadata
+      raise StandardError, 'not implemented'
+    end
+
+    def hyrax_record
+      @hyrax_record ||= ActiveFedora::Base.find(self.identifier)
+    end
+
+    def write_files
+      return if hyrax_record.is_a?(Collection)
+      hyrax_record.file_sets.each do |fs|
+        path = File.join(exporter_export_path, 'files')
+        FileUtils.mkdir_p(path)
+        file = filename(fs)
+        require 'open-uri'
+        io = open(fs.original_file.uri)
+        next if file.blank?
+        File.open(File.join(path, file), 'wb') do |f|
+          f.write(io.read)
+          f.close
+        end
+      end
+    end
+
+    # Append the file_set id to ensure a unique filename
+    def filename(file_set)
+      return if file_set.original_file.blank?
+      fn = file_set.original_file.file_name.first
+      mime = Mime::Type.lookup(file_set.original_file.mime_type)
+      ext_mime = MIME::Types.of(file_set.original_file.file_name).first
+      if fn.include?(file_set.id)
+        return fn if mime.to_s == ext_mime.to_s
+        return "#{fn}.#{mime.to_sym}"
+      else
+        return "#{file_set.id}_#{fn}" if mime.to_s == ext_mime.to_s
+        return "#{file_set.id}_#{fn}.#{mime.to_sym}"
+      end
+    end
+  end
+end

data/app/models/concerns/bulkrax/file_factory.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module FileFactory
+    extend ActiveSupport::Concern
+
+    # Find existing files or upload new files. This assumes a Work will have unique file titles;
+    #   and that those file titles will not have changed
+    # could filter by URIs instead (slower).
+    # When an uploaded_file already exists we do not want to pass its id in `file_attributes`
+    # otherwise it gets reuploaded by `work_actor`.
+    # support multiple files; ensure attributes[:file] is an Array
+    def upload_ids
+      return [] if klass == Collection
+      attributes[:file] = file_paths
+      import_files
+    end
+
+    def file_attributes(update_files = false)
+      @update_files = update_files
+      hash = {}
+      return hash if klass == Collection
+      hash[:uploaded_files] = upload_ids if attributes[:file].present?
+      hash[:remote_files] = new_remote_files if new_remote_files.present?
+      hash
+    end
+
+    # Its possible to get just an array of strings here, so we need to make sure they are all hashes
+    def parsed_remote_files
+      return @parsed_remote_files if @parsed_remote_files.present?
+      @parsed_remote_files = attributes[:remote_files] || []
+      @parsed_remote_files = @parsed_remote_files.map do |file_value|
+        if file_value.is_a?(Hash)
+          file_value
+        elsif file_value.is_a?(String)
+          { url: file_value }
+        else
+          Rails.logger.error("skipped remote file #{file_value} because we do not recognize the type")
+          nil
+        end
+      end
+      @parsed_remote_files.delete(nil)
+      @parsed_remote_files
+    end
+
+    def new_remote_files
+      @new_remote_files ||= if object.present? && object.file_sets.present?
+                              parsed_remote_files.select do |file|
+                                # is the url valid?
+                                is_valid = file[:url]&.match(URI::ABS_URI)
+                                # does the file already exist
+                                is_existing = object.file_sets.detect { |f| f.import_url && f.import_url == file[:url] }
+                                is_valid && !is_existing
+                              end
+                            else
+                              parsed_remote_files.select do |file|
+                                file[:url]&.match(URI::ABS_URI)
+                              end
+                            end
+    end
+
+    def file_paths
+      @file_paths ||= Array.wrap(attributes[:file])&.select { |file| File.exist?(file) }
+    end
+
+    # Retrieve the orginal filenames for the files to be imported
+    def work_files_filenames
+      object.file_sets.map { |fn| fn.original_file.file_name.to_a }.flatten if object.present? && object.file_sets.present?
+    end
+
+    # Retrieve the filenames for the files to be imported
+    def import_files_filenames
+      file_paths.map { |f| f.split('/').last }
+    end
+
+    # Called if #replace_files is true
+    # Destroy all file_sets for this object
+    # Reload the object to ensure the remaining methods have the most up to date object
+    def destroy_existing_files
+      return unless object.present? && object.file_sets.present?
+      object.file_sets.each do |fs|
+        Hyrax::Actors::FileSetActor.new(fs, @user).destroy
+      end
+      @object = object.reload
+      log_deleted_fs(object)
+    end
+
+    def set_removed_filesets
+      local_file_sets.each do |fileset|
+        fileset.files.first.create_version
+        opts = {}
+        opts[:path] = fileset.files.first.id.split('/', 2).last
+        opts[:original_name] = 'removed.png'
+        opts[:mime_type] = 'image/png'
+
+        fileset.add_file(File.open(Bulkrax.removed_image_path), opts)
+        fileset.save
+        ::CreateDerivativesJob.set(wait: 1.minute).perform_later(fileset, fileset.files.first.id)
+      end
+    end
+
+    def local_file_sets
+      @local_file_sets ||= object&.ordered_file_sets
+    end
+
+    def import_files
+      paths = file_paths.map { |path| import_file(path) }.compact
+      set_removed_filesets if local_file_sets.present?
+      paths
+    end
+
+    def import_file(path)
+      u = Hyrax::UploadedFile.new
+      u.user_id = @user.id
+      u.file = CarrierWave::SanitizedFile.new(path)
+      update_filesets(u)
+    end
+
+    def update_filesets(current_file)
+      if @update_files && local_file_sets.present?
+        fileset = local_file_sets.shift
+        return nil if fileset.files.first.checksum.value == Digest::SHA1.file(current_file.file.path).to_s
+
+        fileset.files.first.create_version
+        opts = {}
+        opts[:path] = fileset.files.first.id.split('/', 2).last
+        opts[:original_name] = current_file.file.file.original_filename
+        opts[:mime_type] = current_file.file.content_type
+
+        fileset.add_file(File.open(current_file.file.to_s), opts)
+        fileset.save
+        ::CreateDerivativesJob.set(wait: 1.minute).perform_later(fileset, fileset.files.first.id)
+        nil
+      else
+        current_file.save
+        current_file.id
+      end
+    end
+  end
+end

data/app/models/concerns/bulkrax/has_local_processing.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module HasLocalProcessing
+    def add_local; end
+  end
+end

data/app/models/concerns/bulkrax/has_matchers.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module HasMatchers
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :matchers
+      self.matchers ||= {}
+    end
+
+    class_methods do
+      def matcher_class
+        Bulkrax::ApplicationMatcher
+      end
+
+      def matcher(name, args = {})
+        matcher = matcher_class.new(
+          to: name,
+          parsed: args[:parsed],
+          split: args[:split],
+          if: args[:if],
+          excluded: args[:excluded],
+          nested_type: args[:nested_type]
+        )
+        self.matchers[name] = matcher
+      end
+    end
+
+    def add_metadata(node_name, node_content, index = nil)
+      field_to(node_name).each do |name|
+        matcher = self.class.matcher(name, mapping[name].symbolize_keys) if mapping[name] # the field matched to a pre parsed value in application_matcher.rb
+        object_name = get_object_name(name) || false # the "key" of an object property. e.g. { object_name: { alpha: 'beta' } }
+        multiple = multiple?(name) # the property has multiple values. e.g. 'letters': ['a', 'b', 'c']
+        object_multiple = object_name && multiple?(object_name) # the property's value is an array of object(s)
+
+        next unless field_supported?(name) || (object_name && field_supported?(object_name))
+
+        if object_name
+          Rails.logger.info("Bulkrax Column automatically matched object #{node_name}, #{node_content}")
+          parsed_metadata[object_name] ||= object_multiple ? [{}] : {}
+        end
+
+        value = if matcher
+                  result = matcher.result(self, node_content)
+                  next unless result
+                  matched_metadata(multiple, name, result, object_multiple)
+                elsif multiple
+                  Rails.logger.info("Bulkrax Column automatically matched #{node_name}, #{node_content}")
+                  multiple_metadata(node_content)
+                else
+                  Rails.logger.info("Bulkrax Column automatically matched #{node_name}, #{node_content}")
+                  single_metadata(node_content)
+                end
+
+        set_parsed_data(object_multiple, object_name, name, index, value) if value
+      end
+    end
+
+    def set_parsed_data(object_multiple, object_name, name, index, value)
+      if object_multiple
+        parsed_metadata[object_name][index] ||= {}
+        parsed_metadata[object_name][index][name] ||= []
+        if value.is_a?(Array)
+          parsed_metadata[object_name][index][name] += value
+        else
+          parsed_metadata[object_name][index][name] = value
+        end
+      elsif object_name
+        parsed_metadata[object_name][name] ||= []
+        if value.is_a?(Array)
+          parsed_metadata[object_name][name] += value
+        else
+          parsed_metadata[object_name][name] = value
+        end
+      else
+        parsed_metadata[name] ||= []
+        if value.is_a?(Array)
+          parsed_metadata[name] += value
+        else
+          parsed_metadata[name] = value
+        end
+      end
+    end
+
+    def single_metadata(content)
+      content = content.content if content.is_a?(Nokogiri::XML::NodeSet)
+      return unless content
+      Array.wrap(content.to_s.strip).join('; ')
+    end
+
+    def multiple_metadata(content)
+      content = content.content if content.is_a?(Nokogiri::XML::NodeSet)
+      return unless content
+      content.is_a?(Array) ? content : Array.wrap(content.strip)
+    end
+
+    def matched_metadata(multiple, name, result, object_multiple)
+      if object_multiple
+        if mapping[name]['nested_type'] && mapping[name]['nested_type'] == 'Array'
+          multiple_metadata(result)
+        else
+          single_metadata(result)
+        end
+      elsif multiple
+        multiple_metadata(result)
+      else
+        single_metadata(result)
+      end
+    end
+
+    def field_supported?(field)
+      field = field.gsub('_attributes', '')
+
+      return false if excluded?(field)
+      return true if ['collections', 'file', 'remote_files', 'model', 'delete'].include?(field)
+      return factory_class.method_defined?(field) && factory_class.properties[field].present?
+    end
+
+    def multiple?(field)
+      return true if field == 'file' || field == 'remote_files' || field == 'collections'
+      return false if field == 'model'
+
+      field_supported?(field) && factory_class&.properties&.[](field)&.[]('multiple')
+    end
+
+    def get_object_name(field)
+      mapping&.[](field)&.[]('object')
+    end
+
+    # Hyrax field to use for the given import field
+    # @param field [String] the importer field name
+    # @return [Array] hyrax fields
+    def field_to(field)
+      fields = mapping&.map do |key, value|
+        return unless value
+
+        if value['from'].instance_of?(Array)
+          key if value['from'].include?(field) || key == field
+        elsif (value['from'] == field) || key == field
+          key
+        end
+      end&.compact
+
+      return [field] if fields.blank?
+      return fields
+    end
+
+    # Check whether a field is explicitly excluded in the mapping
+    def excluded?(field)
+      return false if mapping[field].blank?
+      mapping[field]['excluded'] || false
+    end
+  end
+end