bulkrax 1.0.0

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Import Behavior for Entry classes
+  module ImportBehavior
+    extend ActiveSupport::Concern
+
+    def build_for_importer
+      begin
+        build_metadata
+        unless self.importerexporter.validate_only
+          raise CollectionsCreatedError unless collections_created?
+          @item = factory.run!
+        end
+      rescue RSolr::Error::Http, CollectionsCreatedError => e
+        raise e
+      rescue StandardError => e
+        status_info(e)
+      else
+        status_info
+      end
+      return @item
+    end
+
+    def find_or_create_collection_ids
+      self.collection_ids
+    end
+
+    # override this in a sub-class of Entry to ensure any collections have been created before building the work
+    def collections_created?
+      true
+    end
+
+    def build_metadata
+      raise StandardError, 'Not Implemented'
+    end
+
+    def rights_statement
+      parser.parser_fields['rights_statement']
+    end
+
+    # try and deal with a couple possible states for this input field
+    def override_rights_statement
+      %w[true 1].include?(parser.parser_fields['override_rights_statement'].to_s)
+    end
+
+    def add_rights_statement
+      self.parsed_metadata['rights_statement'] = [parser.parser_fields['rights_statement']] if override_rights_statement || self.parsed_metadata['rights_statement'].blank?
+    end
+
+    def add_visibility
+      self.parsed_metadata['visibility'] = importerexporter.visibility if self.parsed_metadata['visibility'].blank?
+    end
+
+    def add_admin_set_id
+      self.parsed_metadata['admin_set_id'] = importerexporter.admin_set_id if self.parsed_metadata['admin_set_id'].blank?
+    end
+
+    def add_collections
+      return if find_or_create_collection_ids.blank?
+      self.parsed_metadata['collections'] = []
+      self.parsed_metadata['collections'] += find_or_create_collection_ids.map { |c| { id: c } }
+    end
+
+    def factory
+      @factory ||= Bulkrax::ObjectFactory.new(attributes: self.parsed_metadata,
+                                              source_identifier_value: identifier,
+                                              work_identifier: parser.work_identifier,
+                                              replace_files: replace_files,
+                                              user: user,
+                                              klass: factory_class,
+                                              update_files: update_files)
+    end
+
+    def factory_class
+      fc = if self.parsed_metadata&.[]('model').present?
+             self.parsed_metadata&.[]('model').is_a?(Array) ? self.parsed_metadata&.[]('model')&.first : self.parsed_metadata&.[]('model')
+           elsif self.mapping&.[]('work_type').present?
+             self.parsed_metadata&.[]('work_type').is_a?(Array) ? self.parsed_metadata&.[]('work_type')&.first : self.parsed_metadata&.[]('work_type')
+           else
+             Bulkrax.default_work_type
+           end
+      fc.constantize
+    rescue NameError
+      nil
+    rescue
+      Bulkrax.default_work_type.constantize
+    end
+  end
+end

data/app/models/concerns/bulkrax/importer_exporter_behavior.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module ImporterExporterBehavior
+    extend ActiveSupport::Concern
+
+    def parser
+      @parser ||= parser_class.new(self)
+    end
+
+    def parser_class
+      self.parser_klass.constantize
+    end
+
+    def last_imported_at
+      @last_imported_at ||= self.importer_runs.last&.created_at
+    end
+
+    def next_import_at
+      (last_imported_at || Time.current) + frequency.to_seconds if schedulable? && last_imported_at.present?
+    end
+
+    def increment_counters(index, collection = false)
+      # Only set the totals if they were not set on initialization
+      if collection
+        current_run.total_collection_entries = index + 1 unless parser.collections_total.positive?
+      else
+        current_run.total_work_entries = index + 1 unless limit.to_i.positive? || parser.total.positive?
+      end
+      current_run.enqueued_records = index + 1
+      current_run.save!
+    end
+  end
+end

data/app/models/concerns/bulkrax/status_info.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+module Bulkrax
+  module StatusInfo
+    extend ActiveSupport::Concern
+
+    included do
+      has_many :statuses, as: :statusable, dependent: :destroy
+      has_one :latest_status,
+              -> { merge(Status.latest_by_statusable) },
+              as: :statusable,
+              class_name: "Bulkrax::Status",
+              inverse_of: :statusable
+    end
+
+    def current_status
+      last_status = self.statuses.last
+      last_status if last_status && last_status.runnable == last_run
+    end
+
+    def failed?
+      current_status&.status_message&.match(/fail/i)
+    end
+
+    def succeeded?
+      current_status&.status_message&.match(/^Complete$/)
+    end
+
+    def status
+      current_status&.status_message || 'Pending'
+    end
+
+    def status_at
+      current_status&.created_at
+    end
+
+    def status_info(e = nil)
+      if e.nil?
+        self.statuses.create!(status_message: 'Complete', runnable: last_run)
+      elsif e.is_a?(String)
+        self.statuses.create!(status_message: e, runnable: last_run)
+      else
+        self.statuses.create!(status_message: 'Failed', runnable: last_run, error_class: e.class.to_s, error_message: e.message, error_backtrace: e.backtrace)
+      end
+    end
+
+    # api compatible with previous error structure
+    def last_error
+      return unless current_status && current_status.error_class.present?
+      {
+        error_class: current_status.error_class,
+        error_message: current_status.error_message,
+        error_trace: current_status.error_backtrace
+      }.with_indifferent_access
+    end
+  end
+end

data/app/parsers/bulkrax/application_parser.rb

@@ -0,0 +1,299 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ApplicationParser
+    attr_accessor :importerexporter
+    alias importer importerexporter
+    alias exporter importerexporter
+    delegate :only_updates, :limit, :current_run, :errors,
+             :seen, :increment_counters, :parser_fields, :user,
+             :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
+             :status, :status_info, :status_at,
+             to: :importerexporter
+
+    def self.parser_fields
+      {}
+    end
+
+    def self.export_supported?
+      false
+    end
+
+    def self.import_supported?
+      true
+    end
+
+    def initialize(importerexporter)
+      @importerexporter = importerexporter
+    end
+
+    # @api
+    def entry_class
+      raise StandardError, 'must be defined'
+    end
+
+    # @api
+    def collection_entry_class
+      raise StandardError, 'must be defined'
+    end
+
+    # @api
+    def records(_opts = {})
+      raise StandardError, 'must be defined'
+    end
+
+    def source_identifier
+      @source_identifier ||= identifier_hash.values.first&.[]("from")&.first&.to_sym || :source_identifier
+    end
+
+    def work_identifier
+      @work_identifier ||= identifier_hash.keys.first&.to_sym || :source
+    end
+
+    def identifier_hash
+      @identifier_hash ||= importerexporter.mapping.select do |_, h|
+        h.key?("source_identifier")
+      end
+      raise StandardError, "more than one source_identifier declared: #{@identifier_hash.keys.join(', ')}" if @identifier_hash.length > 1
+
+      @identifier_hash
+    end
+
+    def perform_method
+      if self.validate_only
+        'perform_now'
+      else
+        'perform_later'
+      end
+    end
+
+    def visibility
+      @visibility ||= self.parser_fields['visibility'] || 'open'
+    end
+
+    def create_collections
+      raise StandardError, 'must be defined' if importer?
+    end
+
+    def create_works
+      raise StandardError, 'must be defined' if importer?
+    end
+
+    # Optional, define if using browse everything for file upload
+    def retrieve_cloud_files(files); end
+
+    def write_import_file(file)
+      path = File.join(path_for_import, file.original_filename)
+      FileUtils.mv(
+        file.path,
+        path
+      )
+      path
+    end
+
+    # Path where we'll store the import metadata and files
+    #  this is used for uploaded and cloud files
+    def path_for_import
+      @path_for_import = File.join(Bulkrax.import_path, importerexporter.path_string)
+      FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
+      @path_for_import
+    end
+
+    # Optional, only used by certain parsers
+    # Other parsers should override with a custom or empty method
+    # Will be skipped unless the #record is a Hash
+    def create_parent_child_relationships
+      parents.each do |key, value|
+        parent = entry_class.where(
+          identifier: key,
+          importerexporter_id: importerexporter.id,
+          importerexporter_type: 'Bulkrax::Importer'
+        ).first
+
+        # not finding the entries here indicates that the given identifiers are incorrect
+        # in that case we should log that
+        children = value.map do |child|
+          entry_class.where(
+            identifier: child,
+            importerexporter_id: importerexporter.id,
+            importerexporter_type: 'Bulkrax::Importer'
+          ).first
+        end.compact.uniq
+
+        if parent.present? && (children.length != value.length)
+          # Increment the failures for the number we couldn't find
+          # Because all of our entries have been created by now, if we can't find them, the data is wrong
+          Rails.logger.error("Expected #{value.length} children for parent entry #{parent.id}, found #{children.length}")
+          break if children.empty?
+          Rails.logger.warn("Adding #{children.length} children to parent entry #{parent.id} (expected #{value.length})")
+        end
+        parent_id = parent.id
+        child_entry_ids = children.map(&:id)
+        ChildRelationshipsJob.perform_later(parent_id, child_entry_ids, current_run.id)
+      end
+    rescue StandardError => e
+      status_info(e)
+    end
+
+    def parents
+      @parents ||= setup_parents
+    end
+
+    def setup_parents
+      pts = []
+      records.each do |record|
+        r = if record.respond_to?(:to_h)
+              record.to_h
+            else
+              record
+            end
+        next unless r.is_a?(Hash)
+        children = if r[:children].is_a?(String)
+                     r[:children].split(/\s*[:;|]\s*/)
+                   else
+                     r[:children]
+                   end
+        next if children.blank?
+        pts << {
+          r[source_identifier] => children
+        }
+      end
+      pts.blank? ? pts : pts.inject(:merge)
+    end
+
+    def setup_export_file
+      raise StandardError, 'must be defined' if exporter?
+    end
+
+    def write_files
+      raise StandardError, 'must be defined' if exporter?
+    end
+
+    def importer?
+      importerexporter.is_a?(Bulkrax::Importer)
+    end
+
+    def exporter?
+      importerexporter.is_a?(Bulkrax::Exporter)
+    end
+
+    # @param limit [Integer] limit set on the importerexporter
+    # @param index [Integer] index of current iteration
+    # @return [boolean]
+    def limit_reached?(limit, index)
+      return false if limit.nil? || limit.zero? # no limit
+      index >= limit
+    end
+
+    # Override to add specific validations
+    def valid_import?
+      true
+    end
+
+    def record_has_source_identifier(record, index)
+      if record[source_identifier].blank?
+        if Bulkrax.fill_in_blank_source_identifiers.present?
+          record[source_identifier] = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
+        else
+          invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
+          false
+        end
+      else
+        true
+      end
+    end
+
+    # rubocop:disable Rails/SkipsModelValidations
+    def invalid_record(message)
+      current_run.invalid_records ||= ""
+      current_run.invalid_records += message
+      current_run.save
+      ImporterRun.find(current_run.id).increment!(:failed_records)
+      ImporterRun.find(current_run.id).decrement!(:enqueued_records) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
+    end
+    # rubocop:enable Rails/SkipsModelValidations
+
+    def required_elements
+      if Bulkrax.fill_in_blank_source_identifiers
+        ['title']
+      else
+        ['title', source_identifier]
+      end
+    end
+
+    def new_entry(entryclass, type)
+      entryclass.new(
+        importerexporter_id: importerexporter.id,
+        importerexporter_type: type
+      )
+    end
+
+    def find_or_create_entry(entryclass, identifier, type, raw_metadata = nil)
+      entry = entryclass.where(
+        importerexporter_id: importerexporter.id,
+        importerexporter_type: type,
+        identifier: identifier
+      ).first_or_create!
+      entry.raw_metadata = raw_metadata
+      entry.save!
+      entry
+    end
+
+    # @todo - review this method - is it ever used?
+    def record(identifier, _opts = {})
+      return @record if @record
+
+      @record = entry_class.new(self, identifier)
+      @record.build
+      return @record
+    end
+
+    def total
+      0
+    end
+
+    def collections_total
+      0
+    end
+
+    def write
+      write_files
+      zip
+    end
+
+    def unzip(file_to_unzip)
+      WillowSword::ZipPackage.new(file_to_unzip, importer_unzip_path).unzip_file
+    end
+
+    def zip
+      FileUtils.rm_rf(exporter_export_zip_path)
+      WillowSword::ZipPackage.new(exporter_export_path, exporter_export_zip_path).create_zip
+    end
+
+    # Is this a file?
+    def file?
+      parser_fields&.[]('import_file_path') && File.file?(parser_fields['import_file_path'])
+    end
+
+    # Is this a zip file?
+    def zip?
+      parser_fields&.[]('import_file_path') && MIME::Types.type_for(parser_fields['import_file_path']).include?('application/zip')
+    end
+
+    # Path for the import
+    def import_file_path
+      @import_file_path ||= real_import_file_path
+    end
+
+    private
+
+    def real_import_file_path
+      if file? && zip?
+        unzip(parser_fields['import_file_path'])
+        return importer_unzip_path
+      else
+        parser_fields['import_file_path']
+      end
+    end
+  end
+end