bulkrax 1.0.0

data/app/matchers/bulkrax/bagit_matcher.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class BagitMatcher < ApplicationMatcher
+  end
+end

data/app/matchers/bulkrax/csv_matcher.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class CsvMatcher < ApplicationMatcher
+  end
+end

data/app/matchers/bulkrax/oai_matcher.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class OaiMatcher < ApplicationMatcher
+  end
+end

data/app/models/bulkrax/application_record.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/bulkrax/csv_collection_entry.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class CsvCollectionEntry < CsvEntry
+    def factory_class
+      Collection
+    end
+
+    def build_metadata
+      self.parsed_metadata = self.raw_metadata
+      add_local
+      return self.parsed_metadata
+    end
+
+    def collections_created?
+      true
+    end
+  end
+end

data/app/models/bulkrax/csv_entry.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Bulkrax
+  class CsvEntry < Entry
+    serialize :raw_metadata, JSON
+
+    def self.fields_from_data(data)
+      data.headers.flatten.compact.uniq
+    end
+
+    # there's a risk that this reads the whole file into memory and could cause a memory leak
+    def self.read_data(path)
+      raise StandardError, 'CSV path empty' if path.blank?
+      CSV.read(path,
+               headers: true,
+               header_converters: :symbol,
+               encoding: 'utf-8')
+    end
+
+    def self.data_for_entry(data, _source_id)
+      # If a multi-line CSV data is passed, grab the first row
+      data = data.first if data.is_a?(CSV::Table)
+      # model has to be separated so that it doesn't get mistranslated by to_h
+      raw_data = data.to_h
+      raw_data[:model] = data[:model]
+      # If the collection field mapping is not 'collection', add 'collection' - the parser needs it
+      raw_data[:collection] = raw_data[collection_field.to_sym] if raw_data.keys.include?(collection_field.to_sym) && collection_field != 'collection'
+      # If the children field mapping is not 'children', add 'children' - the parser needs it
+      raw_data[:children] = raw_data[collection_field.to_sym] if raw_data.keys.include?(children_field.to_sym) && children_field != 'children'
+      return raw_data
+    end
+
+    def self.collection_field
+      Bulkrax.collection_field_mapping[self.class.to_s] || 'collection'
+    end
+
+    def self.children_field
+      Bulkrax.parent_child_field_mapping[self.to_s] || 'children'
+    end
+
+    def keys_without_numbers(keys)
+      keys.map { |key| key_without_numbers(key) }
+    end
+
+    def key_without_numbers(key)
+      key.gsub(/_\d+/, '').sub(/^\d+_/, '')
+    end
+
+    def build_metadata
+      raise StandardError, 'Record not found' if record.nil?
+      raise StandardError, "Missing required elements, missing element(s) are: #{importerexporter.parser.missing_elements(keys_without_numbers(record.keys)).join(', ')}" unless importerexporter.parser.required_elements?(keys_without_numbers(record.keys))
+
+      self.parsed_metadata = {}
+      self.parsed_metadata[work_identifier] = [record[source_identifier]]
+      record.each do |key, value|
+        next if key == 'collection'
+
+        index = key[/\d+/].to_i - 1 if key[/\d+/].to_i != 0
+        add_metadata(key_without_numbers(key), value, index)
+      end
+      add_file
+      add_visibility
+      add_rights_statement
+      add_admin_set_id
+      add_collections
+      add_local
+      self.parsed_metadata
+    end
+
+    def add_file
+      self.parsed_metadata['file'] ||= []
+      if record['file']&.is_a?(String)
+        self.parsed_metadata['file'] = record['file'].split(/\s*[;|]\s*/)
+      elsif record['file'].is_a?(Array)
+        self.parsed_metadata['file'] = record['file']
+      end
+      self.parsed_metadata['file'] = self.parsed_metadata['file'].map { |f| path_to_file(f.tr(' ', '_')) }
+    end
+
+    def build_export_metadata
+      # make_round_trippable
+      self.parsed_metadata = {}
+      self.parsed_metadata['id'] = hyrax_record.id
+      self.parsed_metadata[source_identifier] = hyrax_record.send(work_identifier)
+      self.parsed_metadata['model'] = hyrax_record.has_model.first
+      build_mapping_metadata
+      self.parsed_metadata['collections'] = hyrax_record.member_of_collection_ids.join('; ')
+      unless hyrax_record.is_a?(Collection)
+        self.parsed_metadata['file'] = hyrax_record.file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact.join('; ')
+      end
+      self.parsed_metadata
+    end
+
+    def build_mapping_metadata
+      mapping.each do |key, value|
+        next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
+        next if key == "model"
+        next unless hyrax_record.respond_to?(key.to_s)
+        data = hyrax_record.send(key.to_s)
+        if data.is_a?(ActiveTriples::Relation)
+          self.parsed_metadata[key] = data.map { |d| prepare_export_data(d) }.join('; ').to_s unless value[:excluded]
+        else
+          self.parsed_metadata[key] = prepare_export_data(data)
+        end
+      end
+    end
+
+    def prepare_export_data(datum)
+      if datum.is_a?(ActiveTriples::Resource)
+        datum.to_uri.to_s
+      else
+        datum
+      end
+    end
+
+    # In order for the existing exported hyrax_record, to be updated by a re-import
+    # we need a unique value in system_identifier
+    # add the existing hyrax_record id to system_identifier
+    def make_round_trippable
+      values = hyrax_record.send(work_identifier.to_s).to_a
+      values << hyrax_record.id
+      hyrax_record.send("#{work_identifier}=", values)
+      hyrax_record.save
+    end
+
+    def record
+      @record ||= raw_metadata
+    end
+
+    def self.matcher_class
+      Bulkrax::CsvMatcher
+    end
+
+    def collections_created?
+      return true if record[self.class.collection_field].blank?
+      record[self.class.collection_field].split(/\s*[:;|]\s*/).length == self.collection_ids.length
+    end
+
+    def find_or_create_collection_ids
+      return self.collection_ids if collections_created?
+      valid_system_id(Collection)
+      if record[self.class.collection_field].present?
+        record[self.class.collection_field].split(/\s*[:;|]\s*/).each do |collection|
+          c = find_collection(collection)
+          self.collection_ids << c.id unless c.blank? || self.collection_ids.include?(c.id)
+        end
+      end
+      self.collection_ids
+    end
+
+    # If only filename is given, construct the path (/files/my_file)
+    def path_to_file(file)
+      # return if we already have the full file path
+      return file if File.exist?(file)
+      path = importerexporter.parser.path_to_files
+      f = File.join(path, file)
+      return f if File.exist?(f)
+      raise "File #{f} does not exist"
+    end
+  end
+end

data/app/models/bulkrax/entry.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Custom error class for collections_created?
+  class CollectionsCreatedError < RuntimeError; end
+  class OAIError < RuntimeError; end
+  class Entry < ApplicationRecord
+    include Bulkrax::HasMatchers
+    include Bulkrax::ImportBehavior
+    include Bulkrax::ExportBehavior
+    include Bulkrax::StatusInfo
+    include Bulkrax::HasLocalProcessing
+
+    belongs_to :importerexporter, polymorphic: true
+    alias importer importerexporter
+    alias exporter importerexporter
+
+    serialize :parsed_metadata, JSON
+    # Do not serialize raw_metadata as so we can support xml or other formats
+    serialize :collection_ids, Array
+
+    paginates_per 5
+
+    attr_accessor :all_attrs
+
+    delegate :parser, :mapping, :replace_files, :update_files, to: :importerexporter
+
+    delegate :client,
+             :collection_name,
+             :user,
+             to: :parser
+
+    # Retrieve fields from the file
+    # @param data - the source data
+    # @return Array
+    def self.fields_from_data(_data)
+      raise StandardError, 'Not Implemented'
+    end
+
+    # Read the data from the supplied path
+    # @param path - path to the data file
+    # @return the data from the file
+    def self.read_data(_path)
+      raise StandardError, 'Not Implemented'
+    end
+
+    # Returns formatted data from the given file for a single Entry
+    # @param data - the data from the metadata file
+    # @param path - the path to the metadata file (used by some entries to get the file_paths for import)
+    # @return Hash containing the data (the entry build_metadata method will know what to expect in the hash)
+    def self.data_for_entry(_data, _source_id)
+      raise StandardError, 'Not Implemented'
+    end
+
+    def source_identifier
+      parser&.source_identifier&.to_s || 'source_identifier'
+    end
+
+    def work_identifier
+      parser&.work_identifier&.to_s || 'source'
+    end
+
+    def self.collection_field
+      Bulkrax.collection_field_mapping[self.to_s]
+    end
+
+    def self.children_field
+      Bulkrax.parent_child_field_mapping[self.to_s]
+    end
+
+    def build
+      return if type.nil?
+      self.save if self.new_record? # must be saved for statuses
+
+      return build_for_importer if importer?
+      return build_for_exporter if exporter?
+    end
+
+    def importer?
+      self.importerexporter_type == 'Bulkrax::Importer'
+    end
+
+    def exporter?
+      self.importerexporter_type == 'Bulkrax::Exporter'
+    end
+
+    def valid_system_id(model_class)
+      return true if model_class.properties.keys.include?(work_identifier)
+      raise(
+        "#{model_class} does not implement the system_identifier_field: #{work_identifier}"
+      )
+    end
+
+    def last_run
+      self.importerexporter&.last_run
+    end
+
+    def find_collection(collection_identifier)
+      Collection.where(
+        work_identifier => collection_identifier
+      ).detect { |m| m.send(work_identifier).include?(collection_identifier) }
+    end
+  end
+end

data/app/models/bulkrax/exporter.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+module Bulkrax
+  class Exporter < ApplicationRecord
+    include Bulkrax::ImporterExporterBehavior
+    include Bulkrax::StatusInfo
+
+    serialize :parser_fields, JSON
+    serialize :field_mapping, JSON
+
+    belongs_to :user
+    has_many :exporter_runs, dependent: :destroy
+    has_many :entries, as: :importerexporter, dependent: :destroy
+
+    validates :name, presence: true
+    validates :parser_klass, presence: true
+
+    delegate :write, :create_from_collection, :create_from_importer, :create_from_worktype, to: :parser
+
+    def export
+      current_run && setup_export_path
+      case self.export_from
+      when 'collection'
+        create_from_collection
+      when 'importer'
+        create_from_importer
+      when 'worktype'
+        create_from_worktype
+      end
+    rescue StandardError => e
+      status_info(e)
+    end
+
+    # #export_source accessors
+    # Used in form to prevent it from getting confused as to which value to populate #export_source with.
+    # Also, used to display the correct selected value when rendering edit form.
+    def export_source_importer
+      self.export_source if self.export_from == 'importer'
+    end
+
+    def export_source_collection
+      self.export_source if self.export_from == 'collection'
+    end
+
+    def export_source_worktype
+      self.export_source if self.export_from == 'worktype'
+    end
+
+    def date_filter
+      self.start_date.present? || self.finish_date.present?
+    end
+
+    def work_visibility_list
+      [
+        ['Any', ''],
+        [I18n.t('hyrax.visibility.open.text'), Hydra::AccessControls::AccessRight::VISIBILITY_TEXT_VALUE_PUBLIC],
+        [I18n.t('hyrax.visibility.restricted.text'), Hydra::AccessControls::AccessRight::VISIBILITY_TEXT_VALUE_PRIVATE],
+        [I18n.t('hyrax.visibility.authenticated.text', institution: I18n.t('hyrax.institution_name')), Hydra::AccessControls::AccessRight::VISIBILITY_TEXT_VALUE_AUTHENTICATED]
+      ]
+    end
+
+    def workflow_status_list
+      Sipity::WorkflowState.all.map { |s| [s.name&.titleize, s.name] }.uniq
+    end
+
+    # If field_mapping is empty, setup a default based on the export_properties
+    def mapping
+      @mapping ||= self.field_mapping ||
+                   ActiveSupport::HashWithIndifferentAccess.new(
+                     export_properties.map do |m|
+                       Bulkrax.default_field_mapping.call(m)
+                     end.inject(:merge)
+                   ) ||
+                   [{}]
+    end
+
+    def export_from_list
+      [
+        [I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
+        [I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
+        [I18n.t('bulkrax.exporter.labels.worktype'), 'worktype']
+      ]
+    end
+
+    def export_type_list
+      [
+        [I18n.t('bulkrax.exporter.labels.metadata'), 'metadata'],
+        [I18n.t('bulkrax.exporter.labels.full'), 'full']
+      ]
+    end
+
+    def importers_list
+      Importer.all.map { |i| [i.name, i.id] }
+    end
+
+    def current_run
+      @current_run ||= self.exporter_runs.create!(total_work_entries: self.limit || parser.total)
+    end
+
+    def last_run
+      @last_run ||= self.exporter_runs.last
+    end
+
+    def setup_export_path
+      FileUtils.mkdir_p(exporter_export_path) unless File.exist?(exporter_export_path)
+    end
+
+    def exporter_export_path
+      @exporter_export_path ||= File.join(Bulkrax.export_path, self.id.to_s, self.exporter_runs.last.id.to_s)
+    end
+
+    def exporter_export_zip_path
+      @exporter_export_zip_path ||= File.join(Bulkrax.export_path, "export_#{self.id}_#{self.exporter_runs.last.id}.zip")
+    rescue
+      @exporter_export_zip_path ||= File.join(Bulkrax.export_path, "export_#{self.id}_0.zip")
+    end
+
+    def export_properties
+      properties = Hyrax.config.registered_curation_concern_types.map { |work| work.constantize.properties.keys }.flatten.uniq.sort
+      properties.reject { |prop| Bulkrax.reserved_properties.include?(prop) }
+    end
+  end
+end

data/app/models/bulkrax/exporter_run.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ExporterRun < ApplicationRecord
+    belongs_to :exporter
+  end
+end