bulkrax 1.0.0

data/app/jobs/bulkrax/application_job.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/bulkrax/child_relationships_job.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ChildWorksError < RuntimeError; end
+  class ChildRelationshipsJob < ApplicationJob
+    queue_as :import
+
+    def perform(*args)
+      @args = args
+
+      if entry.factory_class == Collection
+        collection_membership
+      else
+        work_membership
+      end
+      # Not all of the Works/Collections exist yet; reschedule
+    rescue Bulkrax::ChildWorksError
+      reschedule(args[0], args[1], args[2])
+    end
+
+    def collection_membership
+      # add collection to works
+      member_of_collection = []
+      child_works_hash.each { |k, v| member_of_collection << k if v[:class_name] != 'Collection' }
+      member_of_collection.each { |work| work_child_collection_parent(work) }
+
+      # add collections to collection
+      members_collections = []
+      child_works_hash.each { |k, v| members_collections << k if v[:class_name] == 'Collection' }
+      collection_parent_collection_child(members_collections) if members_collections.present?
+    end
+
+    def work_membership
+      # add works to work
+      # reject any Collections, they can't be children of Works
+      members_works = []
+      # reject any Collections, they can't be children of Works
+      child_works_hash.each { |k, v| members_works << k if v[:class_name] != 'Collection' }
+      if members_works.length < child_entries.length # rubocop:disable Style/IfUnlessModifier
+        Rails.logger.warn("Cannot add collections as children of works: #{(@child_entries.length - members_works.length)} collections were discarded for parent entry #{@entry.id} (of #{@child_entries.length})")
+      end
+      work_parent_work_child(members_works) if members_works.present?
+    end
+
+    def entry
+      @entry ||= Bulkrax::Entry.find(@args[0])
+    end
+
+    def child_entries
+      @child_entries ||= @args[1].map { |e| Bulkrax::Entry.find(e) }
+    end
+
+    def child_works_hash
+      @child_works_hash ||= child_entries.each_with_object({}) do |child_entry, hash|
+        work = child_entry.factory.find
+        # If we can't find the Work/Collection, raise a custom error
+        raise ChildWorksError if work.blank?
+        hash[work.id] = { class_name: work.class.to_s, entry.parser.source_identifier => child_entry.identifier }
+      end
+    end
+
+    def importer_run_id
+      @args[2]
+    end
+
+    def user
+      @user ||= entry.importerexporter.user
+    end
+
+    private
+
+    # rubocop:disable Rails/SkipsModelValidations
+    # Work-Collection membership is added to the child as member_of_collection_ids
+    # This is adding the reverse relatinship, from the child to the parent
+    def work_child_collection_parent(work_id)
+      attrs = { id: work_id, collections: [{ id: entry&.factory&.find&.id }] }
+      Bulkrax::ObjectFactory.new(attributes: attrs,
+                                 source_identifier_value: child_works_hash[work_id][entry.parser.source_identifier],
+                                 work_identifier: entry.parser.work_identifier,
+                                 replace_files: false,
+                                 user: user,
+                                 klass: child_works_hash[work_id][:class_name].constantize).run
+      ImporterRun.find(importer_run_id).increment!(:processed_children)
+    rescue StandardError => e
+      entry.status_info(e)
+      ImporterRun.find(importer_run_id).increment!(:failed_children)
+    end
+
+    # Collection-Collection membership is added to the as member_ids
+    def collection_parent_collection_child(member_ids)
+      attrs = { id: entry&.factory&.find&.id, children: member_ids }
+      Bulkrax::ObjectFactory.new(attributes: attrs,
+                                 source_identifier_value: entry.identifier,
+                                 work_identifier: entry.parser.work_identifier,
+                                 replace_files: false,
+                                 user: user,
+                                 klass: entry.factory_class).run
+      ImporterRun.find(importer_run_id).increment!(:processed_children)
+    rescue StandardError => e
+      entry.status_info(e)
+      ImporterRun.find(importer_run_id).increment!(:failed_children)
+    end
+
+    # Work-Work membership is added to the parent as member_ids
+    def work_parent_work_child(member_ids)
+      # build work_members_attributes
+      attrs = { id: entry&.factory&.find&.id,
+                work_members_attributes: member_ids.each.with_index.each_with_object({}) do |(member, index), ids|
+                  ids[index] = { id: member }
+                end }
+      Bulkrax::ObjectFactory.new(attributes: attrs,
+                                 source_identifier_value: entry.identifier,
+                                 work_identifier: entry.parser.work_identifier,
+                                 replace_files: false,
+                                 user: user,
+                                 klass: entry.factory_class).run
+      ImporterRun.find(importer_run_id).increment!(:processed_children)
+    rescue StandardError => e
+      entry.status_info(e)
+      ImporterRun.find(importer_run_id).increment!(:failed_children)
+    end
+    # rubocop:enable Rails/SkipsModelValidations
+
+    def reschedule(entry_id, child_entry_ids, importer_run_id)
+      ChildRelationshipsJob.set(wait: 10.minutes).perform_later(entry_id, child_entry_ids, importer_run_id)
+    end
+  end
+end

data/app/jobs/bulkrax/delete_work_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteWorkJob < ApplicationJob
+    queue_as :import
+
+    # rubocop:disable Rails/SkipsModelValidations
+    def perform(entry, importer_run)
+      work = entry.factory.find
+      work&.delete
+      importer_run.increment!(:deleted_records)
+      importer_run.decrement!(:enqueued_records)
+    end
+    # rubocop:enable Rails/SkipsModelValidations
+  end
+end

data/app/jobs/bulkrax/download_cloud_file_job.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DownloadCloudFileJob < ApplicationJob
+    queue_as :import
+
+    # Retrieve cloud file and write to the imports directory
+    # Note: if using the file system, the mounted directory in
+    #   browse_everything MUST be shared by web and worker servers
+    def perform(file, target_file)
+      retriever = BrowseEverything::Retriever.new
+      retriever.download(file, target_file) do |filename, retrieved, total|
+        # The block is still useful for showing progress, but the
+        # first argument is the filename instead of a chunk of data.
+      end
+    end
+  end
+end

data/app/jobs/bulkrax/export_work_job.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ExportWorkJob < ApplicationJob
+    queue_as :export
+
+    def perform(*args)
+      entry = Entry.find(args[0])
+      begin
+        entry.build
+        entry.save
+      rescue StandardError
+        # rubocop:disable Rails/SkipsModelValidations
+        ExporterRun.find(args[1]).increment!(:failed_records)
+        ExporterRun.find(args[1]).decrement!(:enqueued_records)
+        raise
+      else
+        if entry.failed?
+          ExporterRun.find(args[1]).increment!(:failed_records)
+          ExporterRun.find(args[1]).decrement!(:enqueued_records)
+          raise entry.reload.current_status.error_class.constantize
+        else
+          ExporterRun.find(args[1]).increment!(:processed_records)
+          ExporterRun.find(args[1]).decrement!(:enqueued_records)
+        end
+        # rubocop:enable Rails/SkipsModelValidations
+      end
+      exporter_run = ExporterRun.find(args[1])
+      return if exporter_run.enqueued_records.positive?
+      if exporter_run.failed_records.positive?
+        exporter_run.exporter.status_info('Complete (with failures)')
+      else
+        exporter_run.exporter.status_info('Complete')
+      end
+    end
+  end
+end

data/app/jobs/bulkrax/exporter_job.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ExporterJob < ApplicationJob
+    queue_as :export
+
+    def perform(exporter_id)
+      exporter = Exporter.find(exporter_id)
+      exporter.export
+      exporter.write
+      true
+    end
+  end
+end

data/app/jobs/bulkrax/import_work_collection_job.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ImportWorkCollectionJob < ApplicationJob
+    queue_as :import
+
+    # rubocop:disable Rails/SkipsModelValidations
+    def perform(*args)
+      entry = Entry.find(args[0])
+      begin
+        entry.build
+        entry.save
+        add_user_to_permission_template!(entry)
+        ImporterRun.find(args[1]).increment!(:processed_collections)
+        ImporterRun.find(args[1]).decrement!(:enqueued_records)
+      rescue => e
+        ImporterRun.find(args[1]).increment!(:failed_collections)
+        ImporterRun.find(args[1]).decrement!(:enqueued_records)
+        raise e
+      end
+    end
+    # rubocop:enable Rails/SkipsModelValidations
+
+    private
+
+    def add_user_to_permission_template!(entry)
+      user                = ::User.find(entry.importerexporter.user_id)
+      collection          = entry.factory.find
+      permission_template = Hyrax::PermissionTemplate.find_or_create_by!(source_id: collection.id)
+
+      Hyrax::PermissionTemplateAccess.create!(
+        permission_template_id: permission_template.id,
+        agent_id: user.user_key,
+        agent_type: 'user',
+        access: 'manage'
+      )
+
+      collection.reset_access_controls!
+    end
+  end
+end

data/app/jobs/bulkrax/import_work_job.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ImportWorkJob < ApplicationJob
+    queue_as :import
+
+    # rubocop:disable Rails/SkipsModelValidations
+    def perform(*args)
+      entry = Entry.find(args[0])
+      entry.build
+      if entry.status == "Complete"
+        ImporterRun.find(args[1]).increment!(:processed_records)
+        ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
+      else
+        # do not retry here because whatever parse error kept you from creating a work will likely
+        # keep preventing you from doing so.
+        ImporterRun.find(args[1]).increment!(:failed_records)
+        ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
+      end
+      entry.save!
+      entry.importer.current_run = ImporterRun.find(args[1])
+      entry.importer.record_status
+    rescue Bulkrax::CollectionsCreatedError
+      reschedule(args[0], args[1])
+    end
+    # rubocop:enable Rails/SkipsModelValidations
+
+    def reschedule(entry_id, run_id)
+      ImportWorkJob.set(wait: 1.minute).perform_later(entry_id, run_id)
+    end
+  end
+end

data/app/jobs/bulkrax/importer_job.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ImporterJob < ApplicationJob
+    queue_as :import
+
+    def perform(importer_id, only_updates_since_last_import = false)
+      importer = Importer.find(importer_id)
+      importer.current_run
+      import(importer, only_updates_since_last_import)
+      schedule(importer) if importer.schedulable?
+    end
+
+    def import(importer, only_updates_since_last_import)
+      importer.only_updates = only_updates_since_last_import || false
+      return unless importer.valid_import?
+      importer.import_collections
+      importer.import_works
+      importer.create_parent_child_relationships unless importer.validate_only
+    end
+
+    def schedule(importer)
+      ImporterJob.set(wait_until: importer.next_import_at).perform_later(importer.id, true)
+    end
+  end
+end

data/app/mailers/bulkrax/application_mailer.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ApplicationMailer < ActionMailer::Base
+    default from: 'from@example.com'
+    layout 'mailer'
+  end
+end

data/app/matchers/bulkrax/application_matcher.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'language_list'
+
+module Bulkrax
+  class ApplicationMatcher
+    attr_accessor :to, :from, :parsed, :if, :split, :excluded, :nested_type
+
+    def initialize(args)
+      args.each do |k, v|
+        send("#{k}=", v)
+      end
+    end
+
+    def result(_parser, content)
+      return nil if self.excluded == true || Bulkrax.reserved_properties.include?(self.to)
+      return nil if self.if && (!self.if.is_a?(Array) && self.if.length != 2)
+
+      if self.if
+        return unless content.send(self.if[0], Regexp.new(self.if[1]))
+      end
+
+      @result = content.to_s.gsub(/\s/, ' ') # remove any line feeds and tabs
+      @result.strip!
+      process_split
+      @result = @result[0] if @result.is_a?(Array) && @result.size == 1
+      process_parse
+      return @result
+    end
+
+    def process_split
+      if self.split.is_a?(TrueClass)
+        @result = @result.split(/\s*[:;|]\s*/) # default split by : ; |
+      elsif self.split
+        result = @result.split(Regexp.new(self.split))
+        @result = result.map(&:strip)
+      end
+    end
+
+    def process_parse
+      # New parse methods will need to be added here
+      parsed_fields = ['remote_files', 'language', 'subject', 'types', 'model', 'resource_type', 'format_original']
+      # This accounts for prefixed matchers
+      parser = parsed_fields.find { |field| to&.include? field }
+
+      if @result.is_a?(Array) && self.parsed && self.respond_to?("parse_#{parser}")
+        @result.each_with_index do |res, index|
+          @result[index] = send("parse_#{parser}", res.strip)
+        end
+        @result.delete(nil)
+      elsif self.parsed && self.respond_to?("parse_#{parser}")
+        @result = send("parse_#{parser}", @result)
+      end
+    end
+
+    def parse_remote_files(src)
+      { url: src.strip } if src.present?
+    end
+
+    def parse_language(src)
+      l = ::LanguageList::LanguageInfo.find(src.strip)
+      l ? l.name : src
+    end
+
+    def parse_subject(src)
+      string = src.to_s.strip.downcase
+      return if string.blank?
+
+      string.slice(0, 1).capitalize + string.slice(1..-1)
+    end
+
+    def parse_types(src)
+      src.to_s.strip.titleize
+    end
+
+    # Allow for mapping a model field to the work type or collection
+    def parse_model(src)
+      model = nil
+      if src.is_a?(Array)
+        models = src.map { |m| extract_model(m) }.compact
+        model = models.first if models.present?
+      else
+        model = extract_model(src)
+      end
+      return model
+    end
+
+    def extract_model(src)
+      if src&.match(URI::ABS_URI)
+        src.split('/').last
+      else
+        src
+      end
+    rescue StandardError
+      nil
+    end
+
+    # Only add valid resource types
+    def parse_resource_type(src)
+      Hyrax::ResourceTypesService.label(src.to_s.strip.titleize)
+    rescue KeyError
+      nil
+    end
+
+    def parse_format_original(src)
+      # drop the case completely then upcase the first letter
+      string = src.to_s.strip.downcase
+      return if string.blank?
+
+      string.slice(0, 1).capitalize + string.slice(1..-1)
+    end
+  end
+end