bulkrax 1.0.0

data/app/models/bulkrax/import_failed.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# this is a PORO to help pass errors around
+module Bulkrax
+  class ImportFailed
+    attr_accessor :message, :backtrace
+
+    def initialize(message, backtrace)
+      @message = message
+      @backtrace = backtrace
+    end
+  end
+end

data/app/models/bulkrax/importer.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require 'iso8601'
+
+module Bulkrax
+  class Importer < ApplicationRecord
+    include Bulkrax::ImporterExporterBehavior
+    include Bulkrax::StatusInfo
+
+    serialize :parser_fields, JSON
+    serialize :field_mapping, JSON
+
+    belongs_to :user
+    has_many :importer_runs, dependent: :destroy
+    has_many :entries, as: :importerexporter, dependent: :destroy
+
+    validates :name, presence: true
+    validates :admin_set_id, presence: true
+    validates :parser_klass, presence: true
+
+    delegate :valid_import?, :create_parent_child_relationships,
+             :write_errored_entries_file, :visibility, to: :parser
+
+    attr_accessor :only_updates, :file_style, :file
+    attr_writer :current_run
+
+    def status
+      if self.validate_only
+        'Validated'
+      else
+        super
+      end
+    end
+
+    def record_status
+      importer_run = ImporterRun.find(current_run.id) # make sure fresh
+      return if importer_run.enqueued_records.positive? # still processing
+      if importer_run.failed_records.positive?
+        if importer_run.invalid_records.present?
+          e = Bulkrax::ImportFailed.new('Failed with Invalid Records', importer_run.invalid_records.split("\n"))
+          importer_run.importer.status_info(e)
+        else
+          importer_run.importer.status_info('Complete (with failures)')
+        end
+      else
+        importer_run.importer.status_info('Complete')
+      end
+    end
+
+    # If field_mapping is empty, setup a default based on the export_properties
+    def mapping
+      @mapping ||= if self.field_mapping.blank? || self.field_mapping == [{}]
+                     if parser.import_fields.present? || self.field_mapping == [{}]
+                       ActiveSupport::HashWithIndifferentAccess.new(
+                         parser.import_fields.reject(&:nil?).map do |m|
+                           Bulkrax.default_field_mapping.call(m)
+                         end.inject(:merge)
+                       )
+                     end
+                   else
+                     self.field_mapping
+                   end
+    end
+
+    def parser_fields
+      self[:parser_fields] || {}
+    end
+
+    def self.frequency_enums
+      # these duration values use ISO 8601 Durations (https://en.wikipedia.org/wiki/ISO_8601#Durations)
+      # TLDR; all durations are prefixed with 'P' and the parts are a number with the type of duration.
+      # i.e. P1Y2M3W4DT5H6M7S == 1 Year, 2 Months, 3 Weeks, 4 Days, 5 Hours, 6 Minutes, 7 Seconds
+      [['Daily', 'P1D'], ['Monthly', 'P1M'], ['Yearly', 'P1Y'], ['Once (on save)', 'PT0S']]
+    end
+
+    def frequency=(frequency)
+      self[:frequency] = ISO8601::Duration.new(frequency).to_s
+    end
+
+    def frequency
+      f = self[:frequency] || "PT0S"
+      ISO8601::Duration.new(f)
+    end
+
+    def schedulable?
+      frequency.to_seconds != 0
+    end
+
+    def current_run
+      @current_run ||= self.importer_runs.create!(total_work_entries: self.limit || parser.total, total_collection_entries: parser.collections_total)
+    end
+
+    def last_run
+      @last_run ||= self.importer_runs.last
+    end
+
+    def seen
+      @seen ||= {}
+    end
+
+    def replace_files
+      self.parser_fields['replace_files']
+    end
+
+    def update_files
+      self.parser_fields['update_files']
+    end
+
+    def import_works
+      self.save if self.new_record? # Object needs to be saved for statuses
+      self.only_updates ||= false
+      parser.create_works
+    rescue StandardError => e
+      status_info(e)
+    end
+
+    def import_collections
+      self.save if self.new_record? # Object needs to be saved for statuses
+      parser.create_collections
+    rescue StandardError => e
+      status_info(e)
+    end
+
+    # Prepend the base_url to ensure unique set identifiers
+    # @todo - move to parser, as this is OAI specific
+    def unique_collection_identifier(id)
+      "#{self.parser_fields['base_url'].split('/')[2]}_#{id}"
+    end
+
+    # The format for metadata for the incoming import; corresponds to an Entry class
+    def import_metadata_format
+      [['CSV', 'Bulkrax::CsvEntry'], ['RDF (N-Triples)', 'Bulkrax::RdfEntry']]
+    end
+
+    # The type of metadata for the incoming import, either one file for all works, or one file per work
+    # def import_metadata_type
+    #   [['Single Metadata File for all works', 'single'], ['Multiple Files, one per Work', 'multi']]
+    # end
+
+    # If the import data is zipped, unzip it to this path
+    def importer_unzip_path
+      @importer_unzip_path ||= File.join(Bulkrax.import_path, "import_#{path_string}")
+    end
+
+    def errored_entries_csv_path
+      @errored_entries_csv_path ||= File.join(Bulkrax.import_path, "import_#{path_string}_errored_entries.csv")
+    end
+
+    def path_string
+      "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}_#{self.importer_runs.last.id}"
+    rescue
+      "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
+    end
+  end
+end

data/app/models/bulkrax/importer_run.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ImporterRun < ApplicationRecord
+    belongs_to :importer
+    has_many :statuses, as: :runnable, dependent: :destroy
+  end
+end

data/app/models/bulkrax/oai_dc_entry.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class OaiDcEntry < OaiEntry
+  end
+end

data/app/models/bulkrax/oai_entry.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'ostruct'
+
+module Bulkrax
+  class OaiEntry < Entry
+    serialize :raw_metadata, JSON
+
+    delegate :record, to: :raw_record
+
+    def raw_record
+      @raw_record ||= client.get_record(identifier: identifier, metadata_prefix: parser.parser_fields['metadata_prefix'])
+    end
+
+    def sets
+      record.header.set_spec
+    end
+
+    def context
+      @context ||= OpenStruct.new(record: record, identifier: record.header.identifier)
+    end
+
+    def thumbnail_url
+      ERB.new(parser.parser_fields['thumbnail_url']).result(context.instance_eval { binding })
+    end
+
+    def build_metadata
+      self.parsed_metadata = {}
+      self.parsed_metadata[work_identifier] = [record.header.identifier]
+
+      record.metadata.children.each do |child|
+        child.children.each do |node|
+          add_metadata(node.name, node.content)
+        end
+      end
+      add_metadata('thumbnail_url', thumbnail_url)
+
+      add_visibility
+      add_rights_statement
+      add_admin_set_id
+      add_collections
+      add_local
+
+      return self.parsed_metadata
+    end
+
+    def collections_created?
+      if parser.collection_name == 'all'
+        sets.blank? || (sets.present? && sets.size == self.collection_ids.size)
+      else
+        self.collection_ids.size == 1
+      end
+    end
+
+    # Retrieve list of collections for the entry; add to collection_ids
+    # If OAI-PMH doesn't return setSpec in the headers for GetRecord, use parser.collection_name
+    #   in this case, if 'All' is selected, records will not be added to a collection.
+    def find_or_create_collection_ids
+      return self.collection_ids if collections_created?
+      if sets.blank? || parser.collection_name != 'all'
+        # c = Collection.where(Bulkrax.system_identifier_field => importerexporter.unique_collection_identifier(parser.collection_name)).first
+        collection = find_collection(importerexporter.unique_collection_identifier(parser.collection_name))
+        self.collection_ids << collection.id if collection.present? && !self.collection_ids.include?(collection.id)
+      else # All - collections should exist for all sets
+        sets.each do |set|
+          c = Collection.find_by(work_identifier => importerexporter.unique_collection_identifier(set.content))
+          self.collection_ids << c.id if c.present? && !self.collection_ids.include?(c.id)
+        end
+      end
+      return self.collection_ids
+    end
+  end
+end

data/app/models/bulkrax/oai_qualified_dc_entry.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class OaiQualifiedDcEntry < OaiEntry
+  end
+end

data/app/models/bulkrax/oai_set_entry.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class OaiSetEntry < OaiEntry
+    def factory_class
+      Collection
+    end
+
+    def build_metadata
+      self.parsed_metadata = self.raw_metadata
+      add_local
+      return self.parsed_metadata
+    end
+
+    def collections_created?
+      true
+    end
+  end
+end

data/app/models/bulkrax/rdf_collection_entry.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class RdfCollectionEntry < RdfEntry
+    def record
+      @record ||= self.raw_metadata
+    end
+
+    def build_metadata
+      self.parsed_metadata = self.raw_metadata
+      add_local
+      return self.parsed_metadata
+    end
+
+    def factory_class
+      Collection
+    end
+  end
+end

data/app/models/bulkrax/rdf_entry.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'rdf'
+module Bulkrax
+  class RdfEntry < Entry
+    serialize :raw_metadata, JSON
+
+    def self.read_data(path)
+      RDF::Reader.open(path)
+    end
+
+    def self.fields_from_data(data)
+      data.predicates.map(&:to_s)
+    end
+
+    def self.data_for_entry(data, source_id)
+      reader = data
+      format = reader.class.format.to_sym
+      collections = []
+      children = []
+      delete = nil
+      data = RDF::Writer.for(format).buffer do |writer|
+        reader.each_statement do |statement|
+          collections << statement.object.to_s if collection_field.present? && collection_field == statement.predicate.to_s
+          children << statement.object.to_s if children_field.present? && children_field == statement.predicate.to_s
+          delete = statement.object.to_s if /deleted/.match?(statement.predicate.to_s)
+          writer << statement
+        end
+      end
+      return {
+        source_id => reader.subjects.first.to_s,
+        delete: delete,
+        format: format,
+        data: data,
+        collection: collections,
+        children: children
+      }
+    end
+
+    def self.collection_field
+      Bulkrax.collection_field_mapping[self.to_s]
+    end
+
+    def self.children_field
+      Bulkrax.parent_child_field_mapping[self.to_s]
+    end
+
+    def record
+      @record ||= RDF::Reader.for(self.raw_metadata['format'].to_sym).new(self.raw_metadata['data'])
+    end
+
+    def build_metadata
+      raise StandardError, 'Record not found' if record.nil?
+      raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
+
+      self.parsed_metadata = {}
+      self.parsed_metadata[work_identifier] = [self.raw_metadata[source_identifier]]
+
+      record.each_statement do |statement|
+        # Only process the subject for our record (in case other data is in the file)
+        next unless statement.subject.to_s == self.raw_metadata[source_identifier]
+        add_metadata(statement.predicate.to_s, statement.object.to_s)
+      end
+      add_visibility
+      add_rights_statement
+      add_admin_set_id
+      add_collections
+      add_local
+      self.parsed_metadata['file'] = self.raw_metadata['file']
+
+      self.parsed_metadata
+    end
+
+    def collections_created?
+      return true if self.raw_metadata['collection'].blank?
+      self.raw_metadata['collection'].length == self.collection_ids.length
+    end
+
+    def find_or_create_collection_ids
+      return self.collection_ids if collections_created?
+      if self.raw_metadata['collection'].present?
+        self.raw_metadata['collection'].each do |collection|
+          c = find_collection(collection)
+          self.collection_ids << c.id unless c.blank? || self.collection_ids.include?(c.id)
+        end
+      end
+      return self.collection_ids
+    end
+  end
+end

data/app/models/bulkrax/status.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class Status < ApplicationRecord
+    belongs_to :statusable, polymorphic: true
+    belongs_to :runnable, polymorphic: true
+    serialize :error_backtrace, Array
+
+    scope :for_importers, -> { where(statusable_type: 'Bulkrax::Importer') }
+    scope :for_exporters, -> { where(statusable_type: 'Bulkrax::Exporter') }
+
+    scope :latest_by_statusable, -> { joins(latest_by_statusable_subtable.join_sources) }
+
+    def self.latest_by_statusable_subtable
+      status_table = self.arel_table
+      latest_status_query = status_table.project(status_table[:statusable_id],
+                                                 status_table[:statusable_type],
+                                                 status_table[:id].maximum.as("latest_status_id")).group(status_table[:statusable_id], status_table[:statusable_type])
+
+      latest_status_table = Arel::Table.new(latest_status_query).alias(:latest_status)
+      status_table.join(latest_status_query.as(latest_status_table.name.to_s), Arel::Nodes::InnerJoin)
+                  .on(status_table[:id].eq(latest_status_table[:latest_status_id]))
+    end
+  end
+end

data/app/models/bulkrax/xml_entry.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+module Bulkrax
+  # Generic XML Entry
+  class XmlEntry < Entry
+    serialize :raw_metadata, JSON
+
+    def self.fields_from_data(data); end
+
+    def self.read_data(path)
+      # This doesn't cope with BOM sequences:
+      # Nokogiri::XML(open(path), nil, 'UTF-8').remove_namespaces!
+      Nokogiri::XML(open(path)).remove_namespaces!
+    end
+
+    def self.data_for_entry(data, source_id)
+      collections = []
+      children = []
+      xpath_for_source_id = ".//*[name()='#{source_id}']"
+      return {
+        source_id => data.xpath(xpath_for_source_id).first.text,
+        delete: data.xpath(".//*[name()='delete']").first&.text,
+        data:
+          data.to_xml(
+            encoding: 'UTF-8',
+            save_with:
+              Nokogiri::XML::Node::SaveOptions::NO_DECLARATION | Nokogiri::XML::Node::SaveOptions::NO_EMPTY_TAGS
+          ).delete("\n").delete("\t").squeeze(' '), # Remove newlines, tabs, and extra whitespace
+        collection: collections,
+        children: children
+      }
+    end
+
+    # def self.matcher_class; end
+
+    def record
+      @record ||= Nokogiri::XML(self.raw_metadata['data'], nil, 'UTF-8')
+    end
+
+    def build_metadata
+      raise StandardError, 'Record not found' if record.nil?
+      raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
+      self.parsed_metadata = {}
+      self.parsed_metadata[work_identifier] = [self.raw_metadata[source_identifier]]
+      xml_elements.each do |element_name|
+        elements = record.xpath("//*[name()='#{element_name}']")
+        next if elements.blank?
+        elements.each do |el|
+          el.children.map(&:content).each do |content|
+            add_metadata(element_name, content) if content.present?
+          end
+        end
+      end
+      add_visibility
+      add_rights_statement
+      add_admin_set_id
+      add_collections
+      self.parsed_metadata['file'] = self.raw_metadata['file']
+
+      add_local
+      raise StandardError, "title is required" if self.parsed_metadata['title'].blank?
+      self.parsed_metadata
+    end
+
+    # Grab the class from the real parser
+    def xml_elements
+      Bulkrax.field_mappings[self.importerexporter.parser_klass].map do |_k, v|
+        v[:from]
+      end.flatten.compact.uniq
+    end
+  end
+end