bulkrax 1.0.0

data/db/migrate/20181011230228_create_bulkrax_importer_runs.rb

@@ -0,0 +1,16 @@
+class CreateBulkraxImporterRuns < ActiveRecord::Migration[5.1]
+  def change
+    unless table_exists?(:bulkrax_importer_runs)
+      create_table :bulkrax_importer_runs do |t|
+        t.references :importer, foreign_key: {to_table: :bulkrax_importers}
+        t.integer :total_records, default: 0
+        t.integer :enqueued_records, default: 0
+        t.integer :processed_records, default: 0
+        t.integer :deleted_records, default: 0
+        t.integer :failed_records, default: 0
+
+        t.timestamps
+      end
+    end
+  end
+end

data/db/migrate/20190325183136_create_bulkrax_entries.rb

@@ -0,0 +1,16 @@
+class CreateBulkraxEntries < ActiveRecord::Migration[5.1]
+  def change
+    unless table_exists?(:bulkrax_entries)
+      create_table :bulkrax_entries do |t|
+        t.string :identifier
+        t.string :collection_id
+        t.string :type
+        t.references :importer, foreign_key: {to_table: :bulkrax_importers}
+        t.text :raw_metadata
+        t.text :parsed_metadata
+
+        t.timestamps
+      end
+    end
+  end
+end

data/db/migrate/20190601221109_add_status_to_entry.rb

@@ -0,0 +1,9 @@
+class AddStatusToEntry < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_entries, :last_error, :text unless column_exists?(:bulkrax_entries, :last_error)
+    add_column :bulkrax_entries, :last_error_at, :datetime unless column_exists?(:bulkrax_entries, :last_error_at)
+
+    add_column :bulkrax_entries, :last_succeeded_at, :datetime unless column_exists?(:bulkrax_entries, :last_succeeded_at)
+
+  end
+end

data/db/migrate/20190715161939_add_collections_to_importer_runs.rb

@@ -0,0 +1,6 @@
+class AddCollectionsToImporterRuns < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_importer_runs, :processed_collections, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_collections)
+    add_column :bulkrax_importer_runs, :failed_collections, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_collections)
+  end
+end

data/db/migrate/20190715162044_change_collection_ids_on_entries.rb

@@ -0,0 +1,5 @@
+class ChangeCollectionIdsOnEntries < ActiveRecord::Migration[5.1]
+  def change
+    rename_column :bulkrax_entries, :collection_id, :collection_ids if column_exists?(:bulkrax_entries, :collection_id)
+  end
+end

data/db/migrate/20190729124607_create_bulkrax_exporters.rb

@@ -0,0 +1,19 @@
+class CreateBulkraxExporters < ActiveRecord::Migration[5.1]
+  def change
+    unless table_exists?(:bulkrax_exporters)
+      create_table :bulkrax_exporters do |t|
+        t.string :name
+        t.references :user, foreign_key: false
+        t.string :parser_klass
+        t.integer :limit
+        t.text :parser_fields
+        t.text :field_mapping
+        t.string :export_source
+        t.string :export_from
+        t.string :export_type
+
+        t.timestamps
+      end
+    end
+  end
+end

data/db/migrate/20190729134158_create_bulkrax_exporter_runs.rb

@@ -0,0 +1,14 @@
+class CreateBulkraxExporterRuns < ActiveRecord::Migration[5.1]
+  def change
+    unless table_exists?(:bulkrax_exporter_runs)
+      create_table :bulkrax_exporter_runs do |t|
+        t.references :exporter, foreign_key: { to_table: :bulkrax_exporters }
+        t.integer :total_records, default: 0
+        t.integer :enqueued_records, default: 0
+        t.integer :processed_records, default: 0
+        t.integer :deleted_records, default: 0
+        t.integer :failed_records, default: 0
+      end
+    end
+  end
+end

data/db/migrate/20190731114016_change_importer_and_exporter_to_polymorphic.rb

@@ -0,0 +1,12 @@
+module Bulkrax
+  class Entry < ApplicationRecord
+     belongs_to :importer
+  end
+end
+
+class ChangeImporterAndExporterToPolymorphic < ActiveRecord::Migration[5.1]
+  def change
+    rename_column :bulkrax_entries, :importer_id, :importerexporter_id if column_exists?(:bulkrax_entries, :importer_id)
+    add_column :bulkrax_entries, :importerexporter_type, :string, after: :id, default: 'Bulkrax::Importer' unless column_exists?(:bulkrax_entries, :importerexporter_type)
+  end
+end

data/db/migrate/20191203225129_add_total_collection_records_to_importer_runs.rb

@@ -0,0 +1,5 @@
+class AddTotalCollectionRecordsToImporterRuns < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_importer_runs, :total_collection_entries, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :total_collection_entries)
+  end
+end

data/db/migrate/20191204191623_add_children_to_importer_runs.rb

@@ -0,0 +1,6 @@
+class AddChildrenToImporterRuns < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_importer_runs, :processed_children, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_children)
+    add_column :bulkrax_importer_runs, :failed_children, :integer, default: 0  unless column_exists?(:bulkrax_importer_runs, :failed_children)
+  end
+end

data/db/migrate/20191204223857_change_total_records_to_total_work_entries.rb

@@ -0,0 +1,6 @@
+class ChangeTotalRecordsToTotalWorkEntries < ActiveRecord::Migration[5.1]
+  def change
+    rename_column :bulkrax_importer_runs, :total_records, :total_work_entries if column_exists?(:bulkrax_importer_runs, :total_records)
+    rename_column :bulkrax_exporter_runs, :total_records, :total_work_entries if column_exists?(:bulkrax_exporter_runs, :total_records)
+  end
+end

data/db/migrate/20191212155530_change_entry_last_error.rb

@@ -0,0 +1,19 @@
+class ChangeEntryLastError < ActiveRecord::Migration[5.1]
+
+  def up
+    # Use raw query to change text to JSON as last_error field is now serialized
+    results = ActiveRecord::Base.connection.execute("SELECT id, last_error from bulkrax_entries WHERE last_error IS NOT null AND last_error LIKE '%\n\n%'")
+    results.each do | error |
+      old_errors = error['last_error'].gsub("'","''").split("\n\n")
+      new_error = {
+        'error_class' => 'unknown', 
+        'error_message' => old_errors.first,
+        'error_trace' => old_errors.last
+      }
+      ActiveRecord::Base.connection.execute("UPDATE bulkrax_entries SET last_error = '#{new_error.to_json}' WHERE id = '#{error['id']}'")
+    end
+  end
+
+  def down; end
+
+end

data/db/migrate/20200108194557_add_validate_only_to_bulkrax_importers.rb

@@ -0,0 +1,5 @@
+class AddValidateOnlyToBulkraxImporters < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_importers, :validate_only, :boolean unless column_exists?(:bulkrax_importers, :validate_only)
+  end
+end

data/db/migrate/20200301232856_add_status_to_importers.rb

@@ -0,0 +1,9 @@
+class AddStatusToImporters < ActiveRecord::Migration[5.1]
+  def change
+    if table_exists?(:bulkrax_importers)
+      add_column :bulkrax_importers, :last_error, :text unless column_exists?(:bulkrax_importers, :last_error)
+      add_column :bulkrax_importers, :last_error_at, :datetime unless column_exists?(:bulkrax_importers, :last_error_at)
+      add_column :bulkrax_importers, :last_succeeded_at, :datetime unless column_exists?(:bulkrax_importers, :last_succeeded_at)
+    end
+  end
+end

data/db/migrate/20200312190638_remove_foreign_key_from_bulkrax_entries.rb

@@ -0,0 +1,5 @@
+class RemoveForeignKeyFromBulkraxEntries < ActiveRecord::Migration[5.1]
+  def change
+    remove_foreign_key :bulkrax_entries, :bulkrax_importers if foreign_key_exists?(:bulkrax_entries, :bulkrax_importers)
+  end
+end

data/db/migrate/20200326235838_add_status_to_exporters.rb

@@ -0,0 +1,7 @@
+class AddStatusToExporters < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_exporters, :last_error, :text unless column_exists?(:bulkrax_exporters, :last_error)
+    add_column :bulkrax_exporters, :last_error_at, :datetime unless column_exists?(:bulkrax_exporters, :last_error_at)
+    add_column :bulkrax_exporters, :last_succeeded_at, :datetime unless column_exists?(:bulkrax_exporters, :last_succeeded_at)
+  end
+end

data/db/migrate/20200601204556_add_invalid_record_to_importer_run.rb

@@ -0,0 +1,5 @@
+class AddInvalidRecordToImporterRun < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_importer_runs, :invalid_records, :text unless column_exists?(:bulkrax_importer_runs, :invalid_records)
+  end
+end

data/db/migrate/20200818055819_create_bulkrax_statuses.rb

@@ -0,0 +1,18 @@
+class CreateBulkraxStatuses < ActiveRecord::Migration[5.1]
+  def change
+    unless table_exists?(:bulkrax_statuses)
+      create_table :bulkrax_statuses do |t|
+        t.string :status_message
+        t.string :error_class
+        t.string :error_message
+        t.text :error_backtrace
+        t.integer :statusable_id
+        t.string :statusable_type
+        t.integer :runnable_id
+        t.string :runnable_type
+
+        t.timestamps
+      end
+    end
+  end
+end

data/db/migrate/20200819054016_move_to_statuses.rb

@@ -0,0 +1,30 @@
+class MoveToStatuses < ActiveRecord::Migration[5.1]
+  def change
+    Bulkrax::Importer.find_each do |i|
+      add_status(i)
+    end
+
+    Bulkrax::Exporter.find_each do |i|
+      add_status(i)
+    end
+
+    Bulkrax::Entry.find_each do |i|
+      add_status(i)
+    end
+  end
+
+  def add_status(i)
+    return if i.statuses.present?
+    if i.last_error
+      i.statuses.create(
+        status_message: 'Failed',
+        runnable: i.last_run,
+        error_class: i.last_error['error_class'],
+        error_message: i.last_error['error_message'],
+        error_backtrace: i.last_error['error_trace']
+      )
+    else
+      i.statuses.create(status_message: 'Complete', runnable: i.last_run)
+    end
+  end
+end

data/db/migrate/20201106014204_add_date_filter_and_status_to_bulkrax_exporters.rb

@@ -0,0 +1,7 @@
+class AddDateFilterAndStatusToBulkraxExporters < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_exporters, :start_date, :date unless column_exists?(:bulkrax_exporters, :start_date)
+    add_column :bulkrax_exporters, :finish_date, :date unless column_exists?(:bulkrax_exporters, :finish_date)
+    add_column :bulkrax_exporters, :work_visibility, :string unless column_exists?(:bulkrax_exporters, :work_visibility)
+  end
+end

data/db/migrate/20201117220007_add_workflow_status_to_bulkrax_exporter.rb

@@ -0,0 +1,5 @@
+class AddWorkflowStatusToBulkraxExporter < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_exporters, :workflow_status, :string unless column_exists?(:bulkrax_exporters, :workflow_status)
+  end
+end

data/db/migrate/20210806044408_remove_unused_last_error.rb

@@ -0,0 +1,7 @@
+class RemoveUnusedLastError < ActiveRecord::Migration[5.1]
+  def change
+    remove_column :bulkrax_entries, :last_error
+    remove_column :bulkrax_exporters, :last_error
+    remove_column :bulkrax_importers, :last_error
+  end
+end

data/db/migrate/20210806065737_increase_text_sizes.rb

@@ -0,0 +1,12 @@
+class IncreaseTextSizes < ActiveRecord::Migration[5.1]
+  def change
+    change_column :bulkrax_entries, :raw_metadata, :text, limit: 16777215
+    change_column :bulkrax_entries, :parsed_metadata, :text, limit: 16777215
+    change_column :bulkrax_exporters, :parser_fields, :text, limit: 16777215
+    change_column :bulkrax_exporters, :field_mapping, :text, limit: 16777215
+    change_column :bulkrax_importers, :parser_fields, :text, limit: 16777215
+    change_column :bulkrax_importers, :field_mapping, :text, limit: 16777215
+    change_column :bulkrax_importer_runs, :invalid_records, :text, limit: 16777215
+    change_column :bulkrax_statuses, :error_backtrace, :text, limit: 16777215
+  end
+end

data/lib/bulkrax.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "bulkrax/engine"
+require 'active_support/all'
+
+module Bulkrax
+  class << self
+    mattr_accessor :parsers,
+                   :default_work_type,
+                   :default_field_mapping,
+                   :collection_field_mapping,
+                   :fill_in_blank_source_identifiers,
+                   :parent_child_field_mapping,
+                   :reserved_properties,
+                   :field_mappings,
+                   :import_path,
+                   :export_path,
+                   :removed_image_path,
+                   :server_name,
+                   :api_definition,
+                   :removed_image_path
+
+    self.parsers = [
+      { name: "OAI - Dublin Core", class_name: "Bulkrax::OaiDcParser", partial: "oai_fields" },
+      { name: "OAI - Qualified Dublin Core", class_name: "Bulkrax::OaiQualifiedDcParser", partial: "oai_fields" },
+      { name: "CSV - Comma Separated Values", class_name: "Bulkrax::CsvParser", partial: "csv_fields" },
+      { name: "Bagit", class_name: "Bulkrax::BagitParser", partial: "bagit_fields" },
+      { name: "XML", class_name: "Bulkrax::XmlParser", partial: "xml_fields" }
+    ]
+
+    self.import_path = 'tmp/imports'
+    self.export_path = 'tmp/exports'
+    self.removed_image_path = Bulkrax::Engine.root.join('spec', 'fixtures', 'removed.png').to_s
+    self.server_name = 'bulkrax@example.com'
+
+    # @todo, merge parent_child_field_mapping and collection_field_mapping into field_mappings,
+    # or make them settable per import some other way.
+
+    # Field_mapping for establishing a parent-child relationship (FROM parent TO child)
+    # This can be a Collection to Work, or Work to Work relationship
+    # This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
+    # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
+    # Example:
+    #   {
+    #     'Bulkrax::RdfEntry'  => 'http://opaquenamespace.org/ns/contents',
+    #     'Bulkrax::CsvEntry'  => 'children'
+    #   }
+    # By default no parent-child relationships are added
+    self.parent_child_field_mapping = {}
+
+    # Field_mapping for establishing a collection relationship (FROM work TO collection)
+    # This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
+    # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
+    # The default value for CSV is collection
+    self.collection_field_mapping = {}
+
+    # Hash of Generic field_mappings for use in the view
+    # There must be one field_mappings hash per view parial
+    # Based on Hyrax CoreMetadata && BasicMetadata
+    # Override at application level to change
+    self.field_mappings = {
+      "Bulkrax::OaiDcParser" => {
+        "contributor" => { from: ["contributor"] },
+        # no appropriate mapping for coverage (based_near needs id)
+        #  ""=>{:from=>["coverage"]},
+        "creator" => { from: ["creator"] },
+        "date_created" => { from: ["date"] },
+        "description" => { from: ["description"] },
+        # no appropriate mapping for format
+        # ""=>{:from=>["format"]},
+        "identifier" => { from: ["identifier"] },
+        "language" => { from: ["language"], parsed: true },
+        "publisher" => { from: ["publisher"] },
+        "related_url" => { from: ["relation"] },
+        "rights_statement" => { from: ["rights"] },
+        "source" => { from: ["source"] },
+        "subject" => { from: ["subject"], parsed: true },
+        "title" => { from: ["title"] },
+        "resource_type" => { from: ["type"], parsed: true },
+        "remote_files" => { from: ["thumbnail_url"], parsed: true }
+      },
+      "Bulkrax::OaiQualifiedDcParser" => {
+        "abstract" => { from: ["abstract"] },
+        "alternative_title" => { from: ["alternative"] },
+        "bibliographic_citation" => { from: ["bibliographicCitation"] },
+        "contributor" => { from: ["contributor"] },
+        "creator" => { from: ["creator"] },
+        "date_created" => { from: ["created"] },
+        "description" => { from: ["description"] },
+        "language" => { from: ["language"] },
+        "license" => { from: ["license"] },
+        "publisher" => { from: ["publisher"] },
+        "related_url" => { from: ["relation"] },
+        "rights_holder" => { from: ["rightsHolder"] },
+        "rights_statement" => { from: ["rights"] },
+        "source" => { from: ["source"] },
+        "subject" => { from: ["subject"], parsed: true },
+        "title" => { from: ["title"] },
+        "resource_type" => { from: ["type"], parsed: true },
+        "remote_files" => { from: ["thumbnail_url"], parsed: true }
+      },
+      # When empty, a default_field_mapping will be generated
+      "Bulkrax::CsvParser" => {},
+      'Bulkrax::BagitParser' => {},
+      'Bulkrax::XmlParser' => {}
+    }
+
+    # Lambda to set the default field mapping
+    self.default_field_mapping = lambda do |field|
+      return if field.blank?
+      {
+        field.to_s =>
+        {
+          from: [field.to_s],
+          split: false,
+          parsed: Bulkrax::ApplicationMatcher.method_defined?("parse_#{field}"),
+          if: nil,
+          excluded: false
+        }
+      }
+    end
+
+    # Properties that should not be used in imports. They are reserved for use by Hyrax.
+    self.reserved_properties = %w[
+      create_date
+      modified_date
+      date_modified
+      date_uploaded
+      depositor
+      arkivo_checksum
+      has_model
+      head
+      label
+      import_url
+      on_behalf_of
+      proxy_depositor
+      owner
+      state
+      tail
+      original_url
+      relative_path
+    ]
+  end
+
+  def self.api_definition
+    @api_definition ||= ActiveSupport::HashWithIndifferentAccess.new(
+      YAML.safe_load(
+        ERB.new(
+          File.read(Rails.root.join('config', 'bulkrax_api.yml'))
+        ).result
+      )
+    )
+  end
+
+  self.removed_image_path = 'app/assets/images/bulkrax/removed.png'
+
+  # this function maps the vars from your app into your engine
+  def self.setup
+    yield self
+  end
+end