bulkrax 1.0.2 → 2.1.0

data/lib/bulkrax.rb

@@ -5,12 +5,14 @@ require 'active_support/all'
 
 module Bulkrax
   class << self
+    # TODO: remove collection_field_mapping when releasing v2
     mattr_accessor :parsers,
                    :default_work_type,
                    :default_field_mapping,
                    :collection_field_mapping,
                    :fill_in_blank_source_identifiers,
-                   :parent_child_field_mapping,
+                   :related_children_field_mapping,
+                   :related_parents_field_mapping,
                    :reserved_properties,
                    :field_mappings,
                    :import_path,
@@ -33,26 +35,16 @@ module Bulkrax
     self.removed_image_path = Bulkrax::Engine.root.join('spec', 'fixtures', 'removed.png').to_s
     self.server_name = 'bulkrax@example.com'
 
-    # @todo, merge parent_child_field_mapping and collection_field_mapping into field_mappings,
-    # or make them settable per import some other way.
-
-    # Field_mapping for establishing a parent-child relationship (FROM parent TO child)
-    # This can be a Collection to Work, or Work to Work relationship
-    # This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
-    # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
-    # Example:
-    #   {
-    #     'Bulkrax::RdfEntry'  => 'http://opaquenamespace.org/ns/contents',
-    #     'Bulkrax::CsvEntry'  => 'children'
-    #   }
-    # By default no parent-child relationships are added
-    self.parent_child_field_mapping = {}
-
+    # NOTE: Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.
+    #       Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.
+    # TODO: remove collection_field_mapping when releasing v2
     # Field_mapping for establishing a collection relationship (FROM work TO collection)
     # This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
     # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
     # The default value for CSV is collection
-    self.collection_field_mapping = {}
+    self.collection_field_mapping = {
+      'Bulkrax::CsvEntry' => 'collection'
+    }
 
     # Hash of Generic field_mappings for use in the view
     # There must be one field_mappings hash per view parial

data/lib/generators/bulkrax/templates/bin/importer

@@ -6,18 +6,11 @@ require_relative '../config/environment'
 require 'slop'
 
 def main(opts = {})
-  if opts[:importer_id].blank? && invalid?(opts)
-    puts 'Missing required parameters'
-    help
-  end
-
-  if opts[:auth_token].blank?
-    puts 'Missing Authentication Token --auth_token'
-    exit
-  end
+  check_required_params
 
   update = opts[:importer_id].present?
-  url = build_url(opts.delete(:importer_id), opts.delete(:url))
+  port = opts[:port].presence
+  url = build_url(opts.delete(:importer_id), opts.delete(:url), port)
 
   headers = { 'Content-Type' => 'application/json' }
   headers['Authorization'] = "Token: #{opts.delete(:auth_token)}"
@@ -43,6 +36,18 @@ def main(opts = {})
   puts "#{response.status} - #{response.body.truncate(200)}"
 end
 
+def check_required_params
+  if opts[:importer_id].blank? && invalid?(opts)
+    puts 'Missing required parameters'
+    help
+  end
+
+  if opts[:auth_token].blank? # rubocop:disable Style/GuardClause
+    puts 'Missing Authentication Token --auth_token'
+    exit
+  end
+end
+
 def invalid?(opts)
   required_params.each do |p|
     return true if opts[p.to_sym].blank?
@@ -74,11 +79,12 @@ def build_params(opts = {})
   return params.compact
 end
 
-def build_url(importer_id, url)
+def build_url(importer_id, url, port = nil)
   if url.nil?
     protocol = Rails.application.config.force_ssl ? 'https://' : 'http://'
     host = Rails.application.config.action_mailer.default_url_options[:host]
     url = "#{protocol}#{host}"
+    url = "#{url}:#{port}" if port
   end
   path = Bulkrax::Engine.routes.url_helpers.polymorphic_path(Bulkrax::Importer)
   url = File.join(url, path)

data/lib/generators/bulkrax/templates/config/bulkrax_api.yml

@@ -81,4 +81,6 @@ bulkrax:
         - "Update and Import (importer has not yet been run)"
     auth_token:
       definition: 'Authentication token. Required for JSON requests only.'
-      required: true
+      required: true
+    port:
+      definition: 'Port to use in http request. Defaults to 80. May need to set to 3000 when running in the development environment'

data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb

@@ -19,18 +19,8 @@ Bulkrax.setup do |config|
   # Server name for oai request header
   # config.server_name = 'my_server@name.com'
 
-  # Field_mapping for establishing a parent-child relationship (FROM parent TO child)
-  # This can be a Collection to Work, or Work to Work relationship
-  # This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
-  # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
-  # Example:
-  #   {
-  #     'Bulkrax::RdfEntry'  => 'http://opaquenamespace.org/ns/contents',
-  #     'Bulkrax::CsvEntry'  => 'children'
-  #   }
-  # By default no parent-child relationships are added
-  # config.parent_child_field_mapping = { }
-
+  # NOTE: Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.
+  #       Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.
   # Field_mapping for establishing a collection relationship (FROM work TO collection)
   # This value IS NOT used for OAI, so setting the OAI parser here will have no effect
   # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
@@ -48,6 +38,11 @@ Bulkrax.setup do |config|
   #   e.g. to exclude date
   #   config.field_mappings["Bulkrax::OaiDcParser"]["date"] = { from: ["date"], excluded: true  }
   #
+  #   e.g. to import parent-child relationships
+  #   config.field_mappings['Bulkrax::CsvParser']['parents'] = { from: ['parents'], related_parents_field_mapping: true }
+  #   config.field_mappings['Bulkrax::CsvParser']['children'] = { from: ['children'], related_children_field_mapping: true }
+  #   (For more info on importing relationships, see Bulkrax Wiki: https://github.com/samvera-labs/bulkrax/wiki/Configuring-Bulkrax#parent-child-relationship-field-mappings)
+  #
   # #   e.g. to add the required source_identifier field
   #   #   config.field_mappings["Bulkrax::CsvParser"]["source_id"] = { from: ["old_source_id"], source_identifier: true  }
   # If you want Bulkrax to fill in source_identifiers for you, see below

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.1.0
 platform: ruby
 authors:
 - Rob Kaufman
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-28 00:00:00.000000000 Z
+date: 2022-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -224,7 +224,11 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Description of Bulkrax.
+description: Bulkrax is a batteries included importer for Samvera applications. It
+  currently includes support for OAI-PMH (DC and Qualified DC) and CSV out of the
+  box. It is also designed to be extensible, allowing you to easily add new importers
+  in to your application or to include them with other gems. Bulkrax provides a full
+  admin interface including creating, editing, scheduling and reviewing imports.
 email:
 - rob@notch8.com
 executables: []
@@ -255,12 +259,13 @@ files:
 - app/helpers/bulkrax/importers_helper.rb
 - app/helpers/bulkrax/validation_helper.rb
 - app/jobs/bulkrax/application_job.rb
-- app/jobs/bulkrax/child_relationships_job.rb
+- app/jobs/bulkrax/create_relationships_job.rb
 - app/jobs/bulkrax/delete_work_job.rb
 - app/jobs/bulkrax/download_cloud_file_job.rb
 - app/jobs/bulkrax/export_work_job.rb
 - app/jobs/bulkrax/exporter_job.rb
-- app/jobs/bulkrax/import_work_collection_job.rb
+- app/jobs/bulkrax/import_collection_job.rb
+- app/jobs/bulkrax/import_file_set_job.rb
 - app/jobs/bulkrax/import_work_job.rb
 - app/jobs/bulkrax/importer_job.rb
 - app/mailers/bulkrax/application_mailer.rb
@@ -271,6 +276,7 @@ files:
 - app/models/bulkrax/application_record.rb
 - app/models/bulkrax/csv_collection_entry.rb
 - app/models/bulkrax/csv_entry.rb
+- app/models/bulkrax/csv_file_set_entry.rb
 - app/models/bulkrax/entry.rb
 - app/models/bulkrax/exporter.rb
 - app/models/bulkrax/exporter_run.rb
@@ -286,6 +292,7 @@ files:
 - app/models/bulkrax/status.rb
 - app/models/bulkrax/xml_entry.rb
 - app/models/concerns/bulkrax/download_behavior.rb
+- app/models/concerns/bulkrax/dynamic_record_lookup.rb
 - app/models/concerns/bulkrax/errored_entries.rb
 - app/models/concerns/bulkrax/export_behavior.rb
 - app/models/concerns/bulkrax/file_factory.rb
@@ -351,6 +358,11 @@ files:
 - db/migrate/20201117220007_add_workflow_status_to_bulkrax_exporter.rb
 - db/migrate/20210806044408_remove_unused_last_error.rb
 - db/migrate/20210806065737_increase_text_sizes.rb
+- db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb
+- db/migrate/20211203195233_rename_children_counters_to_relationships.rb
+- db/migrate/20211220195027_add_file_set_counters_to_importer_runs.rb
+- db/migrate/20220118001339_add_import_attempts_to_entries.rb
+- db/migrate/20220119213325_add_work_counters_to_importer_runs.rb
 - lib/bulkrax.rb
 - lib/bulkrax/engine.rb
 - lib/bulkrax/version.rb
@@ -366,7 +378,7 @@ homepage: https://github.com/samvera-labs/bulkrax
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -381,8 +393,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
-summary: Summary of Bulkrax.
+summary: Import and export tool for Hyrax and Hyku
 test_files: []

data/app/jobs/bulkrax/child_relationships_job.rb

@@ -1,128 +0,0 @@
-# frozen_string_literal: true
-
-module Bulkrax
-  class ChildWorksError < RuntimeError; end
-  class ChildRelationshipsJob < ApplicationJob
-    queue_as :import
-
-    def perform(*args)
-      @args = args
-
-      if entry.factory_class == Collection
-        collection_membership
-      else
-        work_membership
-      end
-      # Not all of the Works/Collections exist yet; reschedule
-    rescue Bulkrax::ChildWorksError
-      reschedule(args[0], args[1], args[2])
-    end
-
-    def collection_membership
-      # add collection to works
-      member_of_collection = []
-      child_works_hash.each { |k, v| member_of_collection << k if v[:class_name] != 'Collection' }
-      member_of_collection.each { |work| work_child_collection_parent(work) }
-
-      # add collections to collection
-      members_collections = []
-      child_works_hash.each { |k, v| members_collections << k if v[:class_name] == 'Collection' }
-      collection_parent_collection_child(members_collections) if members_collections.present?
-    end
-
-    def work_membership
-      # add works to work
-      # reject any Collections, they can't be children of Works
-      members_works = []
-      # reject any Collections, they can't be children of Works
-      child_works_hash.each { |k, v| members_works << k if v[:class_name] != 'Collection' }
-      if members_works.length < child_entries.length # rubocop:disable Style/IfUnlessModifier
-        Rails.logger.warn("Cannot add collections as children of works: #{(@child_entries.length - members_works.length)} collections were discarded for parent entry #{@entry.id} (of #{@child_entries.length})")
-      end
-      work_parent_work_child(members_works) if members_works.present?
-    end
-
-    def entry
-      @entry ||= Bulkrax::Entry.find(@args[0])
-    end
-
-    def child_entries
-      @child_entries ||= @args[1].map { |e| Bulkrax::Entry.find(e) }
-    end
-
-    def child_works_hash
-      @child_works_hash ||= child_entries.each_with_object({}) do |child_entry, hash|
-        work = child_entry.factory.find
-        # If we can't find the Work/Collection, raise a custom error
-        raise ChildWorksError if work.blank?
-        hash[work.id] = { class_name: work.class.to_s, entry.parser.source_identifier => child_entry.identifier }
-      end
-    end
-
-    def importer_run_id
-      @args[2]
-    end
-
-    def user
-      @user ||= entry.importerexporter.user
-    end
-
-    private
-
-    # rubocop:disable Rails/SkipsModelValidations
-    # Work-Collection membership is added to the child as member_of_collection_ids
-    # This is adding the reverse relatinship, from the child to the parent
-    def work_child_collection_parent(work_id)
-      attrs = { id: work_id, collections: [{ id: entry&.factory&.find&.id }] }
-      Bulkrax::ObjectFactory.new(attributes: attrs,
-                                 source_identifier_value: child_works_hash[work_id][entry.parser.source_identifier],
-                                 work_identifier: entry.parser.work_identifier,
-                                 replace_files: false,
-                                 user: user,
-                                 klass: child_works_hash[work_id][:class_name].constantize).run
-      ImporterRun.find(importer_run_id).increment!(:processed_children)
-    rescue StandardError => e
-      entry.status_info(e)
-      ImporterRun.find(importer_run_id).increment!(:failed_children)
-    end
-
-    # Collection-Collection membership is added to the as member_ids
-    def collection_parent_collection_child(member_ids)
-      attrs = { id: entry&.factory&.find&.id, children: member_ids }
-      Bulkrax::ObjectFactory.new(attributes: attrs,
-                                 source_identifier_value: entry.identifier,
-                                 work_identifier: entry.parser.work_identifier,
-                                 replace_files: false,
-                                 user: user,
-                                 klass: entry.factory_class).run
-      ImporterRun.find(importer_run_id).increment!(:processed_children)
-    rescue StandardError => e
-      entry.status_info(e)
-      ImporterRun.find(importer_run_id).increment!(:failed_children)
-    end
-
-    # Work-Work membership is added to the parent as member_ids
-    def work_parent_work_child(member_ids)
-      # build work_members_attributes
-      attrs = { id: entry&.factory&.find&.id,
-                work_members_attributes: member_ids.each.with_index.each_with_object({}) do |(member, index), ids|
-                  ids[index] = { id: member }
-                end }
-      Bulkrax::ObjectFactory.new(attributes: attrs,
-                                 source_identifier_value: entry.identifier,
-                                 work_identifier: entry.parser.work_identifier,
-                                 replace_files: false,
-                                 user: user,
-                                 klass: entry.factory_class).run
-      ImporterRun.find(importer_run_id).increment!(:processed_children)
-    rescue StandardError => e
-      entry.status_info(e)
-      ImporterRun.find(importer_run_id).increment!(:failed_children)
-    end
-    # rubocop:enable Rails/SkipsModelValidations
-
-    def reschedule(entry_id, child_entry_ids, importer_run_id)
-      ChildRelationshipsJob.set(wait: 10.minutes).perform_later(entry_id, child_entry_ids, importer_run_id)
-    end
-  end
-end