bulkrax 1.0.2 → 2.0.0

data/app/parsers/bulkrax/oai_dc_parser.rb

@@ -75,7 +75,7 @@ module Bulkrax
 
         new_entry = collection_entry_class.where(importerexporter: importerexporter, identifier: unique_collection_identifier, raw_metadata: metadata).first_or_create!
         # perform now to ensure this gets created before work imports start
-        ImportWorkCollectionJob.perform_now(new_entry.id, importerexporter.current_run.id)
+        ImportCollectionJob.perform_now(new_entry.id, importerexporter.current_run.id)
         increment_counters(index, true)
       end
     end
@@ -119,7 +119,10 @@ module Bulkrax
       end
     end
 
-    def create_parent_child_relationships; end
+    # TODO: change to differentiate between collection and work records when adding ability to import collection metadata
+    def works_total
+      total
+    end
 
     def total
       @total ||= records(quick: true).doc.find(".//resumptionToken").to_a.first.attributes["completeListSize"].to_i

data/app/parsers/bulkrax/xml_parser.rb

@@ -12,6 +12,11 @@ module Bulkrax
     # @todo not yet supported
     def create_collections; end
 
+    # TODO: change to differentiate between collection and work records when adding ability to import collection metadata
+    def works_total
+      total
+    end
+
     # @todo not yet supported
     def import_fields; end
 

data/app/views/bulkrax/exporters/_form.html.erb

@@ -32,7 +32,7 @@
     prompt: 'Select from the list',
     label_html: { class: 'importer export-source-option hidden' },
     input_html: { class: 'importer export-source-option hidden' },
-    collection:  form.object.importers_list %>
+    collection:  form.object.importers_list.sort %>
 
   <%= form.input :export_source_collection,
     prompt: 'Start typing ...',

data/app/views/bulkrax/exporters/show.html.erb

@@ -57,8 +57,9 @@
       <strong><%= t('bulkrax.exporter.labels.limit') %>:</strong>
       <%= @exporter.limit %>
     </p>
+    <%= render partial: 'bulkrax/shared/bulkrax_errors', locals: {item: @exporter} %>
 
-  <%= render partial: 'bulkrax/shared/bulkrax_field_mapping', locals: {item: @exporter} %>
+    <%= render partial: 'bulkrax/shared/bulkrax_field_mapping', locals: {item: @exporter} %>
 
     <%# Currently, no parser-specific fields exist on Exporter,
         thus there's no real reason to always show this field %>

data/app/views/bulkrax/importers/index.html.erb

@@ -31,23 +31,21 @@
           </thead>
           <tbody>
             <% @importers.each do |importer| %>
-            <tr>
-              <th scope="row"><%= link_to importer.name, importer_path(importer) %></th>
-              <td>
-                <%= importer.status %>
-              </td>
-              <td><%= importer.last_imported_at.strftime("%b %d, %Y") if importer.last_imported_at %></td>
-              <td><%= importer.next_import_at.strftime("%b %d, %Y") if importer.next_import_at %></td>
-              <td><%= importer.importer_runs.last&.enqueued_records %></td>
-              <td><%= (importer.importer_runs.last&.processed_collections || 0) + (importer.importer_runs.last&.processed_records || 0)  %></td>
-              <td><%= (importer.importer_runs.last&.failed_collections || 0) + (importer.importer_runs.last&.failed_records || 0)  %></td>
-              <td><%= importer.importer_runs.last&.deleted_records %></td>
-              <td><%= importer.importer_runs.last&.total_collection_entries %></td>
-              <td><%= importer.importer_runs.last&.total_work_entries %></td>
-              <td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), importer_path(importer) %></td>
-              <td><%= link_to raw('<span class="glyphicon glyphicon-pencil"></span>'), edit_importer_path(importer) %></td>
-              <td><%= link_to raw('<span class="glyphicon glyphicon-remove"></span>'), importer, method: :delete, data: { confirm: 'Are you sure?' } %></td>
-            </tr>
+              <tr>
+                <th scope="row"><%= link_to importer.name, importer_path(importer) %></th>
+                <td><%= importer.status %></td>
+                <td><%= importer.last_imported_at.strftime("%b %d, %Y") if importer.last_imported_at %></td>
+                <td><%= importer.next_import_at.strftime("%b %d, %Y") if importer.next_import_at %></td>
+                <td><%= importer.importer_runs.last&.enqueued_records %></td>
+                <td><%= (importer.importer_runs.last&.processed_collections || 0) + (importer.importer_runs.last&.processed_records || 0) %></td>
+                <td><%= (importer.importer_runs.last&.failed_collections || 0) + (importer.importer_runs.last&.failed_records || 0) %></td>
+                <td><%= importer.importer_runs.last&.deleted_records %></td>
+                <td><%= importer.importer_runs.last&.total_collection_entries %></td>
+                <td><%= importer.importer_runs.last&.total_work_entries %></td>
+                <td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), importer_path(importer) %></td>
+                <td><%= link_to raw('<span class="glyphicon glyphicon-pencil"></span>'), edit_importer_path(importer) %></td>
+                <td><%= link_to raw('<span class="glyphicon glyphicon-remove"></span>'), importer, method: :delete, data: { confirm: 'Are you sure?' } %></td>
+              </tr>
             <% end %>
           </tbody>
         </table>

data/app/views/bulkrax/importers/show.html.erb

@@ -101,8 +101,10 @@
                   <td><%= e.id %></td>
                   <% if e.status == "Complete" %>
                     <td><span class="glyphicon glyphicon-ok" style="color: green;"></span> <%= e.status %></td>
+                  <% elsif e.status == "Pending" %>
+                    <td><span class="glyphicon glyphicon-option-horizontal" style="color: blue;"></span> <%= e.status %></td>
                   <% else %>
-                    <td><span class="glyphicon glyphicon-remove" style="color: red;"></span> <%= e.status %></td>
+                    <td><span class="glyphicon glyphicon-remove" style="color: <%= e.status == 'Deleted' ? 'green' : 'red' %>;"></span> <%= e.status %></td>
                   <% end %>
                   <% if e.last_error.present? %>
                     <td><%= link_to e.last_error.dig("error_class"), bulkrax.importer_entry_path(@importer.id, e.id) %></td>
@@ -137,8 +139,10 @@
                   <td><%= e.id %></td>
                   <% if e.status == "Complete" %>
                     <td><span class="glyphicon glyphicon-ok" style="color: green;"></span> <%= e.status %></td>
+                  <% elsif e.status == "Pending" %>
+                    <td><span class="glyphicon glyphicon-option-horizontal" style="color: blue;"></span> <%= e.status %></td>
                   <% else %>
-                    <td><span class="glyphicon glyphicon-remove" style="color: red;"></span> <%= e.status %></td>
+                    <td><span class="glyphicon glyphicon-remove" style="color: <%= e.status == 'Deleted' ? 'green' : 'red' %>;"></span> <%= e.status %></td>
                   <% end %>
                   <% if e.last_error.present? %>
                     <td><%= link_to e.last_error.dig("error_class"), bulkrax.importer_entry_path(@importer.id, e.id) %></td>

data/config/locales/bulkrax.en.yml

@@ -6,6 +6,7 @@ en:
         importers: Importers
     exporter:
       labels:
+        all: All
         collection: Collection
         export_format: Export Format
         export_from: Export From

data/db/migrate/20190731114016_change_importer_and_exporter_to_polymorphic.rb

@@ -6,7 +6,11 @@ end
 
 class ChangeImporterAndExporterToPolymorphic < ActiveRecord::Migration[5.1]
   def change
-    rename_column :bulkrax_entries, :importer_id, :importerexporter_id if column_exists?(:bulkrax_entries, :importer_id)
+    if column_exists?(:bulkrax_entries, :importer_id)
+      remove_foreign_key :bulkrax_entries, column: :importer_id
+      remove_index :bulkrax_entries, :importer_id
+      rename_column :bulkrax_entries, :importer_id, :importerexporter_id
+    end
     add_column :bulkrax_entries, :importerexporter_type, :string, after: :id, default: 'Bulkrax::Importer' unless column_exists?(:bulkrax_entries, :importerexporter_type)
   end
 end

data/db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb

@@ -0,0 +1,5 @@
+class ChangeBulkraxStatusesErrorMessageColumnTypeToText < ActiveRecord::Migration[5.1]
+  def change
+    change_column :bulkrax_statuses, :error_message, :text
+  end
+end

data/db/migrate/20211203195233_rename_children_counters_to_relationships.rb

@@ -0,0 +1,6 @@
+class RenameChildrenCountersToRelationships < ActiveRecord::Migration[5.2]
+  def change
+    rename_column :bulkrax_importer_runs, :processed_children, :processed_relationships
+    rename_column :bulkrax_importer_runs, :failed_children, :failed_relationships
+  end
+end

data/lib/bulkrax/engine.rb

@@ -25,7 +25,7 @@ module Bulkrax
     config.after_initialize do
       my_engine_root = Bulkrax::Engine.root.to_s
       paths = ActionController::Base.view_paths.collect(&:to_s)
-      hyrax_path = paths.detect { |path| path.match('/hyrax-') }
+      hyrax_path = paths.detect { |path| path.match(/\/hyrax-[\d\.]+.*/) }
       paths = if hyrax_path
                 paths.insert(paths.index(hyrax_path), my_engine_root + '/app/views')
               else

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '1.0.2'
+  VERSION = '2.0.0'
 end

data/lib/bulkrax.rb

@@ -5,12 +5,14 @@ require 'active_support/all'
 
 module Bulkrax
   class << self
+    # TODO: remove collection_field_mapping when releasing v2
     mattr_accessor :parsers,
                    :default_work_type,
                    :default_field_mapping,
                    :collection_field_mapping,
                    :fill_in_blank_source_identifiers,
-                   :parent_child_field_mapping,
+                   :related_children_field_mapping,
+                   :related_parents_field_mapping,
                    :reserved_properties,
                    :field_mappings,
                    :import_path,
@@ -33,26 +35,16 @@ module Bulkrax
     self.removed_image_path = Bulkrax::Engine.root.join('spec', 'fixtures', 'removed.png').to_s
     self.server_name = 'bulkrax@example.com'
 
-    # @todo, merge parent_child_field_mapping and collection_field_mapping into field_mappings,
-    # or make them settable per import some other way.
-
-    # Field_mapping for establishing a parent-child relationship (FROM parent TO child)
-    # This can be a Collection to Work, or Work to Work relationship
-    # This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
-    # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
-    # Example:
-    #   {
-    #     'Bulkrax::RdfEntry'  => 'http://opaquenamespace.org/ns/contents',
-    #     'Bulkrax::CsvEntry'  => 'children'
-    #   }
-    # By default no parent-child relationships are added
-    self.parent_child_field_mapping = {}
-
+    # NOTE: Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.
+    #       Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.
+    # TODO: remove collection_field_mapping when releasing v2
     # Field_mapping for establishing a collection relationship (FROM work TO collection)
     # This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
     # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
     # The default value for CSV is collection
-    self.collection_field_mapping = {}
+    self.collection_field_mapping = {
+      'Bulkrax::CsvEntry' => 'collection'
+    }
 
     # Hash of Generic field_mappings for use in the view
     # There must be one field_mappings hash per view parial

data/lib/generators/bulkrax/templates/bin/importer

@@ -6,18 +6,11 @@ require_relative '../config/environment'
 require 'slop'
 
 def main(opts = {})
-  if opts[:importer_id].blank? && invalid?(opts)
-    puts 'Missing required parameters'
-    help
-  end
-
-  if opts[:auth_token].blank?
-    puts 'Missing Authentication Token --auth_token'
-    exit
-  end
+  check_required_params
 
   update = opts[:importer_id].present?
-  url = build_url(opts.delete(:importer_id), opts.delete(:url))
+  port = opts[:port].presence
+  url = build_url(opts.delete(:importer_id), opts.delete(:url), port)
 
   headers = { 'Content-Type' => 'application/json' }
   headers['Authorization'] = "Token: #{opts.delete(:auth_token)}"
@@ -43,6 +36,18 @@ def main(opts = {})
   puts "#{response.status} - #{response.body.truncate(200)}"
 end
 
+def check_required_params
+  if opts[:importer_id].blank? && invalid?(opts)
+    puts 'Missing required parameters'
+    help
+  end
+
+  if opts[:auth_token].blank? # rubocop:disable Style/GuardClause
+    puts 'Missing Authentication Token --auth_token'
+    exit
+  end
+end
+
 def invalid?(opts)
   required_params.each do |p|
     return true if opts[p.to_sym].blank?
@@ -74,11 +79,12 @@ def build_params(opts = {})
   return params.compact
 end
 
-def build_url(importer_id, url)
+def build_url(importer_id, url, port = nil)
   if url.nil?
     protocol = Rails.application.config.force_ssl ? 'https://' : 'http://'
     host = Rails.application.config.action_mailer.default_url_options[:host]
     url = "#{protocol}#{host}"
+    url = "#{url}:#{port}" if port
   end
   path = Bulkrax::Engine.routes.url_helpers.polymorphic_path(Bulkrax::Importer)
   url = File.join(url, path)

data/lib/generators/bulkrax/templates/config/bulkrax_api.yml

@@ -81,4 +81,6 @@ bulkrax:
         - "Update and Import (importer has not yet been run)"
     auth_token:
       definition: 'Authentication token. Required for JSON requests only.'
-      required: true
+      required: true
+    port:
+      definition: 'Port to use in http request. Defaults to 80. May need to set to 3000 when running in the development environment'

data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb

@@ -19,18 +19,8 @@ Bulkrax.setup do |config|
   # Server name for oai request header
   # config.server_name = 'my_server@name.com'
 
-  # Field_mapping for establishing a parent-child relationship (FROM parent TO child)
-  # This can be a Collection to Work, or Work to Work relationship
-  # This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
-  # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
-  # Example:
-  #   {
-  #     'Bulkrax::RdfEntry'  => 'http://opaquenamespace.org/ns/contents',
-  #     'Bulkrax::CsvEntry'  => 'children'
-  #   }
-  # By default no parent-child relationships are added
-  # config.parent_child_field_mapping = { }
-
+  # NOTE: Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.
+  #       Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.
   # Field_mapping for establishing a collection relationship (FROM work TO collection)
   # This value IS NOT used for OAI, so setting the OAI parser here will have no effect
   # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
@@ -48,6 +38,11 @@ Bulkrax.setup do |config|
   #   e.g. to exclude date
   #   config.field_mappings["Bulkrax::OaiDcParser"]["date"] = { from: ["date"], excluded: true  }
   #
+  #   e.g. to import parent-child relationships
+  #   config.field_mappings['Bulkrax::CsvParser']['parents'] = { from: ['parents'], related_parents_field_mapping: true }
+  #   config.field_mappings['Bulkrax::CsvParser']['children'] = { from: ['children'], related_children_field_mapping: true }
+  #   (For more info on importing relationships, see Bulkrax Wiki: https://github.com/samvera-labs/bulkrax/wiki/Configuring-Bulkrax#parent-child-relationship-field-mappings)
+  #
   # #   e.g. to add the required source_identifier field
   #   #   config.field_mappings["Bulkrax::CsvParser"]["source_id"] = { from: ["old_source_id"], source_identifier: true  }
   # If you want Bulkrax to fill in source_identifiers for you, see below

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Rob Kaufman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-28 00:00:00.000000000 Z
+date: 2022-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -255,12 +255,12 @@ files:
 - app/helpers/bulkrax/importers_helper.rb
 - app/helpers/bulkrax/validation_helper.rb
 - app/jobs/bulkrax/application_job.rb
-- app/jobs/bulkrax/child_relationships_job.rb
+- app/jobs/bulkrax/create_relationships_job.rb
 - app/jobs/bulkrax/delete_work_job.rb
 - app/jobs/bulkrax/download_cloud_file_job.rb
 - app/jobs/bulkrax/export_work_job.rb
 - app/jobs/bulkrax/exporter_job.rb
-- app/jobs/bulkrax/import_work_collection_job.rb
+- app/jobs/bulkrax/import_collection_job.rb
 - app/jobs/bulkrax/import_work_job.rb
 - app/jobs/bulkrax/importer_job.rb
 - app/mailers/bulkrax/application_mailer.rb
@@ -351,6 +351,8 @@ files:
 - db/migrate/20201117220007_add_workflow_status_to_bulkrax_exporter.rb
 - db/migrate/20210806044408_remove_unused_last_error.rb
 - db/migrate/20210806065737_increase_text_sizes.rb
+- db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb
+- db/migrate/20211203195233_rename_children_counters_to_relationships.rb
 - lib/bulkrax.rb
 - lib/bulkrax/engine.rb
 - lib/bulkrax/version.rb

data/app/jobs/bulkrax/child_relationships_job.rb

@@ -1,128 +0,0 @@
-# frozen_string_literal: true
-
-module Bulkrax
-  class ChildWorksError < RuntimeError; end
-  class ChildRelationshipsJob < ApplicationJob
-    queue_as :import
-
-    def perform(*args)
-      @args = args
-
-      if entry.factory_class == Collection
-        collection_membership
-      else
-        work_membership
-      end
-      # Not all of the Works/Collections exist yet; reschedule
-    rescue Bulkrax::ChildWorksError
-      reschedule(args[0], args[1], args[2])
-    end
-
-    def collection_membership
-      # add collection to works
-      member_of_collection = []
-      child_works_hash.each { |k, v| member_of_collection << k if v[:class_name] != 'Collection' }
-      member_of_collection.each { |work| work_child_collection_parent(work) }
-
-      # add collections to collection
-      members_collections = []
-      child_works_hash.each { |k, v| members_collections << k if v[:class_name] == 'Collection' }
-      collection_parent_collection_child(members_collections) if members_collections.present?
-    end
-
-    def work_membership
-      # add works to work
-      # reject any Collections, they can't be children of Works
-      members_works = []
-      # reject any Collections, they can't be children of Works
-      child_works_hash.each { |k, v| members_works << k if v[:class_name] != 'Collection' }
-      if members_works.length < child_entries.length # rubocop:disable Style/IfUnlessModifier
-        Rails.logger.warn("Cannot add collections as children of works: #{(@child_entries.length - members_works.length)} collections were discarded for parent entry #{@entry.id} (of #{@child_entries.length})")
-      end
-      work_parent_work_child(members_works) if members_works.present?
-    end
-
-    def entry
-      @entry ||= Bulkrax::Entry.find(@args[0])
-    end
-
-    def child_entries
-      @child_entries ||= @args[1].map { |e| Bulkrax::Entry.find(e) }
-    end
-
-    def child_works_hash
-      @child_works_hash ||= child_entries.each_with_object({}) do |child_entry, hash|
-        work = child_entry.factory.find
-        # If we can't find the Work/Collection, raise a custom error
-        raise ChildWorksError if work.blank?
-        hash[work.id] = { class_name: work.class.to_s, entry.parser.source_identifier => child_entry.identifier }
-      end
-    end
-
-    def importer_run_id
-      @args[2]
-    end
-
-    def user
-      @user ||= entry.importerexporter.user
-    end
-
-    private
-
-    # rubocop:disable Rails/SkipsModelValidations
-    # Work-Collection membership is added to the child as member_of_collection_ids
-    # This is adding the reverse relatinship, from the child to the parent
-    def work_child_collection_parent(work_id)
-      attrs = { id: work_id, collections: [{ id: entry&.factory&.find&.id }] }
-      Bulkrax::ObjectFactory.new(attributes: attrs,
-                                 source_identifier_value: child_works_hash[work_id][entry.parser.source_identifier],
-                                 work_identifier: entry.parser.work_identifier,
-                                 replace_files: false,
-                                 user: user,
-                                 klass: child_works_hash[work_id][:class_name].constantize).run
-      ImporterRun.find(importer_run_id).increment!(:processed_children)
-    rescue StandardError => e
-      entry.status_info(e)
-      ImporterRun.find(importer_run_id).increment!(:failed_children)
-    end
-
-    # Collection-Collection membership is added to the as member_ids
-    def collection_parent_collection_child(member_ids)
-      attrs = { id: entry&.factory&.find&.id, children: member_ids }
-      Bulkrax::ObjectFactory.new(attributes: attrs,
-                                 source_identifier_value: entry.identifier,
-                                 work_identifier: entry.parser.work_identifier,
-                                 replace_files: false,
-                                 user: user,
-                                 klass: entry.factory_class).run
-      ImporterRun.find(importer_run_id).increment!(:processed_children)
-    rescue StandardError => e
-      entry.status_info(e)
-      ImporterRun.find(importer_run_id).increment!(:failed_children)
-    end
-
-    # Work-Work membership is added to the parent as member_ids
-    def work_parent_work_child(member_ids)
-      # build work_members_attributes
-      attrs = { id: entry&.factory&.find&.id,
-                work_members_attributes: member_ids.each.with_index.each_with_object({}) do |(member, index), ids|
-                  ids[index] = { id: member }
-                end }
-      Bulkrax::ObjectFactory.new(attributes: attrs,
-                                 source_identifier_value: entry.identifier,
-                                 work_identifier: entry.parser.work_identifier,
-                                 replace_files: false,
-                                 user: user,
-                                 klass: entry.factory_class).run
-      ImporterRun.find(importer_run_id).increment!(:processed_children)
-    rescue StandardError => e
-      entry.status_info(e)
-      ImporterRun.find(importer_run_id).increment!(:failed_children)
-    end
-    # rubocop:enable Rails/SkipsModelValidations
-
-    def reschedule(entry_id, child_entry_ids, importer_run_id)
-      ChildRelationshipsJob.set(wait: 10.minutes).perform_later(entry_id, child_entry_ids, importer_run_id)
-    end
-  end
-end