bulkrax 6.0.1 → 8.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9892c080d7d1cfa595d1965fae2e1b7372f6ec0b7042b0acc7ceb86a0421abd
-  data.tar.gz: bdd80bd311e47ad4705ed9d2770e24b10248656fa3f78c287943e829ed891c79
+  metadata.gz: 6ca767b1e461d5830774bb4d7c49eade799d14298625bdbdccdcbdc0ac997f88
+  data.tar.gz: b5024ea772132123fcb922fbe183d22950428ea50b7a8b4152c6bd781b94510f
 SHA512:
-  metadata.gz: 0034564d49de171a95cc1b415b9593cb70080102eb4e66cf083eb1e8b2f04e7dce9fd80aef158008ddd17ee88a8e31afec468ce8a2b323d55399fbcabf3c2402
-  data.tar.gz: 861e1573df358bb3b610fde019c884c1f9ef944956aeabb7cd17c729c6b0ec281fb60036f29ccae3d98fe0aec5f46eed358a8ad87996b599c0c5f71e3366a491
+  metadata.gz: 91004f88a97054107a3bbbcbe7f3b3dd84226a286df47992c13bfc5fb483c3bfd4aaa9190e08e567258e5105a317a37bba58fa7857ee6c534153ba930b608ad7
+  data.tar.gz: 89a32c155c3931e486b2d41215942b291eb724c59833a031fdc4815a72d26a2df13b015c1b1e2c0dafd608e58ec481a574e20da7bcf95ed76b3e45c3d5f19287

data/Rakefile

@@ -25,18 +25,18 @@ require 'bundler/gem_tasks'
 
 require 'solr_wrapper/rake_task' unless Rails.env.production?
 
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new(:rubocop) do |t|
+  t.options = ['--display-cop-names', '--ignore-parent-exclusion', '-a']
+end
+
 begin
   require 'rspec/core/rake_task'
 
   RSpec::Core::RakeTask.new(:spec)
 
-  task default: :spec
+  task default: [:rubocop, :spec]
 rescue LoadError # rubocop:disable Lint/HandleExceptions
   # no rspec available
 end
-
-require 'rubocop/rake_task'
-
-RuboCop::RakeTask.new(:rubocop) do |t|
-  t.options = ['--display-cop-names']
-end

data/app/assets/javascripts/bulkrax/bulkrax.js

@@ -8,4 +8,15 @@ $(document).on('turbolinks:load ready', function() {
   $('button#fm_toggle').click(function() {
     $('#field_mapping').toggle();
   });
+  $('#bulkraxItemModal').on('show.bs.modal', function (event) {
+    var button = $(event.relatedTarget) // Button that triggered the modal
+    var recipient = button.data('entry-id') // Extract info from data-* attributes
+    // If necessary, you could initiate an AJAX request here (and then do the updating in a callback).
+    // Update the modal's content. We'll use jQuery here, but you could use a data binding library or other methods instead.
+    var modal = $(this)
+    modal.find('a').each(function() {
+      this.href = this.href.replace(/\d+\?/, recipient + '?')
+    })
+    return true
+  })
 });

data/app/assets/javascripts/bulkrax/datatables.js

@@ -0,0 +1,139 @@
+Blacklight.onLoad(function() {
+  if($('#importer-show-table').length) {
+    $('#importer-show-table').DataTable( {
+      'processing': true,
+      'serverSide': true,
+      "ajax": window.location.href.replace(/(\/(importers|exporters)\/\d+)/, "$1/entry_table.json"),
+      "pageLength": 30,
+      "lengthMenu": [[30, 100, 200], [30, 100, 200]],
+      "columns": [
+        { "data": "identifier" },
+        { "data": "id" },
+        { "data": "status_message" },
+        { "data": "type" },
+        { "data": "updated_at" },
+        { "data": "errors", "orderable": false },
+        { "data": "actions", "orderable": false }
+      ],
+      initComplete: function () {
+        // Add entry class filter
+        entrySelect.bind(this)()
+        // Add status filter
+        statusSelect.bind(this)()
+        // Add refresh link
+        refreshLink.bind(this)()
+      }
+    } );
+  }
+
+    if($('#importers-table').length) {
+    $('#importers-table').DataTable( {
+      'processing': true,
+      'serverSide': true,
+      "ajax": window.location.href.replace(/(\/importers)/, "$1/importer_table.json"),
+      "pageLength": 30,
+      "lengthMenu": [[30, 100, 200], [30, 100, 200]],
+      "columns": [
+        { "data": "name" },
+        { "data": "status_message" },
+        { "data": "last_imported_at" },
+        { "data": "next_import_at" },
+        { "data": "enqueued_records", "orderable": false },
+        { "data": "processed_records", "orderable": false },
+        { "data": "failed_records", "orderable": false },
+        { "data": "deleted_records", "orderable": false },
+        { "data": "total_collection_entries", "orderable": false },
+        { "data": "total_work_entries", "orderable": false },
+        { "data": "total_file_set_entries", "orderable": false },
+        { "data": "actions", "orderable": false }
+      ],
+      initComplete: function () {
+        // Add status filter
+        statusSelect.bind(this)()
+        // Add refresh link
+        refreshLink.bind(this)()
+      }
+    } );
+  }
+
+  if($('#exporters-table').length) {
+    $('#exporters-table').DataTable( {
+      'processing': true,
+      'serverSide': true,
+      "ajax": window.location.href.replace(/(\/exporters)/, "$1/exporter_table.json"),
+      "pageLength": 30,
+      "lengthMenu": [[30, 100, 200], [30, 100, 200]],
+      "columns": [
+        { "data": "name" },
+        { "data": "status_message" },
+        { "data": "created_at" },
+        { "data": "download" },
+        { "data": "actions", "orderable": false }
+      ],
+      initComplete: function () {
+        // Add status filter
+        statusSelect.bind(this)()
+        // Add refresh link
+        refreshLink.bind(this)()
+      }
+    } );
+  }
+
+})
+
+function entrySelect() {
+  let entrySelect = document.createElement('select')
+  entrySelect.id = 'entry-filter'
+  entrySelect.classList.value = 'form-control input-sm'
+  entrySelect.style.marginRight = '10px'
+
+  entrySelect.add(new Option('Filter by Entry Class', ''))
+  // Read the options from the footer and add them to the entrySelect
+  $('#importer-entry-classes').text().split('|').forEach(function (col, i) {
+    entrySelect.add(new Option(col.trim()))
+  })
+  document.querySelector('div#importer-show-table_filter').firstChild.prepend(entrySelect)
+
+  // Apply listener for user change in value
+  entrySelect.addEventListener('change', function () {
+    var val = entrySelect.value;
+    this.api()
+      .search(val ? val : '', false, false)
+      .draw();
+  }.bind(this));
+}
+
+function statusSelect() {
+  let statusSelect = document.createElement('select');
+  statusSelect.id = 'status-filter'
+  statusSelect.classList.value = 'form-control input-sm'
+  statusSelect.style.marginRight = '10px'
+
+  statusSelect.add(new Option('Filter by Status', ''));
+  statusSelect.add(new Option('Complete'))
+  statusSelect.add(new Option('Pending'))
+  statusSelect.add(new Option('Failed'))
+  statusSelect.add(new Option('Skipped'))
+  statusSelect.add(new Option('Deleted'))
+  statusSelect.add(new Option('Complete (with failures)'))
+
+  document.querySelector('div.dataTables_filter').firstChild.prepend(statusSelect)
+
+  // Apply listener for user change in value
+  statusSelect.addEventListener('change', function () {
+    var val = statusSelect.value;
+    this.api()
+      .search(val ? val : '', false, false)
+      .draw();
+  }.bind(this));
+}
+
+function refreshLink() {
+  let refreshLink = document.createElement('a');
+  refreshLink.onclick = function() {
+    this.api().ajax.reload(null, false)
+  }.bind(this)
+  refreshLink.classList.value = 'fa fa-refresh'
+  refreshLink.style.marginLeft = '10px'
+  document.querySelector('div.dataTables_filter').firstChild.append(refreshLink)
+}

data/app/assets/javascripts/bulkrax/exporters.js

@@ -26,14 +26,14 @@ function removeRequired(allSources) {
 
 // hide all export_source
 function hide(allSources) {
-  allSources.addClass('d-none');
-  allSources.find('#exporter_export_source').addClass('.d-none').attr('type', 'd-none');
+  allSources.addClass('d-none hidden');
+  allSources.find('#exporter_export_source').addClass('.d-none hidden').attr('type', 'd-none hidden');
 }
 
 // unhide selected export_source
 function unhideSelected(selectedSource) {
-  selectedSource.removeClass('d-none').removeAttr('type');
-  selectedSource.parent().removeClass('d-none').removeAttr('type');
+  selectedSource.removeClass('d-none hidden').removeAttr('type');
+  selectedSource.parent().removeClass('d-none hidden').removeAttr('type');
 };
 
 // add the autocomplete javascript

data/app/assets/javascripts/bulkrax/importers.js.erb

@@ -74,12 +74,14 @@ function handleFileToggle(file_path) {
     $('#file_path').hide()
     $('#file_upload').hide()
     $('#cloud').hide()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', null)
     $('#file_upload input').attr('required', null)
   } else {
     $('#file_path').show()
     $('#file_upload').hide()
     $('#cloud').hide()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', 'required')
     $('#file_upload input').attr('required', null)
     $('#importer_parser_fields_file_style_specify_a_path_on_the_server').attr('checked', true)
@@ -89,6 +91,7 @@ function handleFileToggle(file_path) {
     $('#file_path').hide()
     $('#file_upload').show()
     $('#cloud').hide()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', null)
     $('#file_upload input').attr('required', 'required')
   })
@@ -96,6 +99,7 @@ function handleFileToggle(file_path) {
     $('#file_path').show()
     $('#file_upload').hide()
     $('#cloud').hide()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', 'required')
     $('#file_upload input').attr('required', null)
   })
@@ -103,9 +107,19 @@ function handleFileToggle(file_path) {
     $('#file_path').hide()
     $('#file_upload').hide()
     $('#cloud').show()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', null)
     $('#file_upload input').attr('required', null)
   })
+  $('#importer_parser_fields_file_style_existing_entries').click(function(e){
+    $('#file_path').hide()
+    $('#file_upload').hide()
+    $('#cloud').hide()
+    $('#existing_options').show()
+    $('#file_path input').attr('required', null)
+    $('#file_upload input').attr('required', null)
+  })
+
 }
 
 function handleParserKlass() {
@@ -189,4 +203,4 @@ function setError(selector, error) {
   selector.attr('disabled', true)
 }
 
-$(document).on({'ready': prepBulkrax, 'turbolinks:load': prepBulkrax})
+$(document).on({'ready': prepBulkrax, 'turbolinks:load': prepBulkrax})

data/app/assets/stylesheets/bulkrax/import_export.scss

@@ -34,4 +34,9 @@ div#s2id_exporter_export_source_collection {
 
 .bulkrax-clear-toggles {
   clear: both;
-}
+}
+
+#existing_options .collection_check_boxes {
+  margin-left: 10px;
+  margin-right: 10px;
+}

data/app/controllers/bulkrax/entries_controller.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require_dependency "bulkrax/application_controller"
-require_dependency "oai"
-
 module Bulkrax
   class EntriesController < ApplicationController
     include Hyrax::ThemedLayoutController if defined?(::Hyrax)
@@ -18,6 +15,58 @@ module Bulkrax
       end
     end
 
+    def update
+      @entry = Entry.find(params[:id])
+      type = case @entry.type.downcase
+             when /fileset/
+               'file_set'
+             when /collection/
+               'collection'
+             else
+               'work'
+             end
+      item = @entry.importerexporter
+      # do not run counters as it loads the whole parser
+      current_run = item.current_run(skip_counts: true)
+      @entry.set_status_info('Pending', current_run)
+      ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: @entry.importer.id)
+
+      if params[:destroy_first]
+        "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.perform_later(@entry, current_run)
+      else
+        "Bulkrax::Import#{type.camelize}Job".constantize.perform_later(@entry.id, current_run.id)
+      end
+
+      entry_path = item.class.to_s.include?('Importer') ? bulkrax.importer_entry_path(item.id, @entry.id) : bulkrax.exporter_entry_path(item.id, @entry.id)
+
+      redirect_back fallback_location: entry_path, notice: "Entry #{@entry.id} update has been queued"
+    end
+
+    def destroy
+      @entry = Entry.find(params[:id])
+      @status = ""
+      begin
+        work = @entry.factory&.find
+        if work.present?
+          work.destroy
+          @entry.destroy
+          @status = "Entry and work deleted"
+        else
+          @entry.destroy
+          @status = "Entry deleted"
+        end
+      rescue StandardError => e
+        @status = "Error: #{e.message}"
+      end
+
+      item = @entry.importerexporter
+      entry_path = item.class.to_s.include?('Importer') ? bulkrax.importer_entry_path(item.id, @entry.id) : bulkrax.exporter_entry_path(item.id, @entry.id)
+
+      redirect_back fallback_location: entry_path, notice: @status
+    end
+
+    protected
+
     # GET /importers/1/entries/1
     def show_importer
       @importer = Importer.find(params[:importer_id])

data/app/controllers/bulkrax/exporters_controller.rb

@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
-require_dependency "bulkrax/application_controller"
-
 module Bulkrax
   class ExportersController < ApplicationController
     include Hyrax::ThemedLayoutController if defined?(::Hyrax)
     include Bulkrax::DownloadBehavior
+    include Bulkrax::DatatablesBehavior
     before_action :authenticate_user!
     before_action :check_permissions
-    before_action :set_exporter, only: [:show, :edit, :update, :destroy]
+    before_action :set_exporter, only: [:show, :entry_table, :edit, :update, :destroy]
     with_themed_layout 'dashboard' if defined?(::Hyrax)
 
     # GET /exporters
@@ -19,16 +18,29 @@ module Bulkrax
       add_exporter_breadcrumbs if defined?(::Hyrax)
     end
 
+    def exporter_table
+      @exporters = Exporter.order(table_order).page(table_page).per(table_per_page)
+      @exporters = @exporters.where(exporter_table_search) if exporter_table_search.present?
+      respond_to do |format|
+        format.json { render json: format_exporters(@exporters) }
+      end
+    end
+
     # GET /exporters/1
     def show
       if defined?(::Hyrax)
         add_exporter_breadcrumbs
         add_breadcrumb @exporter.name
       end
+      @first_entry = @exporter.entries.first
+    end
 
-      @work_entries = @exporter.entries.where(type: @exporter.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
-      @collection_entries = @exporter.entries.where(type: @exporter.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
-      @file_set_entries = @exporter.entries.where(type: @exporter.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
+    def entry_table
+      @entries = @exporter.entries.order(table_order).page(table_page).per(table_per_page)
+      @entries = @entries.where(entry_table_search) if entry_table_search.present?
+      respond_to do |format|
+        format.json { render json: format_entries(@entries, @exporter) }
+      end
     end
 
     # GET /exporters/new
@@ -48,7 +60,7 @@ module Bulkrax
       end
 
       # Correctly populate export_source_collection input
-      @collection = Collection.find(@exporter.export_source) if @exporter.export_source.present? && @exporter.export_from == 'collection'
+      @collection = Bulkrax.object_factory.find(@exporter.export_source) if @exporter.export_source.present? && @exporter.export_from == 'collection'
     end
 
     # POST /exporters
@@ -102,7 +114,7 @@ module Bulkrax
 
     # Use callbacks to share common setup or constraints between actions.
     def set_exporter
-      @exporter = Exporter.find(params[:id])
+      @exporter = Exporter.find(params[:id] || params[:exporter_id])
     end
 
     # Only allow a trusted parameters through.

data/app/controllers/bulkrax/importers_controller.rb

@@ -1,34 +1,40 @@
 # frozen_string_literal: true
 
-require_dependency 'bulkrax/application_controller'
-require_dependency 'oai'
-
 module Bulkrax
   # rubocop:disable Metrics/ClassLength
-  class ImportersController < ApplicationController
+  class ImportersController < ::Bulkrax::ApplicationController
     include Hyrax::ThemedLayoutController if defined?(::Hyrax)
     include Bulkrax::DownloadBehavior
     include Bulkrax::API
+    include Bulkrax::DatatablesBehavior
     include Bulkrax::ValidationHelper
 
     protect_from_forgery unless: -> { api_request? }
     before_action :token_authenticate!, if: -> { api_request? }, only: [:create, :update, :delete]
     before_action :authenticate_user!, unless: -> { api_request? }
     before_action :check_permissions
-    before_action :set_importer, only: [:show, :edit, :update, :destroy]
+    before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy]
     with_themed_layout 'dashboard' if defined?(::Hyrax)
 
     # GET /importers
     def index
       # NOTE: We're paginating this in the browser.
-      @importers = Importer.order(created_at: :desc).all
       if api_request?
+        @importers = Importer.order(created_at: :desc).all
         json_response('index')
       elsif defined?(::Hyrax)
         add_importer_breadcrumbs
       end
     end
 
+    def importer_table
+      @importers = Importer.order(table_order).page(table_page).per(table_per_page)
+      @importers = @importers.where(importer_table_search) if importer_table_search.present?
+      respond_to do |format|
+        format.json { render json: format_importers(@importers) }
+      end
+    end
+
     # GET /importers/1
     def show
       if api_request?
@@ -37,9 +43,15 @@ module Bulkrax
         add_importer_breadcrumbs
         add_breadcrumb @importer.name
       end
-      @work_entries = @importer.entries.where(type: @importer.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
-      @collection_entries = @importer.entries.where(type: @importer.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
-      @file_set_entries = @importer.entries.where(type: @importer.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
+      @first_entry = @importer.entries.first
+    end
+
+    def entry_table
+      @entries = @importer.entries.order(table_order).page(table_page).per(table_per_page)
+      @entries = @entries.where(entry_table_search) if entry_table_search.present?
+      respond_to do |format|
+        format.json { render json: format_entries(@entries, @importer) }
+      end
     end
 
     # GET /importers/new
@@ -202,10 +214,11 @@ module Bulkrax
       return if file.blank? && cloud_files.blank?
       @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
       if cloud_files.present?
+        @importer[:parser_fields]['cloud_file_paths'] = cloud_files
         # For BagIt, there will only be one bag, so we get the file_path back and set import_file_path
         # For CSV, we expect only file uploads, so we won't get the file_path back
         # and we expect the import_file_path to be set already
-        target = @importer.parser.retrieve_cloud_files(cloud_files)
+        target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
         @importer[:parser_fields]['import_file_path'] = target if target.present?
       end
       @importer.save
@@ -213,7 +226,7 @@ module Bulkrax
 
     # Use callbacks to share common setup or constraints between actions.
     def set_importer
-      @importer = Importer.find(params[:id])
+      @importer = Importer.find(params[:id] || params[:importer_id])
     end
 
     def importable_params
@@ -221,7 +234,7 @@ module Bulkrax
     end
 
     def importable_parser_fields
-      params&.[](:importer)&.[](:parser_fields)&.except(:file)&.keys
+      params&.[](:importer)&.[](:parser_fields)&.except(:file, :entry_statuses)&.keys&. + [{ "entry_statuses" => [] }]
     end
 
     # Only allow a trusted parameters through.
@@ -310,10 +323,16 @@ module Bulkrax
     end
 
     def set_files_parser_fields
+      @importer.parser_fields['update_files'] =
+        @importer.parser_fields['replace_files'] =
+          @importer.parser_fields['remove_and_rerun'] =
+            @importer.parser_fields['metadata_only'] = false
       if params[:commit] == 'Update Metadata and Files'
         @importer.parser_fields['update_files'] = true
       elsif params[:commit] == ('Update and Replace Files' || 'Update and Re-Harvest All Items')
         @importer.parser_fields['replace_files'] = true
+      elsif params[:commit] == 'Remove and Rerun'
+        @importer.parser_fields['remove_and_rerun'] = true
       elsif params[:commit] == 'Update and Harvest Updated Items'
         return
       else