bulkrax 1.0.0

data/app/assets/stylesheets/bulkrax/import_export.scss

@@ -0,0 +1,37 @@
+div.importer_parser_fields_file_style span.radio {
+  display: inline;
+  margin-right: 20px;
+}
+
+div#file_upload, div#file_path {
+  display: none;
+}
+
+div.hide-required-tag .required-tag {
+  display: none;
+}
+
+div#s2id_exporter_export_source_collection {
+  border: none;
+  padding: 0;
+}
+
+.bulkrax-nav-tab-top-margin {
+  margin-top: 20px !important;
+}
+
+.bulkrax-nav-tab-bottom-margin {
+  margin-bottom: 20px;
+}
+
+.bulkrax-nav-tab-table-left-align {
+  margin-left: 10px;
+}
+
+.bulkrax-p-align {
+  margin: 15px 0 10px 15px;
+}
+
+.bulkrax-clear-toggles {
+  clear: both;
+}

data/app/controllers/bulkrax/application_controller.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ApplicationController < ::ApplicationController
+    helper Rails.application.class.helpers
+    protect_from_forgery with: :exception
+  end
+end

data/app/controllers/bulkrax/entries_controller.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_dependency "bulkrax/application_controller"
+require_dependency "oai"
+
+module Bulkrax
+  class EntriesController < ApplicationController
+    include Hyrax::ThemedLayoutController
+    before_action :authenticate_user!
+    with_themed_layout 'dashboard'
+
+    def show
+      if params[:importer_id].present?
+        show_importer
+      elsif params[:exporter_id].present?
+        show_exporter
+      end
+    end
+
+    # GET /importers/1/entries/1
+    def show_importer
+      @importer = Importer.find(params[:importer_id])
+      @entry = Entry.find(params[:id])
+
+      add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
+      add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
+      add_breadcrumb 'Importers', bulkrax.importers_path
+      add_breadcrumb @importer.name, bulkrax.importer_path(@importer.id)
+      add_breadcrumb @entry.id
+    end
+
+    # GET /exporters/1/entries/1
+    def show_exporter
+      @exporter = Exporter.find(params[:exporter_id])
+      @entry = Entry.find(params[:id])
+
+      add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
+      add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
+      add_breadcrumb 'Exporters', bulkrax.exporters_path
+      add_breadcrumb @exporter.name, bulkrax.exporter_path(@exporter.id)
+      add_breadcrumb @entry.id
+    end
+  end
+end

data/app/controllers/bulkrax/exporters_controller.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require_dependency "bulkrax/application_controller"
+
+module Bulkrax
+  class ExportersController < ApplicationController
+    include Hyrax::ThemedLayoutController
+    include Bulkrax::DownloadBehavior
+    before_action :authenticate_user!
+    before_action :set_exporter, only: [:show, :edit, :update, :destroy]
+    with_themed_layout 'dashboard'
+
+    # GET /exporters
+    def index
+      @exporters = Exporter.all
+
+      add_exporter_breadcrumbs
+    end
+
+    # GET /exporters/1
+    def show
+      add_exporter_breadcrumbs
+      add_breadcrumb @exporter.name
+
+      @work_entries = @exporter.entries.where(type: @exporter.parser.entry_class.to_s).page(params[:work_entries_page])
+    end
+
+    # GET /exporters/new
+    def new
+      @exporter = Exporter.new
+
+      add_exporter_breadcrumbs
+      add_breadcrumb 'New'
+    end
+
+    # GET /exporters/1/edit
+    def edit
+      add_exporter_breadcrumbs
+      add_breadcrumb @exporter.name, bulkrax.exporter_path(@exporter.id)
+      add_breadcrumb 'Edit'
+
+      # Correctly populate export_source_collection input
+      @collection = Collection.find(@exporter.export_source) if @exporter.export_source.present? && @exporter.export_from == 'collection'
+    end
+
+    # POST /exporters
+    def create
+      @exporter = Exporter.new(exporter_params)
+      field_mapping_params
+
+      if @exporter.save
+        if params[:commit] == 'Create and Export'
+          # Use perform now for export
+          Bulkrax::ExporterJob.perform_now(@exporter.id)
+        end
+        redirect_to exporters_path, notice: 'Exporter was successfully created.'
+      else
+        render :new
+      end
+    end
+
+    # PATCH/PUT /exporters/1
+    def update
+      field_mapping_params
+      if @exporter.update(exporter_params)
+        Bulkrax::ExporterJob.perform_now(@exporter.id) if params[:commit] == 'Update and Re-Export All Items'
+        redirect_to exporters_path, notice: 'Exporter was successfully updated.'
+      else
+        render :edit
+      end
+    end
+
+    # DELETE /exporters/1
+    def destroy
+      @exporter.destroy
+      redirect_to exporters_url, notice: 'Exporter was successfully destroyed.'
+    end
+
+    # GET /exporters/1/download
+    def download
+      @exporter = Exporter.find(params[:exporter_id])
+      send_content
+    end
+
+    private
+
+    # Use callbacks to share common setup or constraints between actions.
+    def set_exporter
+      @exporter = Exporter.find(params[:id])
+    end
+
+    # Only allow a trusted parameters through.
+    def exporter_params
+      params[:exporter][:export_source] = params[:exporter]["export_source_#{params[:exporter][:export_from]}".to_sym]
+      if params[:exporter][:date_filter] == "1"
+        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type,
+                                       :parser_klass, :limit, :start_date, :finish_date, :work_visibility,
+                                       :workflow_status, field_mapping: {})
+      else
+        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type,
+                                       :parser_klass, :limit, :work_visibility, :workflow_status,
+                                       field_mapping: {}).merge(start_date: nil, finish_date: nil)
+      end
+    end
+
+    # Add the field_mapping from the Bulkrax configuration
+    def field_mapping_params
+      # @todo replace/append once mapping GUI is in place
+      field_mapping_key = Bulkrax.parsers.map { |m| m[:class_name] if m[:class_name] == params[:exporter][:parser_klass] }.compact.first
+      @exporter.field_mapping = Bulkrax.field_mappings[field_mapping_key] if field_mapping_key
+    end
+
+    def add_exporter_breadcrumbs
+      add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
+      add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
+      add_breadcrumb 'Exporters', bulkrax.exporters_path
+    end
+
+    # Download methods
+
+    def file_path
+      @exporter.exporter_export_zip_path
+    end
+  end
+end

data/app/controllers/bulkrax/importers_controller.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+require_dependency 'bulkrax/application_controller'
+require_dependency 'oai'
+
+module Bulkrax
+  # rubocop:disable Metrics/ClassLength
+  class ImportersController < ApplicationController
+    include Hyrax::ThemedLayoutController
+    include Bulkrax::DownloadBehavior
+    include Bulkrax::API
+    include Bulkrax::ValidationHelper
+
+    protect_from_forgery unless: -> { api_request? }
+    before_action :token_authenticate!, if: -> { api_request? }, only: [:create, :update, :delete]
+    before_action :authenticate_user!, unless: -> { api_request? }
+    before_action :set_importer, only: [:show, :edit, :update, :destroy]
+    with_themed_layout 'dashboard'
+
+    # GET /importers
+    def index
+      @importers = Importer.all
+      if api_request?
+        json_response('index')
+      else
+        add_importer_breadcrumbs
+      end
+    end
+
+    # GET /importers/1
+    def show
+      if api_request?
+        json_response('show')
+      else
+        add_importer_breadcrumbs
+        add_breadcrumb @importer.name
+
+        @work_entries = @importer.entries.where(type: @importer.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
+        @collection_entries = @importer.entries.where(type: @importer.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
+      end
+    end
+
+    # GET /importers/new
+    def new
+      @importer = Importer.new
+      if api_request?
+        json_response('new')
+      else
+        add_importer_breadcrumbs
+        add_breadcrumb 'New'
+      end
+    end
+
+    # GET /importers/1/edit
+    def edit
+      if api_request?
+        json_response('edit')
+      else
+        add_importer_breadcrumbs
+        add_breadcrumb @importer.name, bulkrax.importer_path(@importer.id)
+        add_breadcrumb 'Edit'
+      end
+    end
+
+    # POST /importers
+    # rubocop:disable Metrics/MethodLength
+    def create
+      # rubocop:disable Style/IfInsideElse
+      if api_request?
+        return return_json_response unless valid_create_params?
+      end
+      file = file_param
+      cloud_files = cloud_params
+
+      @importer = Importer.new(importer_params)
+      field_mapping_params
+      @importer.validate_only = true if params[:commit] == 'Create and Validate'
+      if @importer.save
+        files_for_import(file, cloud_files)
+        if params[:commit] == 'Create and Import'
+          Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
+          render_request('Importer was successfully created and import has been queued.')
+        elsif params[:commit] == 'Create and Validate'
+          Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
+          render_request('Importer validation completed. Please review and choose to either Continue with or Discard the import.', true)
+        else
+          render_request('Importer was successfully created.')
+        end
+      else
+        if api_request?
+          json_response('create', :unprocessable_entity)
+        else
+          render :new
+        end
+      end
+      # rubocop:enable Style/IfInsideElse
+    end
+
+    # PATCH/PUT /importers/1
+    # # @todo refactor so as to not need to disable rubocop
+    # rubocop:disable all
+    def update
+      if api_request?
+        return return_json_response unless valid_update_params?
+      end
+
+      file = file_param
+      cloud_files = cloud_params
+
+      # Skipped during a continue
+      field_mapping_params if params[:importer][:parser_fields].present?
+
+      if @importer.update(importer_params)
+        files_for_import(file, cloud_files) unless file.nil? && cloud_files.nil?
+        # do not perform the import
+        if params[:commit] == 'Update Importer'
+        # do nothing
+        # OAI-only - selective re-harvest
+        elsif params[:commit] == 'Update and Harvest Updated Items'
+          Bulkrax::ImporterJob.perform_later(@importer.id, true)
+        elsif params[:commit] == 'Update All (update metadata and update files)'
+          @importer.parser_fields['update_files'] = true
+          @importer.save
+          Bulkrax::ImporterJob.perform_later(@importer.id)
+          # Perform a full metadata and files re-import; do the same for an OAI re-harvest of all items
+        elsif params[:commit] == ('Update and Replace (update metadata and replace files)' || 'Update and Re-Harvest All Items')
+          @importer.parser_fields['replace_files'] = true
+          @importer.save
+          Bulkrax::ImporterJob.perform_later(@importer.id)
+        # In all other cases, perform a metadata-only re-import
+        else
+          Bulkrax::ImporterJob.perform_later(@importer.id)
+        end
+        if api_request?
+          json_response('updated', :ok, 'Importer was successfully updated.')
+        else
+          redirect_to importers_path, notice: 'Importer was successfully updated.'
+        end
+      else
+        if api_request?
+          json_response('update', :unprocessable_entity, 'Something went wrong.')
+        else
+          render :edit
+        end
+      end
+    end
+    # rubocop:enable all
+    # rubocop:enable Metrics/MethodLength
+
+    # DELETE /importers/1
+    def destroy
+      @importer.destroy
+      if api_request?
+        json_response('destroy', :ok, notice: 'Importer was successfully destroyed.')
+      else
+        redirect_to importers_url, notice: 'Importer was successfully destroyed.'
+      end
+    end
+
+    # PUT /importers/1
+    def continue
+      @importer = Importer.find(params[:importer_id])
+      params[:importer] = { name: @importer.name }
+      @importer.validate_only = false
+      update
+    end
+
+    # GET /importer/1/upload_corrected_entries
+    def upload_corrected_entries
+      @importer = Importer.find(params[:importer_id])
+      add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
+      add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
+      add_breadcrumb 'Importers', bulkrax.importers_path
+      add_breadcrumb @importer.name, bulkrax.importer_path(@importer.id)
+      add_breadcrumb 'Upload Corrected Entries'
+    end
+
+    # POST /importer/1/upload_corrected_entries_file
+    def upload_corrected_entries_file
+      file = params[:importer][:parser_fields].delete(:file)
+      @importer = Importer.find(params[:importer_id])
+      if file.present?
+        @importer[:parser_fields]['partial_import_file_path'] = @importer.parser.write_partial_import_file(file)
+        @importer.save
+        Bulkrax::ImporterJob.perform_later(@importer.id, true)
+        redirect_to importer_path(@importer), notice: 'Corrected entries uploaded successfully.'
+      else
+        redirect_to importer_upload_corrected_entries_path(@importer), alert: 'Importer failed to update with new file.'
+      end
+    end
+
+    def external_sets
+      if list_external_sets
+        render json: { base_url: params[:base_url], sets: @sets }
+      else
+        render json: { base_url: params[:base_url], error: "unable to pull data from #{params[:base_url]}" }
+      end
+    end
+
+    # GET /importers/1/export_errors
+    def export_errors
+      @importer = Importer.find(params[:importer_id])
+      @importer.write_errored_entries_file
+      send_content
+    end
+
+    private
+
+    def files_for_import(file, cloud_files)
+      return if file.blank? && cloud_files.blank?
+      @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
+      if cloud_files.present?
+        # For BagIt, there will only be one bag, so we get the file_path back and set import_file_path
+        # For CSV, we expect only file uploads, so we won't get the file_path back
+        # and we expect the import_file_path to be set already
+        target = @importer.parser.retrieve_cloud_files(cloud_files)
+        @importer[:parser_fields]['import_file_path'] = target if target.present?
+      end
+      @importer.save
+    end
+
+    # Use callbacks to share common setup or constraints between actions.
+    def set_importer
+      @importer = Importer.find(params[:id])
+    end
+
+    def importable_params
+      params.except(:selected_files)
+    end
+
+    def importable_parser_fields
+      params&.[](:importer)&.[](:parser_fields)&.except(:file)&.keys
+    end
+
+    # Only allow a trusted parameters through.
+    def importer_params
+      importable_params.require(:importer).permit(
+        :name,
+        :admin_set_id,
+        :user_id,
+        :frequency,
+        :parser_klass,
+        :limit,
+        :validate_only,
+        selected_files: {},
+        field_mapping: {},
+        parser_fields: [importable_parser_fields]
+      )
+    end
+
+    def list_external_sets
+      url = params[:base_url] || (@harvester ? @harvester.base_url : nil)
+      setup_client(url) if url.present?
+
+      @sets = [['All', 'all']]
+
+      begin
+        @client.list_sets.each do |s|
+          @sets << [s.name, s.spec]
+        end
+      rescue
+        return false
+      end
+
+      @sets
+    end
+
+    def file_param
+      params.require(:importer).require(:parser_fields).fetch(:file) if params&.[](:importer)&.[](:parser_fields)&.[](:file)
+    end
+
+    def cloud_params
+      params.permit(selected_files: {}).fetch(:selected_files).to_h if params&.[](:selected_files)
+    end
+
+    # Add the field_mapping from the Bulkrax configuration
+    def field_mapping_params
+      # @todo replace/append once mapping GUI is in place
+      field_mapping_key = Bulkrax.parsers.map { |m| m[:class_name] if m[:class_name] == params[:importer][:parser_klass] }.compact.first
+      @importer.field_mapping = Bulkrax.field_mappings[field_mapping_key] if field_mapping_key
+    end
+
+    def add_importer_breadcrumbs
+      add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
+      add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
+      add_breadcrumb 'Importers', bulkrax.importers_path
+    end
+
+    def setup_client(url)
+      return false if url.nil?
+      headers = { from: Bulkrax.server_name }
+      @client ||= OAI::Client.new(url, headers: headers, parser: 'libxml', metadata_prefix: 'oai_dc')
+    end
+
+    # Download methods
+
+    def file_path
+      @importer.errored_entries_csv_path
+    end
+
+    def download_content_type
+      'text/csv'
+    end
+
+    def render_request(message, validate_only = false)
+      if api_request?
+        json_response('create', :created, message)
+      else
+        path = validate_only ? importer_path(@importer) : importers_path
+        redirect_to path, notice: message
+      end
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end