bulkrax 9.3.4 → 9.4.0

data/app/controllers/bulkrax/guided_imports_controller.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class GuidedImportsController < ::Bulkrax::ApplicationController
+    include Hyrax::ThemedLayoutController if defined?(::Hyrax)
+    include Bulkrax::GuidedImportDemoScenarios if Bulkrax.config.guided_import_demo_scenarios_enabled
+    include Bulkrax::ImporterFileHandler
+    helper Bulkrax::ImportersHelper
+
+    before_action :authenticate_user!
+    before_action :check_permissions
+    with_themed_layout 'dashboard' if defined?(::Hyrax)
+
+    # trigger form to allow upload
+    def new
+      @importer = Importer.new
+      return unless defined?(::Hyrax)
+      add_importer_breadcrumbs
+      add_breadcrumb I18n.t('bulkrax.importer.guided_import.breadcrumb')
+    end
+
+    # AJAX endpoint to validate uploaded files
+    def validate
+      set_locale_from_params
+
+      files, error = resolve_validation_files
+      return render json: error, status: :ok if error
+      return render json: StepperResponseFormatter.error(message: I18n.t('bulkrax.importer.guided_import.validation.no_files_uploaded')), status: :ok unless files.any?
+
+      csv_file, zip_file = select_csv_and_zip(files)
+
+      unless csv_file
+        return render json: StepperResponseFormatter.error(message: I18n.t('bulkrax.importer.guided_import.validation.no_csv_uploaded')), status: :ok unless zip_file
+
+        csv_file, error = extract_csv_from_zip(zip_file)
+        return render json: error, status: :ok if error
+      end
+
+      admin_set_id = params[:importer]&.[](:admin_set_id)
+      validation_result = run_validation(csv_file, zip_file, admin_set_id: admin_set_id)
+      raw_csv_data = validation_result.delete(:raw_csv_data)
+      cache_key = cache_validation_errors(validation_result, raw_csv_data, csv_file)
+      formatted = StepperResponseFormatter.format(validation_result)
+      formatted[:validationErrorsCacheKey] = cache_key
+      render json: formatted, status: :ok
+    ensure
+      close_file_handles(files)
+    end
+
+    def download_validation_errors
+      cache_key = params[:key].to_s
+      expected_prefix = "guided_import_errors:#{session.id}:"
+      return head :not_found unless cache_key.start_with?(expected_prefix)
+
+      cached = Rails.cache.read(cache_key)
+      return head :not_found unless cached
+
+      csv = ValidationErrorCsvBuilder.build(
+        headers: cached[:headers],
+        csv_data: cached[:csv_data],
+        row_errors: cached[:row_errors],
+        file_errors: cached[:file_errors]
+      )
+      send_data csv, filename: error_csv_filename(cached[:original_filename]), type: 'text/csv', disposition: 'attachment'
+    end
+
+    def create
+      files = nil
+      files = resolve_create_files
+      return render_invalid_uploaded_files_response if params[:uploaded_files].present? && files.empty?
+
+      @importer = Importer.new(importer_params)
+      @importer.parser_klass = 'Bulkrax::CsvParser'
+      @importer.user = current_user if respond_to?(:current_user) && current_user.present?
+      apply_field_mapping
+
+      if @importer.save
+        write_files(files)
+        Bulkrax::ImporterJob.perform_later(@importer.id)
+
+        respond_to do |format|
+          format.html { redirect_to bulkrax.importers_path, notice: I18n.t('bulkrax.importer.guided_import.flash.import_started') }
+          format.json { render json: { success: true, importer_id: @importer.id }, status: :created }
+        end
+      else
+        respond_to do |format|
+          format.html { render :new, status: :unprocessable_entity }
+          format.json { render json: { errors: @importer.errors.full_messages }, status: :unprocessable_entity }
+        end
+      end
+    ensure
+      close_file_handles(files)
+    end
+
+    private
+
+    def render_invalid_uploaded_files_response
+      respond_to do |format|
+        format.html { render :new, status: :unprocessable_entity }
+        format.json { render json: { errors: ['No valid uploaded files found'] }, status: :unprocessable_entity }
+      end
+    end
+
+    # Runs validation via the real service.
+    # @param csv_file [File, StringIO] the CSV to validate
+    # @param zip_file [File, nil] an optional ZIP containing file attachments
+    # @param admin_set_id [String, nil] optional admin set ID for validation context
+    # @return [Hash] validation result data
+    def cache_validation_errors(validation_result, raw_csv_data, csv_file)
+      has_errors = validation_result[:rowErrors]&.any? ||
+                   validation_result[:missingRequired]&.any? ||
+                   validation_result[:unrecognized]&.any? ||
+                   validation_result[:emptyColumns]&.any? ||
+                   validation_result[:missingFiles]&.any?
+      return nil unless has_errors
+
+      key = "guided_import_errors:#{session.id}:#{Time.now.to_i}"
+      Rails.cache.write(
+        key,
+        {
+          headers: validation_result[:headers],
+          csv_data: raw_csv_data,
+          row_errors: validation_result[:rowErrors] || [],
+          file_errors: {
+            missing_required: validation_result[:missingRequired] || [],
+            unrecognized: validation_result[:unrecognized] || {},
+            empty_columns: validation_result[:emptyColumns] || [],
+            missing_files: validation_result[:missingFiles] || []
+          },
+          original_filename: filename_for(csv_file)
+        },
+        expires_in: 1.hour
+      )
+      key
+    end
+
+    def run_validation(csv_file, zip_file, admin_set_id: nil)
+      CsvParser.validate_csv(csv_file: csv_file, zip_file: zip_file, admin_set_id: admin_set_id)
+    end
+
+    def importer_params
+      params.require(:importer).permit(
+        :name,
+        :admin_set_id,
+        :limit,
+        parser_fields: [:visibility, :rights_statement, :override_rights_statement, :import_file_path, :file_style]
+      )
+    end
+
+    def apply_field_mapping
+      @importer.field_mapping = Bulkrax.field_mappings['Bulkrax::CsvParser']
+    end
+
+    def error_csv_filename(original_filename)
+      return 'import_errors.csv' if original_filename.blank?
+
+      base = File.basename(original_filename, '.*')
+      "#{base}_errors.csv"
+    end
+
+    def set_locale_from_params
+      I18n.locale = params[:locale] if params[:locale].present? && I18n.available_locales.include?(params[:locale].to_sym)
+    end
+
+    def add_importer_breadcrumbs
+      add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
+      add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
+      add_breadcrumb 'Importers', bulkrax.importers_path
+    end
+
+    def check_permissions
+      raise CanCan::AccessDenied unless current_ability.can_import_works?
+    end
+  end
+end

data/app/controllers/bulkrax/importers_controller.rb

@@ -8,6 +8,7 @@ module Bulkrax
     include Bulkrax::API
     include Bulkrax::DatatablesBehavior
     include Bulkrax::ValidationHelper
+    include Bulkrax::ImporterFileHandler
 
     protect_from_forgery unless: -> { api_request? }
     before_action :token_authenticate!, if: -> { api_request? }, only: [:create, :update, :delete]
@@ -28,7 +29,8 @@ module Bulkrax
     end
 
     def importer_table
-      @importers = Importer.order(table_order).page(table_page).per(table_per_page)
+      order = table_order.presence || Arel.sql('last_imported_at DESC NULLS LAST')
+      @importers = Importer.order(order).page(table_page).per(table_per_page)
       @importers = @importers.where(importer_table_search) if importer_table_search.present?
       respond_to do |format|
         format.json { render json: format_importers(@importers) }
@@ -65,6 +67,16 @@ module Bulkrax
       end
     end
 
+    # GET /importers/sample_csv_file
+    def sample_csv_file
+      admin_set_id = params[:admin_set_id].presence
+      sample = Bulkrax::CsvParser.generate_template(models: 'all', output: 'file', admin_set_id: admin_set_id)
+      send_file sample, filename: File.basename(sample), type: 'text/csv', disposition: 'attachment'
+    rescue StandardError => e
+      flash[:error] = "Unable to generate sample CSV file: #{e.message}"
+      redirect_back fallback_location: bulkrax.importers_path
+    end
+
     # GET /importers/1/edit
     def edit
       if api_request?
@@ -84,7 +96,7 @@ module Bulkrax
       if api_request?
         return return_json_response unless valid_create_params?
       end
-      uploads = Hyrax::UploadedFile.find(params[:uploaded_files]) if params[:uploaded_files].present?
+      uploads = uploaded_files_scope
       file = file_param
       cloud_files = cloud_params
 
@@ -123,7 +135,7 @@ module Bulkrax
       if api_request?
         return return_json_response unless valid_update_params?
       end
-      uploads = Hyrax::UploadedFile.find(params[:uploaded_files]) if params[:uploaded_files].present?
+      uploads = uploaded_files_scope
       file = file_param
       cloud_files = cloud_params
 
@@ -205,10 +217,26 @@ module Bulkrax
     end
 
     def original_file
-      if @importer.original_file?
-        send_file @importer.original_file
-      else
+      file_type = params[:file_type]&.to_sym
+
+      files = @importer.original_files
+      if files.empty?
         redirect_to @importer, alert: 'Importer does not support file re-download or the imported file is not found on the server.'
+        return
+      end
+
+      # If file_type is specified, find that specific file
+      if file_type
+        file = files.find { |f| f[:type] == file_type }
+        if file
+          send_file file[:path], filename: file[:name], disposition: 'attachment'
+        else
+          redirect_to @importer, alert: "File type '#{file_type}' not found."
+        end
+      else
+        # Default behavior: send the first file (CSV) for backward compatibility
+        file = files.first
+        send_file file[:path], filename: file[:name], disposition: 'attachment'
       end
     end
 
@@ -221,28 +249,6 @@ module Bulkrax
 
     private
 
-    def files_for_import(file, cloud_files, uploads)
-      return if file.blank? && cloud_files.blank? && uploads.blank?
-
-      @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
-      if cloud_files.present?
-        @importer[:parser_fields]['cloud_file_paths'] = cloud_files
-        # For BagIt, there will only be one bag, so we get the file_path back and set import_file_path
-        # For CSV, we expect only file uploads, so we won't get the file_path back
-        # and we expect the import_file_path to be set already
-        target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
-        @importer[:parser_fields]['import_file_path'] = target if target.present?
-      end
-
-      if uploads.present?
-        uploads.each do |upload|
-          @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(upload.file.file)
-        end
-      end
-
-      @importer.save
-    end
-
     # Use callbacks to share common setup or constraints between actions.
     def set_importer
       @importer = Importer.find(params[:id] || params[:importer_id])

data/app/controllers/concerns/bulkrax/guided_import_demo_scenarios.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # rubocop:disable Metrics/ModuleLength
+  module GuidedImportDemoScenarios
+    extend ActiveSupport::Concern
+
+    # Serve demo scenario fixtures for frontend testing
+    def demo_scenarios
+      file_path = Bulkrax::Engine.root.join('lib', 'bulkrax', 'data', 'demo_scenarios.json')
+      if File.exist?(file_path)
+        render json: File.read(file_path), status: :ok
+      else
+        render json: { error: I18n.t('bulkrax.importer.guided_import.flash.demo_not_available') }, status: :not_found
+      end
+    end
+
+    private
+
+    def run_validation(csv_file, zip_file, admin_set_id: nil)
+      if ENV['DEMO_MODE'] == 'true'
+        generate_validation_response(csv_file, zip_file)
+      else
+        super
+      end
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    # Hardcoded mock response generator for demo mode
+    def generate_validation_response(_csv_file, zip_file)
+      # Generate mock collections
+      collections = [
+        { id: 'col-1', title: 'Historical Photographs Collection', type: 'collection', parentIds: [], childrenIds: ['work-shared-1'] },
+        { id: 'col-2', title: 'Manuscripts & Letters', type: 'collection', parentIds: [], childrenIds: [] },
+        { id: 'col-3', title: 'Audio Recordings', type: 'collection', parentIds: [], childrenIds: ['work-shared-2'] }
+      ]
+
+      # Generate mock works
+      works = []
+      189.times do |i|
+        parent_ids = if i < 75
+                       ['col-1']
+                     elsif i < 140
+                       ['col-2']
+                     elsif i < 189
+                       ['col-3']
+                     end
+
+        works << {
+          id: "work-#{i + 1}",
+          title: "Work #{i + 1}",
+          type: 'work',
+          parentIds: parent_ids
+        }
+      end
+
+      # Multi-parent examples
+      works << { id: 'work-shared-1', title: 'Cross-Collection Photograph', type: 'work', parentIds: ['col-1', 'col-2'] }
+      works << { id: 'work-shared-2', title: 'Interdisciplinary Recording', type: 'work', parentIds: ['col-2', 'col-3'] }
+
+      # Generate mock file sets
+      file_sets = []
+      55.times do |i|
+        file_sets << {
+          id: "fs-#{i + 1}",
+          title: "FileSet #{i + 1}",
+          type: 'file_set'
+        }
+      end
+
+      # Mock headers with one unrecognized field
+      headers = ['source_identifier', 'title', 'creator', 'model', 'parents', 'children', 'file', 'description', 'date_created', 'legacy_id', 'subject']
+      unrecognized = ['legacy_id']
+      missing_required = []
+      missing_files = ['photo_087.tiff', 'letter_scan_12.pdf', 'recording_03.wav']
+      zip_included = zip_file.present?
+
+      {
+        headers: headers,
+        missingRequired: missing_required,
+        unrecognized: unrecognized,
+        rowCount: 247,
+        isValid: true,
+        hasWarnings: true,
+        collections: collections,
+        works: works,
+        fileSets: file_sets,
+        totalItems: collections.length + works.length + file_sets.length,
+        fileReferences: 55,
+        missingFiles: missing_files,
+        foundFiles: 52,
+        zipIncluded: zip_included,
+        messages: build_validation_messages(
+          headers: headers, unrecognized: unrecognized, missing_required: missing_required,
+          missing_files: missing_files, zip_included: zip_included, row_count: 247,
+          is_valid: true, has_warnings: true, file_references: 55
+        )
+      }
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    # Builds the structured messages hash from validation results.
+    # @param results [Hash] with keys: headers, unrecognized, missing_required,
+    #   missing_files, zip_included, row_count, is_valid, has_warnings, file_references
+    def build_validation_messages(results)
+      issues = []
+      issues << missing_required_issue(results[:missing_required]) if results[:missing_required]&.any?
+      issues << unrecognized_fields_issue(results[:unrecognized]) if results[:unrecognized]&.any?
+      issues << file_references_issue(results) if results[:file_references]&.positive?
+
+      {
+        validationStatus: validation_status(results),
+        issues: issues.compact
+      }
+    end
+
+    def validation_status(results)
+      severity, icon, title = validation_status_level(results[:is_valid], results[:has_warnings])
+      recognized = results[:headers] - (results[:unrecognized] || [])
+
+      {
+        severity: severity,
+        icon: icon,
+        title: title,
+        summary: I18n.t('bulkrax.importer.guided_import.validation.columns_detected', columns: results[:headers].length, records: results[:row_count]),
+        details: results[:is_valid] ? I18n.t('bulkrax.importer.guided_import.validation.recognized_fields', fields: recognized.join(', ')) : I18n.t('bulkrax.importer.guided_import.validation.critical_errors'),
+        defaultOpen: true
+      }
+    end
+
+    def validation_status_level(is_valid, has_warnings)
+      if !is_valid
+        ['error', 'fa-times-circle', I18n.t('bulkrax.importer.guided_import.validation.failed')]
+      elsif has_warnings
+        ['warning', 'fa-exclamation-triangle', I18n.t('bulkrax.importer.guided_import.validation.passed_warnings')]
+      else
+        ['success', 'fa-check-circle', I18n.t('bulkrax.importer.guided_import.validation.passed')]
+      end
+    end
+
+    def missing_required_issue(missing_required)
+      {
+        type: 'missing_required_fields',
+        severity: 'error',
+        icon: 'fa-times-circle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.missing_required_title'),
+        count: missing_required.length,
+        description: I18n.t('bulkrax.importer.guided_import.validation.missing_required_desc'),
+        items: missing_required.map { |field| { field: field, message: I18n.t('bulkrax.importer.guided_import.validation.missing_required_hint') } },
+        defaultOpen: false
+      }
+    end
+
+    def unrecognized_fields_issue(unrecognized)
+      {
+        type: 'unrecognized_fields',
+        severity: 'warning',
+        icon: 'fa-exclamation-triangle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.unrecognized_title'),
+        count: unrecognized.length,
+        description: I18n.t('bulkrax.importer.guided_import.validation.unrecognized_desc'),
+        items: unrecognized.map { |field| { field: field, message: nil } },
+        defaultOpen: false
+      }
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def file_references_issue(results)
+      file_references = results[:file_references]
+      missing_files = results[:missing_files] || []
+      found_files = file_references - missing_files.length
+
+      if missing_files.any? && results[:zip_included]
+        {
+          type: 'file_references',
+          severity: 'warning',
+          icon: 'fa-info-circle',
+          title: I18n.t('bulkrax.importer.guided_import.validation.file_references_title'),
+          count: file_references,
+          summary: I18n.t('bulkrax.importer.guided_import.validation.files_found_in_zip', found: found_files, total: file_references),
+          description: I18n.t('bulkrax.importer.guided_import.validation.files_missing_from_zip', count: missing_files.length, files_word: 'file'.pluralize(missing_files.length)),
+          items: missing_files.map { |file| { field: file, message: I18n.t('bulkrax.importer.guided_import.validation.missing_from_zip') } },
+          defaultOpen: false
+        }
+      elsif !results[:zip_included]
+        {
+          type: 'file_references',
+          severity: 'warning',
+          icon: 'fa-exclamation-triangle',
+          title: I18n.t('bulkrax.importer.guided_import.validation.file_references_title'),
+          count: file_references,
+          summary: I18n.t('bulkrax.importer.guided_import.validation.files_referenced', count: file_references),
+          description: I18n.t('bulkrax.importer.guided_import.validation.no_zip_desc'),
+          items: [],
+          defaultOpen: false
+        }
+      end
+    end # rubocop:enable Metrics/MethodLength
+  end
+  # rubocop:enable Metrics/ModuleLength
+end