bulkrax 9.3.5 → 9.4.1

data/app/services/bulkrax/stepper_response_formatter.rb

@@ -0,0 +1,347 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  ##
+  # Formats validation data from CsvParser.validate_csv into the structure
+  # expected by the importers_stepper.js frontend component.
+  #
+  # This service acts as a presentation layer, transforming raw validation data
+  # into a structured response with proper status messages, severity levels,
+  # and formatted issue lists that the JavaScript can render correctly.
+  #
+  # @example Basic usage
+  #   validation_data = CsvParser.validate_csv(csv_file: file, zip_file: zip)
+  #   formatted_response = StepperResponseFormatter.format(validation_data)
+  #   render json: formatted_response
+  #
+  # @example Error response
+  #   error_response = StepperResponseFormatter.error(message: "Unable to process files")
+  #   render json: error_response, status: :ok
+  #
+  # rubocop:disable Metrics/ClassLength
+  class StepperResponseFormatter
+    # Format validation data for the stepper frontend
+    #
+    # @param data [Hash] Raw validation data from CsvParser.validate_csv containing:
+    #   - headers: Array of CSV column names
+    #   - missingRequired: Array of hashes of missing required fields by model (e.g. {model: 'GenericWork', field: 'source_identifier'})
+    #   - unrecognized: Array of unrecognized column names
+    #   - rowCount: Total number of data rows
+    #   - isValid: Boolean indicating validation success
+    #   - hasWarnings: Boolean indicating presence of warnings
+    #   - collections: Array of collection items with id, title, type, parentIds (array), childIds (array)
+    #   - works: Array of work items with id, title, type, parentIds (array), childIds (array)
+    #   - fileSets: Array of file set items
+    #   - totalItems: Total count of items
+    #   - fileReferences: Count of file references
+    #   - missingFiles: Array of missing file names
+    #   - foundFiles: Count of found files
+    #   - zipIncluded: Boolean indicating if zip was provided
+    # @return [Hash] Formatted response ready for JSON rendering
+    def self.format(data)
+      new(data).format
+    end
+
+    # Generate an error response for validation failures
+    #
+    # @param message [String] Error message to display
+    # @param summary [String] Optional summary (defaults to standard message)
+    # @return [Hash] Error response structure
+    def self.error(message: I18n.t('bulkrax.importer.guided_import.validation.unable_to_process'), summary: nil)
+      {
+        totalItems: 0,
+        collections: [],
+        works: [],
+        fileSets: [],
+        isValid: false,
+        hasWarnings: false,
+        messages: {
+          validationStatus: {
+            severity: 'error',
+            icon: 'fa-times-circle',
+            title: I18n.t('bulkrax.importer.guided_import.validation.failed'),
+            summary: summary || message,
+            details: I18n.t('bulkrax.importer.guided_import.validation.critical_errors'),
+            defaultOpen: true
+          },
+          issues: []
+        }
+      }
+    end
+
+    def initialize(data)
+      @data = data
+    end
+
+    # Format the validation data with messages structure
+    # If data already contains a messages structure, return it as-is
+    #
+    # @return [Hash] Complete formatted response
+    def format
+      # Check if data is already formatted (has messages structure)
+      return @data if already_formatted?
+
+      # Build formatted response with messages structure
+      {
+        headers: @data[:headers],
+        missingRequired: @data[:missingRequired],
+        unrecognized: @data[:unrecognized],
+        rowCount: @data[:rowCount],
+        isValid: @data[:isValid],
+        hasWarnings: @data[:hasWarnings],
+        rowErrors: @data[:rowErrors],
+        collections: @data[:collections],
+        works: @data[:works],
+        fileSets: @data[:fileSets],
+        totalItems: @data[:totalItems],
+        fileReferences: @data[:fileReferences],
+        missingFiles: @data[:missingFiles],
+        foundFiles: @data[:foundFiles],
+        zipIncluded: @data[:zipIncluded],
+        messages: build_messages
+      }
+    end
+
+    private
+
+    # Check if data is already formatted with messages structure
+    #
+    # @return [Boolean] true if data already has proper messages structure
+    def already_formatted?
+      @data.key?(:messages) &&
+        @data[:messages].is_a?(Hash) &&
+        @data[:messages].key?(:validationStatus)
+    end
+
+    # Build the messages structure with validationStatus and issues
+    #
+    # @return [Hash] Messages structure for frontend
+    def build_messages
+      issues = []
+      issues << missing_required_issue if @data[:missingRequired]&.any?
+      issues << notices_issue if @data[:notices]&.any?
+      issues << unrecognized_fields_issue if @data[:unrecognized]&.any? || @data[:emptyColumns]&.any?
+      issues << file_references_issue if @data[:fileReferences]&.positive?
+      issues << row_errors_issue if @data[:rowErrors]&.any? { |e| e[:severity] == 'error' }
+      issues << row_warnings_issue if @data[:rowErrors]&.any? { |e| e[:severity] == 'warning' }
+
+      {
+        validationStatus: validation_status,
+        issues: issues.compact
+      }
+    end
+
+    # Generate the main validation status object
+    #
+    # @return [Hash] Validation status with severity, icon, title, summary, details
+    def validation_status
+      severity, icon, title = determine_severity_level
+      recognized = @data[:headers].reject(&:blank?) - (@data[:unrecognized].keys || [])
+
+      {
+        severity: severity,
+        icon: icon,
+        title: title,
+        summary: I18n.t('bulkrax.importer.guided_import.validation.columns_detected', columns: @data[:headers].length, records: @data[:rowCount]),
+        details: details_message(recognized),
+        defaultOpen: true
+      }
+    end
+
+    # Determine severity level based on validation state
+    #
+    # @return [Array<String>] [severity, icon, title]
+    def determine_severity_level
+      if !@data[:isValid]
+        ['error', 'fa-times-circle', I18n.t('bulkrax.importer.guided_import.validation.failed')]
+      elsif @data[:hasWarnings]
+        ['warning', 'fa-exclamation-triangle', I18n.t('bulkrax.importer.guided_import.validation.passed_warnings')]
+      else
+        ['success', 'fa-check-circle', I18n.t('bulkrax.importer.guided_import.validation.passed')]
+      end
+    end
+
+    # Generate details message for validation status
+    #
+    # @param recognized [Array<String>] List of recognized field names
+    # @return [String] Details message
+    def details_message(recognized)
+      if @data[:isValid]
+        I18n.t('bulkrax.importer.guided_import.validation.recognized_fields', fields: recognized.join(', '))
+      else
+        I18n.t('bulkrax.importer.guided_import.validation.critical_errors')
+      end
+    end
+
+    # Format missing required fields issue
+    #
+    # @return [Hash] Missing required fields issue structure
+    def missing_required_issue
+      only_rights_statement = @data[:missingRequired]&.all? { |h| h[:field].to_s == 'rights_statement' }
+
+      if only_rights_statement
+        {
+          type: 'missing_required_fields',
+          severity: 'warning',
+          icon: 'fa-exclamation-triangle',
+          title: I18n.t('bulkrax.importer.guided_import.validation.missing_required_title'),
+          count: @data[:missingRequired].length,
+          description: I18n.t('bulkrax.importer.guided_import.validation.missing_rights_desc'),
+          items: @data[:missingRequired],
+          defaultOpen: false
+        }
+      else
+        {
+          type: 'missing_required_fields',
+          severity: 'error',
+          icon: 'fa-times-circle',
+          title: I18n.t('bulkrax.importer.guided_import.validation.missing_required_title'),
+          count: @data[:missingRequired].length,
+          description: I18n.t('bulkrax.importer.guided_import.validation.missing_required_desc'),
+          items: @data[:missingRequired],
+          defaultOpen: false
+        }
+      end
+    end
+
+    # Format unrecognized fields issue
+    #
+    # @return [Hash] Unrecognized fields issue structure
+    def unrecognized_fields_issue
+      all_items = unrecognized_fields_issue_items
+      {
+        type: 'unrecognized_fields',
+        severity: 'warning',
+        icon: 'fa-exclamation-triangle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.unrecognized_title'),
+        count: all_items.length,
+        description: I18n.t('bulkrax.importer.guided_import.validation.unrecognized_desc'),
+        items: all_items,
+        defaultOpen: false
+      }
+    end
+
+    def unrecognized_fields_issue_items
+      named = (@data[:unrecognized] || {}).partition(&:last)
+              .flatten(1)
+              .map { |field| { field: field.first, message: field.last ? I18n.t('bulkrax.importer.guided_import.validation.did_you_mean', suggestion: field.last) : nil } }
+      empty = (@data[:emptyColumns] || []).map do |col|
+        { field: I18n.t('bulkrax.importer.guided_import.validation.empty_column', column: col), message: nil }
+      end
+      named + empty
+    end
+
+    # Format file references issue
+    #
+    # @return [Hash, nil] File references issue structure or nil if not applicable
+    def file_references_issue
+      missing_files = @data[:missingFiles] || []
+
+      if missing_files.any? && @data[:zipIncluded]
+        missing_files_issue
+      elsif !@data[:zipIncluded]
+        no_zip_issue
+      end
+    end
+
+    # Format issue for missing files in ZIP
+    #
+    # @return [Hash] Missing files issue structure
+    def missing_files_issue
+      missing_files = @data[:missingFiles]
+
+      {
+        type: 'file_references',
+        severity: 'warning',
+        icon: 'fa-info-circle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.file_references_title'),
+        count: @data[:fileReferences],
+        summary: I18n.t('bulkrax.importer.guided_import.validation.files_found_in_zip', found: @data[:foundFiles], total: @data[:fileReferences]),
+        description: I18n.t('bulkrax.importer.guided_import.validation.files_missing_from_zip', count: missing_files.length, files_word: 'file'.pluralize(missing_files.length)),
+        items: missing_files.map { |file| { field: file, message: I18n.t('bulkrax.importer.guided_import.validation.missing_from_zip') } },
+        defaultOpen: false
+      }
+    end
+
+    # Format issue for no ZIP uploaded
+    #
+    # @return [Hash] No ZIP issue structure
+    def no_zip_issue
+      {
+        type: 'file_references',
+        severity: 'warning',
+        icon: 'fa-exclamation-triangle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.file_references_title'),
+        count: @data[:fileReferences],
+        summary: I18n.t('bulkrax.importer.guided_import.validation.files_referenced', count: @data[:fileReferences]),
+        description: I18n.t('bulkrax.importer.guided_import.validation.no_zip_desc'),
+        items: [],
+        defaultOpen: false
+      }
+    end
+
+    def row_errors_issue
+      entries = filtered_row_errors.select { |e| e[:severity] == 'error' }
+      return nil if entries.empty?
+
+      {
+        type: 'row_level_errors',
+        severity: 'error',
+        icon: 'fa-times-circle',
+        title: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.title_errors'),
+        count: entries.length,
+        description: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.description'),
+        items: row_error_items(entries),
+        defaultOpen: false
+      }
+    end
+
+    def row_warnings_issue
+      entries = filtered_row_errors.select { |e| e[:severity] == 'warning' }
+      return nil if entries.empty?
+
+      {
+        type: 'row_level_warnings',
+        severity: 'warning',
+        icon: 'fa-exclamation-triangle',
+        title: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.title_warnings'),
+        count: entries.length,
+        description: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.description'),
+        items: row_error_items(entries),
+        defaultOpen: false
+      }
+    end
+
+    def notices_issue
+      {
+        type: 'notices',
+        severity: 'warning',
+        icon: 'fa-info-circle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.notices_title'),
+        count: @data[:notices].length,
+        description: I18n.t('bulkrax.importer.guided_import.validation.notices_desc'),
+        items: @data[:notices].map { |n| { field: n[:field], message: [n[:message], n[:suggestion]].compact.join(' ') } },
+        defaultOpen: false
+      }
+    end
+
+    def filtered_row_errors
+      missing_required_columns = @data[:missingRequired]&.map { |h| h[:field].to_s } || []
+      notice_columns = @data[:notices]&.map { |n| n[:field].to_s } || []
+      suppressed_columns = (missing_required_columns + notice_columns).uniq
+      @data[:rowErrors].reject { |e| suppressed_columns.include?(e[:column].to_s) }
+    end
+
+    def row_error_items(errors)
+      errors.map do |error|
+        message = error[:message]
+        message = [message, error[:suggestion]].compact.join(' ') if error[:suggestion].present?
+        {
+          field: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.row_label', row: error[:row], column: error[:column]),
+          message: message
+        }
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength

data/app/services/bulkrax/validation_error_csv_builder.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Bulkrax
+  # Builds a CSV string containing all validation errors from a guided import.
+  # File-level errors (missing required columns, unrecognized headers, empty
+  # columns, missing files) appear first as summary rows with a blank `row`
+  # cell. Row-level errors follow, one output row per errored data row.
+  #
+  # Usage:
+  #   csv = Bulkrax::ValidationErrorCsvBuilder.build(
+  #     headers:     result[:headers],
+  #     csv_data:    result[:raw_csv_data],
+  #     row_errors:  result[:rowErrors],
+  #     file_errors: {
+  #       missing_required: result[:missingRequired],
+  #       unrecognized:     result[:unrecognized],
+  #       empty_columns:    result[:emptyColumns],
+  #       missing_files:    result[:missingFiles]
+  #     }
+  #   )
+  class ValidationErrorCsvBuilder
+    # @param headers [Array<String>] original CSV headers in order
+    # @param csv_data [Array<Hash>] one entry per data row; each hash has
+    #   :raw_row (String-keyed hash of column=>value)
+    # @param row_errors [Array<Hash>] each hash has :row (Integer) and :message (String)
+    # @param file_errors [Hash] file-level issues:
+    #   - :missing_required [Array<Hash>] each hash has :model and :field
+    #   - :unrecognized [Hash] column_name => suggestion_or_nil
+    #   - :empty_columns [Array<Integer>] 1-based column positions with no header
+    #   - :missing_files [Array<String>] filenames referenced but not found
+    # @return [String] CSV content
+    def self.build(headers:, csv_data:, row_errors:, file_errors: {})
+      new(headers: headers, csv_data: csv_data, row_errors: row_errors, file_errors: file_errors).build
+    end
+
+    def initialize(headers:, csv_data:, row_errors:, file_errors:)
+      @headers    = headers
+      @csv_data   = csv_data
+      @row_errors = row_errors
+      @file_errors = file_errors
+    end
+
+    def build
+      errors_by_row = group_errors_by_row
+      blank_data    = Array.new(@headers.length)
+
+      CSV.generate(force_quotes: false) do |csv|
+        csv << ['row', 'errors'] + @headers
+
+        file_level_error_rows.each do |message|
+          csv << [nil, message] + blank_data
+        end
+
+        @csv_data.each_with_index do |record, index|
+          row_number = index + 2 # header is row 1; first data row is row 2
+          error_messages = errors_by_row[row_number]&.map { |e| e[:message] }&.join(' | ')
+          raw_row = record[:raw_row] || {}
+          csv << [row_number, error_messages] + @headers.map { |h| raw_row[h] }
+        end
+      end
+    end
+
+    private
+
+    def group_errors_by_row
+      @row_errors.each_with_object({}) do |error, hash|
+        row_num = error[:row]
+        hash[row_num] ||= []
+        hash[row_num] << error
+      end
+    end
+
+    def file_level_error_rows
+      messages = []
+
+      Array(@file_errors[:missing_required]).each do |entry|
+        messages << "Missing required column '#{entry[:field]}' (#{entry[:model]})"
+      end
+
+      Hash(@file_errors[:unrecognized]).each do |col, suggestion|
+        msg = "Unrecognized column '#{col}'"
+        msg += " (did you mean '#{suggestion}'?)" if suggestion.present?
+        messages << msg
+      end
+
+      Array(@file_errors[:empty_columns]).each do |pos|
+        messages << "Column #{pos + 2} has no header and will be ignored during import"
+      end
+
+      Array(@file_errors[:missing_files]).each do |filename|
+        messages << "Missing file: #{filename}"
+      end
+
+      messages
+    end
+  end
+end

data/app/validators/bulkrax/csv_row/child_reference.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvRow
+    ##
+    # Validates that any child references in a row point to source identifiers
+    # that exist either elsewhere in the same CSV or as existing repository records.
+    # Uses context[:all_ids] (Set of all source identifiers) to validate references
+    # within the CSV, and context[:find_record_by_source_identifier] (callable) to
+    # look up existing records in the same way the importer does at runtime.
+    # Skips validation when all_ids is empty and fill_in_blank_source_identifiers is
+    # configured, since generated identifiers cannot be cross-referenced at validation time.
+    module ChildReference
+      def self.call(record, row_index, context)
+        all_ids = context[:all_ids]
+        return if all_ids.empty? && Bulkrax.fill_in_blank_source_identifiers.present?
+
+        find_record = context[:find_record_by_source_identifier]
+
+        collect_child_ids(record, context).each do |child_id|
+          next if all_ids.include?(child_id)
+          next if find_record&.call(child_id)
+
+          context[:errors] << {
+            row: row_index,
+            source_identifier: record[:source_identifier],
+            severity: 'error',
+            category: 'invalid_child_reference',
+            column: 'children',
+            value: child_id,
+            message: I18n.t('bulkrax.importer.guided_import.validation.child_reference_validator.errors.message',
+                            value: child_id,
+                            field: 'source_identifier'),
+            suggestion: I18n.t('bulkrax.importer.guided_import.validation.child_reference_validator.errors.suggestion')
+          }
+        end
+      end
+
+      def self.collect_child_ids(record, context)
+        split_pattern = context[:child_split_pattern] || '|'
+        children_column = context[:children_column] || 'children'
+
+        base_ids = record[:children].to_s.split(split_pattern).map(&:strip).reject(&:blank?)
+
+        suffix_pattern = /\A#{Regexp.escape(children_column)}_\d+\z/
+        suffix_ids = record[:raw_row]
+                     .select { |k, _| k.to_s.match?(suffix_pattern) }
+                     .values
+                     .map(&:to_s).map(&:strip).reject(&:blank?)
+
+        (base_ids + suffix_ids).uniq
+      end
+      private_class_method :collect_child_ids
+    end
+  end
+end

data/app/validators/bulkrax/csv_row/circular_reference.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvRow
+    ##
+    # Detects circular parent-child relationships in the CSV.
+    # A circular reference occurs when following the parent chain from any record
+    # eventually leads back to itself (e.g. A→B→C→A).
+    #
+    # The validator builds a directed graph (child → parents) from all records on
+    # first invocation and caches the set of all record ids involved in any cycle.
+    # Subsequent per-row calls simply check membership in that set.
+    #
+    # Requires context key:
+    #   :relationship_graph  – Hash { source_identifier => [parent_ids] } built by
+    #                          run_row_validators before iterating rows.
+    module CircularReference
+      def self.call(record, row_index, context)
+        cycle_ids = context[:circular_reference_ids] ||= detect_cycle_ids(context[:relationship_graph] || {})
+        return unless cycle_ids.include?(record[:source_identifier])
+
+        context[:errors] << {
+          row: row_index,
+          source_identifier: record[:source_identifier],
+          severity: 'error',
+          category: 'circular_reference',
+          column: 'parents',
+          value: record[:source_identifier],
+          message: I18n.t('bulkrax.importer.guided_import.validation.circular_reference_validator.errors.message',
+                          value: record[:source_identifier]),
+          suggestion: I18n.t('bulkrax.importer.guided_import.validation.circular_reference_validator.errors.suggestion')
+        }
+      end
+
+      # Returns the set of all source identifiers that participate in at least one cycle.
+      # Uses recursive DFS with a per-branch ancestry set to detect back-edges.
+      def self.detect_cycle_ids(graph)
+        all_nodes = graph.keys.to_set | graph.values.flatten.to_set
+        visited   = Set.new
+        cycle_ids = Set.new
+
+        all_nodes.each do |node|
+          next if visited.include?(node)
+          dfs(node, graph, visited, [], cycle_ids)
+        end
+
+        cycle_ids
+      end
+      private_class_method :detect_cycle_ids
+
+      def self.dfs(node, graph, visited, ancestors, cycle_ids) # rubocop:disable Metrics/MethodLength
+        visited.add(node)
+        ancestors.push(node)
+
+        (graph[node] || []).each do |neighbor|
+          if ancestors.include?(neighbor)
+            # Back-edge found: mark every node in the cycle path
+            cycle_start = ancestors.index(neighbor)
+            ancestors[cycle_start..].each { |n| cycle_ids.add(n) }
+            cycle_ids.add(neighbor)
+          elsif !visited.include?(neighbor)
+            dfs(neighbor, graph, visited, ancestors, cycle_ids)
+          end
+        end
+
+        ancestors.pop
+      end
+      private_class_method :dfs
+    end
+  end
+end

data/app/validators/bulkrax/csv_row/controlled_vocabulary.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvRow
+    ##
+    # Validates that controlled vocabulary values in each row are valid according
+    # to the QA authority for that field.
+    module ControlledVocabulary
+      def self.call(record, row_index, context) # rubocop:disable Metrics/MethodLength
+        field_metadata = context[:field_metadata]
+        return if field_metadata.blank?
+
+        model = record[:model]
+        metadata = field_metadata[model]
+        return if metadata.blank?
+
+        controlled_terms = metadata[:controlled_vocab_terms] || []
+        return if controlled_terms.blank?
+
+        controlled_terms.each do |field|
+          value = record[:raw_row][field]
+          next if value.blank?
+
+          authority = load_authority(field)
+          next if authority.nil?
+
+          term = authority.find(value)
+          next unless term.blank? || term.dig('active') == false
+
+          context[:errors] << {
+            row: row_index,
+            source_identifier: record[:source_identifier],
+            severity: 'error',
+            category: 'invalid_controlled_value',
+            column: field,
+            value: value,
+            message: I18n.t('bulkrax.importer.guided_import.validation.controlled_vocabulary_validator.errors.message',
+                            value: value, field: field),
+            suggestion: suggestion(value, authority)
+          }
+        end
+      end
+
+      def self.load_authority(field)
+        Qa::Authorities::Local.subauthority_for(field.pluralize)
+      rescue Qa::InvalidSubAuthority
+        begin
+          Qa::Authorities::Local.subauthority_for(field)
+        rescue Qa::InvalidSubAuthority
+          nil
+        end
+      end
+      private_class_method :load_authority
+
+      def self.suggestion(value, authority)
+        suggestion = DidYouMean::SpellChecker.new(dictionary: dictionary_for(authority)).correct(value).first
+        return fallback_suggestion if suggestion.nil?
+
+        I18n.t('bulkrax.importer.guided_import.validation.did_you_mean', suggestion: suggestion)
+      end
+      private_class_method :suggestion
+
+      def self.fallback_suggestion
+        I18n.t('bulkrax.importer.guided_import.validation.controlled_vocabulary_validator.errors.suggestion')
+      end
+      private_class_method :fallback_suggestion
+
+      def self.dictionary_for(authority)
+        authority.all.filter_map { |term| term['label'] if term['active'] == true }.uniq
+      end
+      private_class_method :dictionary_for
+    end
+  end
+end