bulkrax 9.3.4 → 9.4.0

data/app/services/bulkrax/csv_template/schema_analyzer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Analyzes model schemas for required and controlled vocabulary fields
+    class SchemaAnalyzer
+      def initialize(klass:, admin_set_id: nil)
+        @klass = klass
+        @admin_set_id = admin_set_id
+        @schema = load_schema
+      end
+
+      def required_terms
+        return [] if @schema.blank?
+
+        @schema.select do |field|
+          field.respond_to?(:meta) &&
+            field.meta["form"].is_a?(Hash) &&
+            field.meta["form"]["required"] == true
+        end.map(&:name).map(&:to_s)
+      rescue StandardError
+        []
+      end
+
+      def controlled_vocab_terms
+        return [] unless @schema
+
+        controlled_properties = extract_controlled_properties
+        controlled_properties.empty? ? registered_controlled_vocab_fields : controlled_properties
+      rescue StandardError
+        []
+      end
+
+      private
+
+      def load_schema
+        return nil unless @klass.respond_to?(:schema)
+
+        if @admin_set_id.present? && defined?(Hyrax) && Hyrax.respond_to?(:schema_for)
+          Hyrax.schema_for(klass: @klass, admin_set_id: @admin_set_id)
+        else
+          @klass.new.singleton_class.schema || @klass.schema
+        end
+      rescue StandardError
+        nil
+      end
+
+      def extract_controlled_properties
+        return [] unless @schema
+
+        @schema.filter_map do |property|
+          next unless property.respond_to?(:meta)
+          sources = property.meta&.dig('controlled_values', 'sources')
+          next if sources.nil? || sources == ['null'] || sources == 'null'
+          property.name.to_s
+        end
+      end
+
+      def registered_controlled_vocab_fields
+        qa_registry.filter_map do |k, v|
+          k.singularize if v.klass == Qa::Authorities::Local::FileBasedAuthority
+        end
+      end
+
+      def qa_registry
+        @qa_registry ||= Qa::Authorities::Local.registry.instance_variable_get('@hash')
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/split_formatter.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Formats split pattern descriptions
+    class SplitFormatter
+      def format(split_value)
+        return "Property does not split." if split_value.nil?
+
+        if split_value == true
+          parse_pattern(Bulkrax.multi_value_element_split_on.source)
+        elsif split_value.is_a?(String)
+          parse_pattern(split_value)
+        else
+          split_value
+        end
+      end
+
+      private
+
+      def parse_pattern(pattern)
+        chars = extract_characters(pattern)
+        format_message(chars)
+      end
+
+      def extract_characters(pattern)
+        if (match = pattern.match(/\[([^\]]+)\]/))
+          match[1]
+        elsif (single = pattern.match(/\\(.)/))
+          single[1]
+        else
+          pattern
+        end
+      end
+
+      def format_message(chars)
+        formatted = chars.chars.then do |c|
+          c.length > 1 ? "#{c[0..-2].join(' ')}, or #{c.last}" : c.first
+        end
+        "Split multiple values with #{formatted}"
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/value_determiner.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Determines values for CSV cells
+    class ValueDeterminer
+      def initialize(service)
+        @service = service
+        @column_builder = CsvTemplate::ColumnBuilder.new(service)
+      end
+
+      def determine_value(column, model_name, field_list)
+        key = @service.mapping_manager.mapped_to_key(column)
+        required_terms = field_list.dig(model_name, 'required_terms')
+
+        if field_list.dig(model_name, "properties")&.include?(key)
+          mark_required_or_optional(key, required_terms)
+        elsif special_column?(column, key)
+          special_value(column, key, model_name, required_terms)
+        end
+      end
+
+      private
+
+      def special_column?(column, key)
+        descriptor = CsvTemplate::ColumnDescriptor.new
+        visibility_cols = descriptor.send(:extract_column_names, :visibility)
+
+        key.in?(['model', 'work_type']) ||
+          column.in?(visibility_cols) ||
+          column == 'source_identifier' ||
+          column == 'rights_statement' ||
+          relationship_column?(column) ||
+          file_column?(column)
+      end
+
+      def special_value(column, key, model_name, required_terms)
+        return CsvTemplate::ModelLoader.determine_klass_for(model_name).to_s if key.in?(['model', 'work_type'])
+        return 'Required' if column == 'source_identifier'
+        return mark_required_or_optional(key, required_terms) if column == 'rights_statement'
+        return nil if file_column?(column) && model_name.in?([Bulkrax.collection_model_class].compact.map(&:to_s))
+        'Optional'
+      end
+
+      def mark_required_or_optional(field, required_terms)
+        return 'Unknown' unless required_terms
+        required_terms.include?(field) ? 'Required' : 'Optional'
+      end
+
+      def relationship_column?(column)
+        relationships = [
+          @service.mapping_manager.find_by_flag("related_children_field_mapping", 'children'),
+          @service.mapping_manager.find_by_flag("related_parents_field_mapping", 'parents')
+        ]
+        column.in?(relationships)
+      end
+
+      def file_column?(column)
+        file_cols = CsvTemplate::ColumnDescriptor::COLUMN_DESCRIPTIONS[:files].flat_map do |property_hash|
+          property_hash.keys.filter_map do |key|
+            @service.mappings.dig(key, "from")&.first
+          end
+        end
+        column.in?(file_cols)
+      end
+    end
+  end
+end

data/app/services/bulkrax/stepper_response_formatter.rb

@@ -0,0 +1,347 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  ##
+  # Formats validation data from CsvParser.validate_csv into the structure
+  # expected by the importers_stepper.js frontend component.
+  #
+  # This service acts as a presentation layer, transforming raw validation data
+  # into a structured response with proper status messages, severity levels,
+  # and formatted issue lists that the JavaScript can render correctly.
+  #
+  # @example Basic usage
+  #   validation_data = CsvParser.validate_csv(csv_file: file, zip_file: zip)
+  #   formatted_response = StepperResponseFormatter.format(validation_data)
+  #   render json: formatted_response
+  #
+  # @example Error response
+  #   error_response = StepperResponseFormatter.error(message: "Unable to process files")
+  #   render json: error_response, status: :ok
+  #
+  # rubocop:disable Metrics/ClassLength
+  class StepperResponseFormatter
+    # Format validation data for the stepper frontend
+    #
+    # @param data [Hash] Raw validation data from CsvParser.validate_csv containing:
+    #   - headers: Array of CSV column names
+    #   - missingRequired: Array of hashes of missing required fields by model (e.g. {model: 'GenericWork', field: 'source_identifier'})
+    #   - unrecognized: Array of unrecognized column names
+    #   - rowCount: Total number of data rows
+    #   - isValid: Boolean indicating validation success
+    #   - hasWarnings: Boolean indicating presence of warnings
+    #   - collections: Array of collection items with id, title, type, parentIds (array), childIds (array)
+    #   - works: Array of work items with id, title, type, parentIds (array), childIds (array)
+    #   - fileSets: Array of file set items
+    #   - totalItems: Total count of items
+    #   - fileReferences: Count of file references
+    #   - missingFiles: Array of missing file names
+    #   - foundFiles: Count of found files
+    #   - zipIncluded: Boolean indicating if zip was provided
+    # @return [Hash] Formatted response ready for JSON rendering
+    def self.format(data)
+      new(data).format
+    end
+
+    # Generate an error response for validation failures
+    #
+    # @param message [String] Error message to display
+    # @param summary [String] Optional summary (defaults to standard message)
+    # @return [Hash] Error response structure
+    def self.error(message: I18n.t('bulkrax.importer.guided_import.validation.unable_to_process'), summary: nil)
+      {
+        totalItems: 0,
+        collections: [],
+        works: [],
+        fileSets: [],
+        isValid: false,
+        hasWarnings: false,
+        messages: {
+          validationStatus: {
+            severity: 'error',
+            icon: 'fa-times-circle',
+            title: I18n.t('bulkrax.importer.guided_import.validation.failed'),
+            summary: summary || message,
+            details: I18n.t('bulkrax.importer.guided_import.validation.critical_errors'),
+            defaultOpen: true
+          },
+          issues: []
+        }
+      }
+    end
+
+    def initialize(data)
+      @data = data
+    end
+
+    # Format the validation data with messages structure
+    # If data already contains a messages structure, return it as-is
+    #
+    # @return [Hash] Complete formatted response
+    def format
+      # Check if data is already formatted (has messages structure)
+      return @data if already_formatted?
+
+      # Build formatted response with messages structure
+      {
+        headers: @data[:headers],
+        missingRequired: @data[:missingRequired],
+        unrecognized: @data[:unrecognized],
+        rowCount: @data[:rowCount],
+        isValid: @data[:isValid],
+        hasWarnings: @data[:hasWarnings],
+        rowErrors: @data[:rowErrors],
+        collections: @data[:collections],
+        works: @data[:works],
+        fileSets: @data[:fileSets],
+        totalItems: @data[:totalItems],
+        fileReferences: @data[:fileReferences],
+        missingFiles: @data[:missingFiles],
+        foundFiles: @data[:foundFiles],
+        zipIncluded: @data[:zipIncluded],
+        messages: build_messages
+      }
+    end
+
+    private
+
+    # Check if data is already formatted with messages structure
+    #
+    # @return [Boolean] true if data already has proper messages structure
+    def already_formatted?
+      @data.key?(:messages) &&
+        @data[:messages].is_a?(Hash) &&
+        @data[:messages].key?(:validationStatus)
+    end
+
+    # Build the messages structure with validationStatus and issues
+    #
+    # @return [Hash] Messages structure for frontend
+    def build_messages
+      issues = []
+      issues << missing_required_issue if @data[:missingRequired]&.any?
+      issues << notices_issue if @data[:notices]&.any?
+      issues << unrecognized_fields_issue if @data[:unrecognized]&.any? || @data[:emptyColumns]&.any?
+      issues << file_references_issue if @data[:fileReferences]&.positive?
+      issues << row_errors_issue if @data[:rowErrors]&.any? { |e| e[:severity] == 'error' }
+      issues << row_warnings_issue if @data[:rowErrors]&.any? { |e| e[:severity] == 'warning' }
+
+      {
+        validationStatus: validation_status,
+        issues: issues.compact
+      }
+    end
+
+    # Generate the main validation status object
+    #
+    # @return [Hash] Validation status with severity, icon, title, summary, details
+    def validation_status
+      severity, icon, title = determine_severity_level
+      recognized = @data[:headers].reject(&:blank?) - (@data[:unrecognized].keys || [])
+
+      {
+        severity: severity,
+        icon: icon,
+        title: title,
+        summary: I18n.t('bulkrax.importer.guided_import.validation.columns_detected', columns: @data[:headers].length, records: @data[:rowCount]),
+        details: details_message(recognized),
+        defaultOpen: true
+      }
+    end
+
+    # Determine severity level based on validation state
+    #
+    # @return [Array<String>] [severity, icon, title]
+    def determine_severity_level
+      if !@data[:isValid]
+        ['error', 'fa-times-circle', I18n.t('bulkrax.importer.guided_import.validation.failed')]
+      elsif @data[:hasWarnings]
+        ['warning', 'fa-exclamation-triangle', I18n.t('bulkrax.importer.guided_import.validation.passed_warnings')]
+      else
+        ['success', 'fa-check-circle', I18n.t('bulkrax.importer.guided_import.validation.passed')]
+      end
+    end
+
+    # Generate details message for validation status
+    #
+    # @param recognized [Array<String>] List of recognized field names
+    # @return [String] Details message
+    def details_message(recognized)
+      if @data[:isValid]
+        I18n.t('bulkrax.importer.guided_import.validation.recognized_fields', fields: recognized.join(', '))
+      else
+        I18n.t('bulkrax.importer.guided_import.validation.critical_errors')
+      end
+    end
+
+    # Format missing required fields issue
+    #
+    # @return [Hash] Missing required fields issue structure
+    def missing_required_issue
+      only_rights_statement = @data[:missingRequired]&.all? { |h| h[:field].to_s == 'rights_statement' }
+
+      if only_rights_statement
+        {
+          type: 'missing_required_fields',
+          severity: 'warning',
+          icon: 'fa-exclamation-triangle',
+          title: I18n.t('bulkrax.importer.guided_import.validation.missing_required_title'),
+          count: @data[:missingRequired].length,
+          description: I18n.t('bulkrax.importer.guided_import.validation.missing_rights_desc'),
+          items: @data[:missingRequired],
+          defaultOpen: false
+        }
+      else
+        {
+          type: 'missing_required_fields',
+          severity: 'error',
+          icon: 'fa-times-circle',
+          title: I18n.t('bulkrax.importer.guided_import.validation.missing_required_title'),
+          count: @data[:missingRequired].length,
+          description: I18n.t('bulkrax.importer.guided_import.validation.missing_required_desc'),
+          items: @data[:missingRequired],
+          defaultOpen: false
+        }
+      end
+    end
+
+    # Format unrecognized fields issue
+    #
+    # @return [Hash] Unrecognized fields issue structure
+    def unrecognized_fields_issue
+      all_items = unrecognized_fields_issue_items
+      {
+        type: 'unrecognized_fields',
+        severity: 'warning',
+        icon: 'fa-exclamation-triangle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.unrecognized_title'),
+        count: all_items.length,
+        description: I18n.t('bulkrax.importer.guided_import.validation.unrecognized_desc'),
+        items: all_items,
+        defaultOpen: false
+      }
+    end
+
+    def unrecognized_fields_issue_items
+      named = (@data[:unrecognized] || {}).partition(&:last)
+              .flatten(1)
+              .map { |field| { field: field.first, message: field.last ? I18n.t('bulkrax.importer.guided_import.validation.did_you_mean', suggestion: field.last) : nil } }
+      empty = (@data[:emptyColumns] || []).map do |col|
+        { field: I18n.t('bulkrax.importer.guided_import.validation.empty_column', column: col), message: nil }
+      end
+      named + empty
+    end
+
+    # Format file references issue
+    #
+    # @return [Hash, nil] File references issue structure or nil if not applicable
+    def file_references_issue
+      missing_files = @data[:missingFiles] || []
+
+      if missing_files.any? && @data[:zipIncluded]
+        missing_files_issue
+      elsif !@data[:zipIncluded]
+        no_zip_issue
+      end
+    end
+
+    # Format issue for missing files in ZIP
+    #
+    # @return [Hash] Missing files issue structure
+    def missing_files_issue
+      missing_files = @data[:missingFiles]
+
+      {
+        type: 'file_references',
+        severity: 'warning',
+        icon: 'fa-info-circle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.file_references_title'),
+        count: @data[:fileReferences],
+        summary: I18n.t('bulkrax.importer.guided_import.validation.files_found_in_zip', found: @data[:foundFiles], total: @data[:fileReferences]),
+        description: I18n.t('bulkrax.importer.guided_import.validation.files_missing_from_zip', count: missing_files.length, files_word: 'file'.pluralize(missing_files.length)),
+        items: missing_files.map { |file| { field: file, message: I18n.t('bulkrax.importer.guided_import.validation.missing_from_zip') } },
+        defaultOpen: false
+      }
+    end
+
+    # Format issue for no ZIP uploaded
+    #
+    # @return [Hash] No ZIP issue structure
+    def no_zip_issue
+      {
+        type: 'file_references',
+        severity: 'warning',
+        icon: 'fa-exclamation-triangle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.file_references_title'),
+        count: @data[:fileReferences],
+        summary: I18n.t('bulkrax.importer.guided_import.validation.files_referenced', count: @data[:fileReferences]),
+        description: I18n.t('bulkrax.importer.guided_import.validation.no_zip_desc'),
+        items: [],
+        defaultOpen: false
+      }
+    end
+
+    def row_errors_issue
+      entries = filtered_row_errors.select { |e| e[:severity] == 'error' }
+      return nil if entries.empty?
+
+      {
+        type: 'row_level_errors',
+        severity: 'error',
+        icon: 'fa-times-circle',
+        title: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.title_errors'),
+        count: entries.length,
+        description: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.description'),
+        items: row_error_items(entries),
+        defaultOpen: false
+      }
+    end
+
+    def row_warnings_issue
+      entries = filtered_row_errors.select { |e| e[:severity] == 'warning' }
+      return nil if entries.empty?
+
+      {
+        type: 'row_level_warnings',
+        severity: 'warning',
+        icon: 'fa-exclamation-triangle',
+        title: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.title_warnings'),
+        count: entries.length,
+        description: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.description'),
+        items: row_error_items(entries),
+        defaultOpen: false
+      }
+    end
+
+    def notices_issue
+      {
+        type: 'notices',
+        severity: 'warning',
+        icon: 'fa-info-circle',
+        title: I18n.t('bulkrax.importer.guided_import.validation.notices_title'),
+        count: @data[:notices].length,
+        description: I18n.t('bulkrax.importer.guided_import.validation.notices_desc'),
+        items: @data[:notices].map { |n| { field: n[:field], message: [n[:message], n[:suggestion]].compact.join(' ') } },
+        defaultOpen: false
+      }
+    end
+
+    def filtered_row_errors
+      missing_required_columns = @data[:missingRequired]&.map { |h| h[:field].to_s } || []
+      notice_columns = @data[:notices]&.map { |n| n[:field].to_s } || []
+      suppressed_columns = (missing_required_columns + notice_columns).uniq
+      @data[:rowErrors].reject { |e| suppressed_columns.include?(e[:column].to_s) }
+    end
+
+    def row_error_items(errors)
+      errors.map do |error|
+        message = error[:message]
+        message = [message, error[:suggestion]].compact.join(' ') if error[:suggestion].present?
+        {
+          field: I18n.t('bulkrax.importer.guided_import.stepper_response_formatter.row_errors_issue.row_label', row: error[:row], column: error[:column]),
+          message: message
+        }
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength

data/app/services/bulkrax/validation_error_csv_builder.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Bulkrax
+  # Builds a CSV string containing all validation errors from a guided import.
+  # File-level errors (missing required columns, unrecognized headers, empty
+  # columns, missing files) appear first as summary rows with a blank `row`
+  # cell. Row-level errors follow, one output row per errored data row.
+  #
+  # Usage:
+  #   csv = Bulkrax::ValidationErrorCsvBuilder.build(
+  #     headers:     result[:headers],
+  #     csv_data:    result[:raw_csv_data],
+  #     row_errors:  result[:rowErrors],
+  #     file_errors: {
+  #       missing_required: result[:missingRequired],
+  #       unrecognized:     result[:unrecognized],
+  #       empty_columns:    result[:emptyColumns],
+  #       missing_files:    result[:missingFiles]
+  #     }
+  #   )
+  class ValidationErrorCsvBuilder
+    # @param headers [Array<String>] original CSV headers in order
+    # @param csv_data [Array<Hash>] one entry per data row; each hash has
+    #   :raw_row (String-keyed hash of column=>value)
+    # @param row_errors [Array<Hash>] each hash has :row (Integer) and :message (String)
+    # @param file_errors [Hash] file-level issues:
+    #   - :missing_required [Array<Hash>] each hash has :model and :field
+    #   - :unrecognized [Hash] column_name => suggestion_or_nil
+    #   - :empty_columns [Array<Integer>] 1-based column positions with no header
+    #   - :missing_files [Array<String>] filenames referenced but not found
+    # @return [String] CSV content
+    def self.build(headers:, csv_data:, row_errors:, file_errors: {})
+      new(headers: headers, csv_data: csv_data, row_errors: row_errors, file_errors: file_errors).build
+    end
+
+    def initialize(headers:, csv_data:, row_errors:, file_errors:)
+      @headers    = headers
+      @csv_data   = csv_data
+      @row_errors = row_errors
+      @file_errors = file_errors
+    end
+
+    def build
+      errors_by_row = group_errors_by_row
+      blank_data    = Array.new(@headers.length)
+
+      CSV.generate(force_quotes: false) do |csv|
+        csv << ['row', 'errors'] + @headers
+
+        file_level_error_rows.each do |message|
+          csv << [nil, message] + blank_data
+        end
+
+        @csv_data.each_with_index do |record, index|
+          row_number = index + 2 # header is row 1; first data row is row 2
+          error_messages = errors_by_row[row_number]&.map { |e| e[:message] }&.join(' | ')
+          raw_row = record[:raw_row] || {}
+          csv << [row_number, error_messages] + @headers.map { |h| raw_row[h] }
+        end
+      end
+    end
+
+    private
+
+    def group_errors_by_row
+      @row_errors.each_with_object({}) do |error, hash|
+        row_num = error[:row]
+        hash[row_num] ||= []
+        hash[row_num] << error
+      end
+    end
+
+    def file_level_error_rows
+      messages = []
+
+      Array(@file_errors[:missing_required]).each do |entry|
+        messages << "Missing required column '#{entry[:field]}' (#{entry[:model]})"
+      end
+
+      Hash(@file_errors[:unrecognized]).each do |col, suggestion|
+        msg = "Unrecognized column '#{col}'"
+        msg += " (did you mean '#{suggestion}'?)" if suggestion.present?
+        messages << msg
+      end
+
+      Array(@file_errors[:empty_columns]).each do |pos|
+        messages << "Column #{pos + 2} has no header and will be ignored during import"
+      end
+
+      Array(@file_errors[:missing_files]).each do |filename|
+        messages << "Missing file: #{filename}"
+      end
+
+      messages
+    end
+  end
+end

data/app/validators/bulkrax/csv_row/child_reference.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvRow
+    ##
+    # Validates that any child references in a row point to source identifiers
+    # that exist either elsewhere in the same CSV or as existing repository records.
+    # Uses context[:all_ids] (Set of all source identifiers) to validate references
+    # within the CSV, and context[:find_record_by_source_identifier] (callable) to
+    # look up existing records in the same way the importer does at runtime.
+    # Skips validation when all_ids is empty and fill_in_blank_source_identifiers is
+    # configured, since generated identifiers cannot be cross-referenced at validation time.
+    module ChildReference
+      def self.call(record, row_index, context)
+        all_ids = context[:all_ids]
+        return if all_ids.empty? && Bulkrax.fill_in_blank_source_identifiers.present?
+
+        find_record = context[:find_record_by_source_identifier]
+
+        collect_child_ids(record, context).each do |child_id|
+          next if all_ids.include?(child_id)
+          next if find_record&.call(child_id)
+
+          context[:errors] << {
+            row: row_index,
+            source_identifier: record[:source_identifier],
+            severity: 'error',
+            category: 'invalid_child_reference',
+            column: 'children',
+            value: child_id,
+            message: I18n.t('bulkrax.importer.guided_import.validation.child_reference_validator.errors.message',
+                            value: child_id,
+                            field: 'source_identifier'),
+            suggestion: I18n.t('bulkrax.importer.guided_import.validation.child_reference_validator.errors.suggestion')
+          }
+        end
+      end
+
+      def self.collect_child_ids(record, context)
+        split_pattern = context[:child_split_pattern] || '|'
+        children_column = context[:children_column] || 'children'
+
+        base_ids = record[:children].to_s.split(split_pattern).map(&:strip).reject(&:blank?)
+
+        suffix_pattern = /\A#{Regexp.escape(children_column)}_\d+\z/
+        suffix_ids = record[:raw_row]
+                     .select { |k, _| k.to_s.match?(suffix_pattern) }
+                     .values
+                     .map(&:to_s).map(&:strip).reject(&:blank?)
+
+        (base_ids + suffix_ids).uniq
+      end
+      private_class_method :collect_child_ids
+    end
+  end
+end