bulkrax 9.3.4 → 9.4.0

data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class CsvParser < ApplicationParser
+    # Hierarchy-building helpers for CsvValidation. Handles extracting and
+    # categorising items from parsed CSV data for the guided import tree view.
+    module CsvValidationHierarchy
+      def extract_validation_items(csv_data, all_ids = Set.new, find_record = nil, parent_split_pattern: nil, child_split_pattern: '|')
+        child_to_parents = build_child_to_parents_map(csv_data, child_split_pattern: child_split_pattern)
+        collections = []
+        works       = []
+        file_sets   = []
+
+        csv_data.each do |item|
+          categorise_validation_item(item, child_to_parents, all_ids, collections, works, file_sets, find_record,
+                                     parent_split_pattern: parent_split_pattern, child_split_pattern: child_split_pattern)
+        end
+
+        [collections, works, file_sets]
+      end
+
+      def build_child_to_parents_map(csv_data, child_split_pattern: '|')
+        Hash.new { |h, k| h[k] = [] }.tap do |map|
+          csv_data.each do |item|
+            next if item[:source_identifier].blank?
+
+            collect_relationship_ids(item[:children], item[:raw_row], 'children', split_pattern: child_split_pattern).each do |child_id|
+              map[child_id] << item[:source_identifier]
+            end
+          end
+        end
+      end
+
+      def categorise_validation_item(item, child_to_parents, all_ids, collections, works, file_sets, find_record = nil, parent_split_pattern: nil, child_split_pattern: '|') # rubocop:disable Metrics/ParameterLists
+        item_id   = item[:source_identifier]
+        model_str = item[:model].to_s
+
+        opts = { type: nil, find_record: find_record, parent: parent_split_pattern, child: child_split_pattern }
+        if model_str.casecmp('collection').zero? || model_str.casecmp('collectionresource').zero?
+          collections << build_item_hash(item, child_to_parents, all_ids, opts.merge(type: 'collection'))
+        elsif model_str.casecmp('fileset').zero? || model_str.casecmp('hyrax::fileset').zero?
+          file_sets << { id: item_id, title: item[:raw_row]['title'] || item_id, type: 'file_set' }
+        else
+          works << build_item_hash(item, child_to_parents, all_ids, opts.merge(type: 'work'))
+        end
+      end
+
+      def build_item_hash(item, child_to_parents, all_ids, opts = {}) # rubocop:disable Metrics/MethodLength
+        type = opts[:type]
+        find_record = opts[:find_record]
+        item_id  = item[:source_identifier]
+        title    = item[:raw_row]['title'] || item_id
+        parents  = collect_relationship_ids(item[:parent],   item[:raw_row], 'parents',  split_pattern: opts[:parent])
+        children = collect_relationship_ids(item[:children], item[:raw_row], 'children', split_pattern: opts[:child] || '|')
+
+        {
+          id: item_id,
+          title: title,
+          type: type,
+          existing: find_record&.call(item_id) || false,
+          parentIds: (resolvable_ids(parents, all_ids) + resolvable_ids(child_to_parents[item_id] || [], all_ids)).uniq,
+          childIds: resolvable_ids(children, all_ids),
+          existingParentIds: external_ids(parents, all_ids, find_record),
+          existingChildIds: external_ids(children, all_ids, find_record)
+        }
+      end
+
+      def parse_relationship_field(value, split_pattern: '|')
+        return [] if value.blank?
+        value.to_s.split(split_pattern).map(&:strip).reject(&:blank?)
+      end
+
+      def collect_relationship_ids(base_value, raw_row, column, split_pattern: '|')
+        base_ids = parse_relationship_field(base_value, split_pattern: split_pattern)
+        suffix_pattern = /\A#{Regexp.escape(column)}_\d+\z/
+        suffix_ids = raw_row
+                     .select { |k, _| k.to_s.match?(suffix_pattern) }
+                     .values
+                     .map(&:to_s).map(&:strip).reject(&:blank?)
+        (base_ids + suffix_ids).uniq
+      end
+
+      def resolvable_ids(ids, all_ids)
+        ids.select { |id| all_ids.include?(id) }
+      end
+
+      # Returns ids from the list that are NOT in the CSV but exist in the repository.
+      def external_ids(ids, all_ids, find_record)
+        return [] if find_record.nil?
+
+        ids.reject { |id| all_ids.include?(id) }
+           .select { |id| find_record.call(id) }
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/column_builder.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Builds column headers for CSV
+    class ColumnBuilder
+      def initialize(service)
+        @service = service
+        @descriptor = CsvTemplate::ColumnDescriptor.new
+      end
+
+      def all_columns
+        required_columns + property_columns
+      end
+
+      def required_columns
+        mapped_core_columns +
+          relationship_columns +
+          file_columns
+      end
+
+      private
+
+      def mapped_core_columns
+        @descriptor.core_columns.map do |column|
+          @service.mapping_manager.key_to_mapped_column(column)
+        end
+      end
+
+      def property_columns
+        field_lists = @service.all_models.map do |m|
+          @service.field_analyzer.find_or_create_field_list_for(model_name: m)
+        end
+
+        properties = field_lists
+                     .flat_map { |item| item.values.flat_map { |config| config["properties"] || [] } }
+                     .uniq
+                     .map { |property| @service.mapping_manager.key_to_mapped_column(property) }
+                     .uniq
+
+        (properties - required_columns).sort
+      end
+
+      def relationship_columns
+        [
+          @service.mapping_manager.find_by_flag("related_children_field_mapping", 'children'),
+          @service.mapping_manager.find_by_flag("related_parents_field_mapping", 'parents')
+        ]
+      end
+
+      def file_columns
+        CsvTemplate::ColumnDescriptor::COLUMN_DESCRIPTIONS[:files].flat_map do |property_hash|
+          property_hash.keys.map do |key|
+            @service.mapping_manager.key_to_mapped_column(key)
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/column_descriptor.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Manages column descriptions and metadata
+    class ColumnDescriptor
+      COLUMN_DESCRIPTIONS = {
+        include_first: [
+          { "model" => "The work types configured in your repository are listed below.\nIf left blank, your default work type, #{Bulkrax.default_work_type}, is used." },
+          { "source_identifier" => "This must be a unique identifier.\nIt can be alphanumeric with some special characters (e.g. hyphens, colons), and URLs are also supported." },
+          { "id" => "This column would optionally be included only if it is a re-import, i.e. for updating or deleting records.\nThis is a key identifier used by the system, which you wouldn't have for new imports." },
+          { "rights_statement" => "Rights statement URI for the work.\nIf not included, uses the value specified on the bulk import configuration screen." }
+        ],
+        visibility: [
+          { "visibility" => "Uses the value specified on the bulk import configuration screen if not added here.\nValid options: open, authenticated, restricted, embargo, lease" },
+          { "embargo_release_date" => "Required for embargo (yyyy-mm-dd)" },
+          { "visibility_during_embargo" => "Required for embargo" },
+          { "visibility_after_embargo" => "Required for embargo" },
+          { "lease_expiration_date" => "Required for lease (yyyy-mm-dd)" },
+          { "visibility_during_lease" => "Required for lease" },
+          { "visibility_after_lease" => "Required for lease" }
+        ],
+        files: [
+          { "file" => "Use filenames exactly matching those in your files folder.\nZip your CSV and files folder together and attach this to your importer." },
+          { "remote_files" => "Use the URLs to remote files to be attached to the work." }
+        ],
+        relationships: [
+          { "parents" => "The source_identifier or id of work or collection to be attached as parent." },
+          { "children" => "The source_identifier or id of work or file to be attached as child." }
+        ],
+        other: [
+          { "hide_from_catalog_search" => "Set to 1 to hide the collection from catalog search results." },
+          { "show_pdf_download_button" => "Set to 1 to show a PDF download link on the work's page." },
+          { "show_pdf_viewer" => "Set to 1 to show a PDF viewer on the work's page." },
+          { "video_embed" => "A valid URL to a hosted video that can appear in an iframe, beginning with 'http://' or 'https://'." }
+        ]
+      }.freeze
+
+      def core_columns
+        extract_column_names(:include_first) + extract_column_names(:visibility)
+      end
+
+      def find_description_for(column)
+        COLUMN_DESCRIPTIONS.each_value do |group|
+          prop = group.find { |hash| hash.key?(column) }
+          return prop[column] if prop
+        end
+        nil
+      end
+
+      private
+
+      def extract_column_names(group)
+        COLUMN_DESCRIPTIONS[group].map { |hash| hash.keys.first }
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/csv_builder.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Builds CSV content
+    class CsvBuilder
+      IGNORED_PROPERTIES = %w[
+        admin_set_id alternate_ids
+        bulkrax_identifier
+        collection_type_gid contexts created_at
+        date date_modified date_uploaded depositor
+        embargo embargo_id
+        file_ids
+        has_model head
+        internal_resource is_child
+        lease lease_id
+        member_ids member_of_collection_ids modified_date
+        new_record
+        on_behalf_of owner proxy_depositor
+        rendering_ids representative_id
+        schema_version split_from_pdf_id state tail
+        thumbnail_id
+        updated_at
+      ].freeze
+
+      def initialize(service)
+        @service = service
+        @column_builder = CsvTemplate::ColumnBuilder.new(service)
+        @row_builder = CsvTemplate::RowBuilder.new(service)
+        @header_row = nil
+        @required_headings = []
+      end
+
+      def write_to_file(file_path)
+        FileUtils.mkdir_p(File.dirname(file_path))
+        CSV.open(file_path, "w") { |csv| write_rows(csv) }
+      end
+
+      def generate_string
+        CSV.generate { |csv| write_rows(csv) }
+      end
+
+      private
+
+      def write_rows(csv)
+        csv_rows.each { |row| csv << row }
+      end
+
+      def csv_rows
+        @header_row = fill_header_row
+        rows = [
+          @header_row,
+          @row_builder.build_explanation_row(@header_row),
+          *@row_builder.build_model_rows(@header_row)
+        ]
+        remove_empty_columns(rows)
+      end
+
+      def fill_header_row
+        @required_headings = @column_builder.required_columns
+        all_columns = @column_builder.all_columns
+        filtered = all_columns - IGNORED_PROPERTIES
+        @required_headings = @column_builder.required_columns & filtered
+        filtered
+      end
+
+      def remove_empty_columns(rows)
+        return rows if rows.empty?
+
+        columns = rows.transpose
+        non_empty_columns = columns.select { |col| keep_column?(col) }
+        non_empty_columns.transpose
+      end
+
+      def keep_column?(column)
+        heading = column[0]
+        return true if @required_headings.include?(heading)
+
+        column[2..-1].any? { |value| !value.nil? && value != "" && value != "---" }
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/explanation_builder.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Builds explanations for CSV columns
+    class ExplanationBuilder
+      def initialize(service)
+        @service = service
+        @descriptor = CsvTemplate::ColumnDescriptor.new
+        @split_formatter = CsvTemplate::SplitFormatter.new
+      end
+
+      def build_explanations(header_row)
+        header_row.map do |column|
+          { column => build_explanation(column) }
+        end
+      end
+
+      private
+
+      def build_explanation(column)
+        mapping_key = @service.mapping_manager.mapped_to_key(column)
+
+        column_description = source_identifier_description(column) || @descriptor.find_description_for(column)
+        controlled_vocab_info = controlled_vocab_text(mapping_key)
+        split_info = split_text(mapping_key, controlled_vocab_info)
+
+        components = [
+          column_description,
+          controlled_vocab_info,
+          split_info
+        ].compact
+
+        components.join("\n")
+      end
+
+      def source_identifier_description(column)
+        return unless @service.mapping_manager.mapped_to_key(column) == 'source_identifier'
+        return if Bulkrax.fill_in_blank_source_identifiers.blank?
+        "Will be auto-generated if left blank.\nProviding one allows round-tripping and deduplication across imports."
+      end
+
+      def controlled_vocab_text(field_name)
+        vocab_terms = @service.field_analyzer.controlled_vocab_terms
+        return unless vocab_terms.include?(field_name) || field_name == 'based_near'
+        'This property uses a controlled vocabulary.'
+      end
+
+      def split_text(mapping_key, controlled_vocab_info)
+        return nil if controlled_vocab_info.present? && !mapping_key.in?(%w[location resource_type])
+        split_value = @service.mapping_manager.split_value_for(mapping_key)
+        return nil unless split_value
+        @split_formatter.format(split_value)
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/field_analyzer.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Analyzes model fields and schemas
+    class FieldAnalyzer
+      attr_reader :field_list
+
+      def initialize(mappings, admin_set_id = nil)
+        @mappings = mappings
+        @field_list = []
+        @schema = nil
+        @admin_set_id = admin_set_id
+      end
+
+      def find_or_create_field_list_for(model_name:)
+        existing = @field_list.find { |entry| entry.key?(model_name) }
+        return existing if existing.present?
+
+        klass = CsvTemplate::ModelLoader.determine_klass_for(model_name)
+        return {} if klass.nil?
+
+        model_entry = build_field_list_entry(model_name, klass)
+        @field_list << model_entry
+        model_entry
+      end
+
+      def controlled_vocab_terms
+        @field_list.flat_map do |hash|
+          hash.values.flat_map { |data| data["controlled_vocab_terms"] || [] }
+        end.uniq
+      end
+
+      private
+
+      def build_field_list_entry(model_name, klass)
+        schema_analyzer = CsvTemplate::SchemaAnalyzer.new(klass: klass, admin_set_id: @admin_set_id)
+        {
+          model_name => {
+            'properties' => extract_properties(klass),
+            'required_terms' => schema_analyzer.required_terms,
+            'controlled_vocab_terms' => schema_analyzer.controlled_vocab_terms
+          }
+        }
+      end
+
+      def extract_properties(klass)
+        if klass.respond_to?(:schema)
+          Bulkrax::ValkyrieObjectFactory.schema_properties(klass: klass, admin_set_id: @admin_set_id).map(&:to_s)
+        else
+          klass.properties.keys.map(&:to_s)
+        end
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/file_path_generator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Utility class for generating template file paths
+    class FilePathGenerator
+      TEMPLATE_PREFIX = 'import_template'
+
+      def self.default_path(admin_set_id)
+        context = load_context(admin_set_id)
+        tenant = load_tenant
+        filename = build_filename(context, tenant)
+        path = Rails.root.join('tmp', 'imports', filename)
+        FileUtils.mkdir_p(path.dirname.to_s)
+        path
+      end
+
+      def self.load_context(admin_set_id)
+        return nil if admin_set_id.blank?
+
+        admin_set = Bulkrax.object_factory.find(admin_set_id)
+        admin_set.respond_to?(:contexts) ? admin_set.contexts.first : nil
+      end
+
+      def self.load_tenant
+        return nil unless defined?(Apartment::Tenant) && defined?(Account)
+
+        tenant_id = Apartment::Tenant.current
+        return nil if tenant_id.blank?
+
+        Account.find_by(tenant: tenant_id)&.name
+      end
+
+      def self.build_filename(context, tenant)
+        parts = [TEMPLATE_PREFIX]
+        parts << "context-#{context}" if context.present?
+        parts << "tenant-#{tenant}" if tenant.present?
+        parts << timestamp
+        "#{parts.join('_')}.csv"
+      end
+
+      def self.timestamp
+        Time.current.utc.strftime('%Y%m%d_%H%M%S')
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/file_validator.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    ##
+    # Validates file references in CSV against zip archive contents
+    class FileValidator
+      attr_reader :csv_data, :zip_file
+
+      def initialize(csv_data, zip_file = nil, admin_set_id = nil)
+        @csv_data = csv_data
+        @zip_file = zip_file
+        @admin_set_id = admin_set_id
+      end
+
+      def count_references
+        @csv_data.count { |item| item[:file].present? }
+      end
+
+      def missing_files
+        return [] unless @zip_file
+
+        referenced_files - zip_file_list
+      end
+
+      def found_files_count
+        return 0 unless @zip_file
+
+        (referenced_files & zip_file_list).count
+      end
+
+      def zip_included?
+        @zip_file.present?
+      end
+
+      def possible_missing_files?
+        return false unless referenced_files.any?
+        return true if @zip_file.blank?
+
+        false
+      end
+
+      private
+
+      def referenced_files
+        @referenced_files ||= @csv_data.flat_map do |item|
+          next [] if item[:file].blank?
+
+          item[:file].split(Bulkrax.multi_value_element_split_on).map { |f| File.basename(f.strip) }
+        end.compact
+      end
+
+      def zip_file_list
+        @zip_file_list ||= begin
+          return [] unless @zip_file
+
+          zip_path = @zip_file.respond_to?(:path) ? @zip_file.path : @zip_file
+          Zip::File.open(zip_path) do |zip|
+            zip.entries.select(&:file?).map { |entry| File.basename(entry.name) }
+          end
+                           rescue StandardError => e
+                             Rails.logger.error("Error reading zip file: #{e.message}")
+                             []
+        end
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/mapping_manager.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Handles loading and filtering of Bulkrax field mappings
+    class MappingManager
+      attr_reader :mappings
+
+      def initialize
+        @mappings = load_mappings
+      end
+
+      def mapped_to_key(column_str)
+        @mappings.find { |_k, v| v["from"].include?(column_str) }&.first || column_str
+      end
+
+      def key_to_mapped_column(key)
+        @mappings.dig(key, "from")&.first || key
+      end
+
+      def find_by_flag(field_name, default)
+        @mappings.find { |_k, v| v[field_name] == true }&.first || default
+      end
+
+      def split_value_for(mapping_key)
+        @mappings.dig(mapping_key, "split")
+      end
+
+      def resolve_column_name(key: nil, flag: nil, default: nil)
+        if flag
+          mapped_key = find_by_flag(flag, nil)
+          if mapped_key
+            mapped_options = @mappings.dig(mapped_key, "from") || []
+            return mapped_options if mapped_options.any?
+          end
+        end
+
+        if key
+          mapped_options = @mappings.dig(key, "from") || []
+          return mapped_options if mapped_options.any?
+        end
+
+        default ? [default] : []
+      end
+
+      private
+
+      def load_mappings
+        Bulkrax.field_mappings["Bulkrax::CsvParser"].reject do |_key, value|
+          value["generated"] == true
+        end
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/model_loader.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Handles model loading based on configuration
+    class ModelLoader
+      attr_reader :models
+
+      def initialize(models)
+        @models = load_models(models)
+      end
+
+      def self.determine_klass_for(model_name)
+        if Bulkrax.config.object_factory == Bulkrax::ValkyrieObjectFactory
+          Valkyrie.config.resource_class_resolver.call(model_name)
+        else
+          model_name.constantize
+        end
+      rescue StandardError
+        nil
+      end
+
+      private
+
+      def load_models(models)
+        case models
+        when Array
+          return all_available_models if models.empty?
+          return all_available_models if models.include?('all')
+          models.filter_map { |model| safe_constantize(model) }
+        else
+          all_available_models
+        end
+      rescue StandardError
+        []
+      end
+
+      def safe_constantize(model_name)
+        model_name.constantize && model_name
+      rescue NameError
+        nil
+      end
+
+      def all_available_models
+        Hyrax.config.curation_concerns.map(&:name) +
+          [Bulkrax.collection_model_class&.name, Bulkrax.file_model_class&.name].compact
+      end
+    end
+  end
+end

data/app/services/bulkrax/csv_template/row_builder.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module CsvTemplate
+    # Builds CSV rows (explanations and model data)
+    class RowBuilder
+      def initialize(service)
+        @service = service
+        @explanation_builder = CsvTemplate::ExplanationBuilder.new(service)
+        @value_determiner = CsvTemplate::ValueDeterminer.new(service)
+      end
+
+      def build_explanation_row(header_row)
+        @explanation_builder.build_explanations(header_row).map { |prop| prop.values.join(" ") }
+      end
+
+      def build_model_rows(header_row)
+        @service.all_models.map { |m| model_breakdown(m, header_row) }
+      end
+
+      private
+
+      def model_breakdown(model_name, header_row)
+        klass = CsvTemplate::ModelLoader.determine_klass_for(model_name)
+        return [] if klass.nil?
+
+        field_list = @service.field_analyzer.find_or_create_field_list_for(model_name: model_name)
+
+        header_row.map do |column|
+          @value_determiner.determine_value(column, model_name, field_list)
+        end
+      end
+    end
+  end
+end