card-mod-csv_import 0.8

data/lib/import_manager/conflicts.rb

@@ -0,0 +1,50 @@
+class ImportManager
+  # Methods to deal with conflicts with existing cards
+  module Conflicts
+    def override?
+      @conflict_strategy == :override
+    end
+
+    def check_for_duplicates name
+      key = name.to_name.key
+      if @imported_keys.include? key
+        report :duplicate_in_file, name
+        throw :skip_row, :skipped
+      else
+        @imported_keys << key
+      end
+    end
+
+    def with_conflict_strategy strategy
+      tmp_cs = @conflict_strategy
+      @conflict_strategy = strategy if strategy
+      yield
+    ensure
+      @conflict_strategy = tmp_cs
+    end
+
+    def handle_conflict name, strategy: nil
+      with_conflict_strategy strategy do
+        with_conflict_resolution name do |duplicate|
+          yield duplicate
+        end
+      end
+    end
+
+    def with_conflict_resolution name
+      return yield unless (dup = duplicate(name))
+
+      case @conflict_strategy
+      when :skip      then throw :skip_row, :skipped
+      when :skip_card then dup
+      else
+        @status = :overridden
+        yield dup
+      end
+    end
+
+    def duplicate name
+      Card[name]
+    end
+  end
+end

data/lib/import_manager/script_import_manager.rb

@@ -0,0 +1,48 @@
+class ImportError < StandardError
+end
+
+# {ImportManager} for scripts. Main difference is you don't have an act card and
+# you can choose a error policy. For example throw  an exception on the first error or
+# collect all errors and report in the console at the end.
+# You can also specify a user who does the imports.
+# Unlike the other ImportManagers the ScriptImportManager import doesn't support
+# extra data to override fields.
+class ScriptImportManager < ImportManager
+  def initialize csv_file, conflict_strategy: :skip, error_policy: :fail, user: nil
+    super(csv_file, conflict_strategy, {})
+    @error_policy = error_policy
+    @user = user
+  end
+
+  def import_rows row_indices
+    with_user do
+      super
+    end
+  end
+
+  def with_user
+    if @user
+      Card::Auth.with(@user) { yield }
+    elsif Card::Auth.signed_in?
+      yield
+    else
+      raise StandardError, "can't import as anonymous"
+    end
+  end
+
+  def row_failed _csv_row
+    case @error_policy
+    when :fail then
+      raise ImportError, @import_status[:errors].inspect
+    when :report then
+      puts @import_status[:errors].inspect
+    when :skip then
+      nil
+    end
+  end
+
+  def log_status
+    puts "#{@current_row.row_index}:  #{@current_row.name}"
+    super
+  end
+end

data/lib/import_manager/status.rb

@@ -0,0 +1,50 @@
+class ImportManager
+  class Status < Hash
+    require_dependency "import_manager/status/counts"
+
+    # @param initial_status [Hash, Integer, String] a hash, a hash as json string or just
+    #   the total number of imports
+    def initialize status
+      hash = normalize_init_args status
+      counts = hash.delete(:counts)
+      replace hash
+      self[:counts] = Counts.new counts
+      init_missing_values
+    end
+
+    def init_missing_values
+      self[:errors] ||= Hash.new { |h, k| h[k] = [] }
+      self[:reports] ||= Hash.new { |h, k| h[k] = [] }
+      %i[imported skipped overridden failed].each do |n|
+        self[n] ||= {}
+      end
+    end
+
+    def normalize_init_args status
+      sym_hash = case status
+                 when Integer
+                   { counts: { total: status } }
+                 when String
+                   JSON.parse status
+                 when Hash
+                   status
+                 else
+                   {}
+                 end
+      unstringify_keys sym_hash
+    rescue JSON::ParserError => _e
+      {}
+    end
+
+    def unstringify_keys hash
+      hash.deep_symbolize_keys!
+      hash.keys.each do |k|
+        next if k == :counts || !hash[k].is_a?(Hash)
+        hash[k] = hash[k].each_with_object({}) do |(key, value), options|
+          options[(Integer(key.to_s) rescue key)] = value
+        end
+      end
+      hash
+    end
+  end
+end

data/lib/import_manager/status/counts.rb

@@ -0,0 +1,31 @@
+class ImportManager
+  class Status
+    class Counts < Hash
+      def initialize hash
+        hash ||= {}
+        replace hash
+      end
+
+      def default _key
+        0
+      end
+
+      def count key
+        if key.is_a? Array
+          key.inject(0) { |sum, k| sum + self[k] }
+        else
+          self[key]
+        end
+      end
+
+      def step key
+        self[key] += 1
+      end
+
+      def percentage key
+        return 0 if count(:total) == 0 || count(key).nil?
+        (count(key) / count(:total).to_f * 100).floor(2)
+      end
+    end
+  end
+end

data/lib/import_manager/status_log.rb

@@ -0,0 +1,80 @@
+class ImportManager
+  # Methods to collect errors and report the status of the import
+  module StatusLog
+    def log_status
+      import_status[@current_row.status] ||= {}
+      import_status[@current_row.status][@current_row.row_index] = @current_row.name
+      import_status[:counts].step @current_row.status
+    end
+
+    def report key, msg
+      case key
+      when :duplicate_in_file
+        msg = "#{msg} duplicate in this file"
+      end
+      import_status[:reports][@current_row.row_index] ||= []
+      import_status[:reports][@current_row.row_index] << msg
+    end
+
+    def import_status
+      @import_status || init_import_status
+    end
+
+    # used by {CSVRow} objects
+    def report_error msg
+      import_status[:errors][@current_row.row_index] ||= []
+      import_status[:errors][@current_row.row_index] << msg
+    end
+
+    def errors_by_row_index
+      @import_status[:errors].each do |index, msgs|
+        yield index, msgs
+      end
+    end
+
+    def pick_up_card_errors card=nil
+      card = yield if block_given?
+      if card
+        card.errors.each do |error_key, msg|
+          report_error "#{card.name} (#{error_key}): #{msg}"
+        end
+        card.errors.clear
+      end
+      card
+    end
+
+    def errors? row=nil
+      if row
+        errors(row).present?
+      else
+        errors.values.flatten.present?
+      end
+    end
+
+    def errors row=nil
+      if row
+        import_status.dig(:errors, row.row_index) || []
+      else
+        import_status[:errors] || {}
+      end
+    end
+
+    def error_list
+      @import_status[:errors].each_with_object([]) do |(index, errors), list|
+        next if errors.empty?
+        list << "##{index + 1}: #{errors.join('; ')}"
+      end
+    end
+
+    private
+
+    def init_import_status row_count=nil
+      @import_status = ImportManager::Status.new(row_count || 0)
+    end
+
+    def specify_success_status status
+      return status if status.in? %i[failed skipped]
+      @status == :overridden ? :overridden : :imported
+    end
+  end
+end

data/lib/import_manager/validation_manager.rb

@@ -0,0 +1,40 @@
+# ValidateManager doesn't import anything. It is used for collecting invalid data
+# to show it in the import table interface.
+class ValidationManager < ImportManager
+  def validate row_indices=nil, &block
+    @after_validation = block
+    validate_rows row_indices
+  end
+
+  def validate_rows row_indices
+    row_count = row_indices ? row_indices.size : @csv_file.row_count
+    @import_status = Status.new counts: { total: row_count }
+
+    @csv_file.each_row self, row_indices do |csv_row|
+      validate_row csv_row
+    end
+  end
+
+  def validate_row csv_row
+    handle_import csv_row do
+      csv_row.prepare_import
+    end
+  end
+
+  def add_card args
+    handle_conflict args[:name], strategy: :skip_card do
+      card = Card.new args
+      card.validate
+      pick_up_card_errors card
+    end
+  end
+
+  def each_row
+    @csv_file.each_row self do |row, i|
+    end
+  end
+
+  def row_finished row
+    @after_validation.call row
+  end
+end

data/set/abstract/import.rb

@@ -0,0 +1,89 @@
+card_accessor :import_status
+card_accessor :imported_rows
+
+delegate :mark_as_imported, :already_imported?, to: :imported_rows_card
+
+def import_file?
+  true
+end
+
+def csv_file
+  # maybe we have to use file.read ?
+  CSVFile.new file, csv_row_class, headers: :detect
+end
+
+def clean_html?
+  false
+end
+
+def csv_only? # for override
+  true
+end
+
+event :validate_import_format_on_create, :validate,
+      on: :create, when: :save_preliminary_upload? do
+  validate_file_card upload_cache_card
+end
+
+event :validate_import_format, :validate,
+      on: :update, when: :save_preliminary_upload? do
+  validate_file_card self
+end
+
+def validate_file_card file_card
+  if file_card.csv?
+    validate_csv file_card
+  elsif csv_only?
+    abort :failure, "file must be CSV but was '#{file_card.attachment.content_type}'"
+  end
+end
+
+def validate_csv file_card
+  CSVFile.new file_card.attachment, csv_row_class, headers: :detect
+rescue CSV::MalformedCSVError => e
+  abort :failure, "malformed csv: #{e.message}"
+end
+
+format :html do
+  before :new do
+    voo.help = help_text
+    voo.show! :help
+  end
+
+  before :edit do
+    voo.help = help_text
+    voo.show! :help
+  end
+
+  def help_text
+    rows = card.csv_row_class.columns.map { |s| s.to_s.humanize }
+    "expected csv format: #{rows.join ' | '}"
+  end
+
+  def new_view_hidden
+    hidden_tags success: { id: "_self", soft_redirect: false, redirect: true, view: :import }
+  end
+
+  view :core do
+    output [
+      download_link,
+      import_link,
+      last_import_status
+    ]
+  end
+
+  def download_link
+    handle_source do |source|
+      %(<a href="#{source}" rel="nofollow">Download "#{_render_title}"</a><br />)
+    end.html_safe
+  end
+
+  def import_link
+    link_to_view :import, "Import ...", rel: "nofollow", remote: false
+  end
+
+  def last_import_status
+    return unless card.import_status.present?
+    link_to_card card.import_status_card, "Status of last import"
+  end
+end

data/set/abstract/import/01_table_row.rb

@@ -0,0 +1,112 @@
+#! no set module
+
+# An ImportRow object generates the html for a row in the import table.
+# It needs a format with `column_keys` and a {CSVRow} object.
+# For each value in column_keys in generates a cell. It takes the content from the CSVRow
+# object if it has data for that key.
+# Otherwise it expects a `<missing_key>_field` method to produce the cell content.
+class TableRow
+  attr_reader :match_type, :csv_row, :format
+
+  delegate :row_index, :status, to: :csv_row
+  delegate :corrections_input_name, to: :format
+
+  # @param csv_row [CSVRow] a CSVRow object
+  # @param format the format of an import file. It has to respond to `column_keys`.
+  #        It is also used to generate form elements.
+  def initialize csv_row, format
+    @csv_row = csv_row
+    @format = format
+  end
+
+  def valid?
+    !@csv_row.errors?
+  end
+
+  def imported?
+    @format.already_imported? @csv_row.row_index
+  end
+
+  # The return value is supposed to be passed to the table helper method.
+  # @return [Hash] the :content value is an array with the text/html for each cell
+  def render
+    res = { content: fields, data: { csv_row_index: @csv_row.row_index } }
+    format.add_class res, row_css_classes
+    res
+  end
+
+  def row_css_classes
+    valid? ? [] : ["table-danger"]
+  end
+
+  # def render
+  #   super.merge class: "table-#{row_context}"
+  # end
+  #
+  private
+
+  # @return [Array] values for every cell in the table row
+  def fields
+    @format.column_keys.map do |key|
+      send "#{key}_field"
+    end
+  end
+
+  def checked?
+    true
+  end
+
+  def extra_data
+    {}
+  end
+
+  def corrections
+    {}
+  end
+
+  def row_index_field
+    @csv_row.row_index + 1
+  end
+
+  def checkbox_field
+    # disable if data is invalid
+    @format.check_box_tag("import_rows[#{@csv_row.row_index}]", true, valid? && checked?,
+                          disabled: !valid?) + extra_data_tags.html_safe
+  end
+
+  def extra_data_tags
+    extra_data.each_with_object([]) do |(k, v), a|
+      a << @format.hidden_field_tag(extra_data_input_name(k), v)
+    end.join "\n"
+  end
+
+  def correction_tag key, value, index=@csv_row.row_index
+    name = extra_data_input_name(index, :corrections, key)
+    @format.hidden_field_tag name, value
+  end
+
+  def extra_data_input_name *subfields
+    @format.extra_data_input_name @csv_row.row_index, *subfields
+  end
+
+  def corrections_input_name key
+    @format.corrections_input_name @csv_row.row_index, key
+  end
+
+  def method_missing method, *_args, &_block
+    return super unless (field = field_from_method(method))
+    @csv_row[field.to_sym]
+  end
+
+  def respond_to_missing? method, _include_private=false
+    field_method?(method) || super
+  end
+
+  def field_from_method method
+    method =~ /^(\w+)_field$/ ? Regexp.last_match(1) : nil
+  end
+
+  def field_method? method
+    method =~ /^\w+_field$/
+  end
+end