card-mod-csv_import 0.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 25acd866e5b9397247a10490a13ecda10c0abc89f77b8e99bd817aa71f16901b
+  data.tar.gz: a4362aeffc395a82e8eab678cfa3f559a58bf0f57c60f6a8f8cf463bc606907b
+SHA512:
+  metadata.gz: 887b48faecdd473f0234e1ee6942d442646618b22c4efad4786040148e271ac4e2286454009fb2bf8da47410fb96d19db2f1395313bcd782e1d57b2960ddc990
+  data.tar.gz: d7762714d04d219e0387adba9964ac8ccb083474f0ddbade53017d9c0e7099f1ba0efb737b972b7a03fc31a67d5633ff1fc86c52fa39a6c940a36a040b5cb6be

data/config/initializers/import_logger.rb

@@ -0,0 +1,2 @@
+ImportLog.logger = Logger.new(ImportLog::LogFile)
+ImportLog.logger.level = "debug"

data/lib/csv_file.rb

@@ -0,0 +1,137 @@
+# CSVFile loads csv data from a given path or file handle and provides methods
+# to iterate over the data.
+class CSVFile
+  # @param headers [true, false, :detect] (false) if true the import raises an error
+  #    if the csv file has no or wrong headers
+  def initialize path_or_file, row_class, col_sep: ",", encoding: "utf-8", headers: false
+    raise ArgumentError, "no row class given" unless row_class.is_a?(Class)
+    raise ArgumentError, "#{row_class} must inherit from CSVRow" unless row_class < CSVRow
+    @row_class = row_class
+    @col_sep = col_sep
+    @encoding = encoding
+
+    read_csv path_or_file
+    initialize_column_map headers
+  end
+
+  # yields the rows of the csv file as CSVRow objects
+  def each_row import_manager, rows=nil & block
+    each_row_hash rows do |row_hash, index|
+      yield @row_class.new(row_hash, index, import_manager)
+    end
+  end
+
+  # yields the rows of the csv file as simple hashes
+  def each_row_hash rows=nil, &block
+    if rows
+      selected_rows rows, &block
+    else
+      all_rows &block
+    end
+  end
+
+  def row_count
+    @rows.size
+  end
+
+  private
+
+  def read_csv path_or_file
+    @rows =
+      if path_or_file.respond_to?(:read)
+        read_csv_from_file_handle path_or_file
+      else
+        read_csv_from_path path_or_file
+      end
+  end
+
+  def read_csv_from_path path
+    raise StandardError, "file does not exist: #{path}" unless File.exist? path
+    rescue_encoding_error do
+      CSV.read path, csv_options
+    end
+  end
+
+  def read_csv_from_file_handle file
+    CSV.parse to_utf_8(file.read), csv_options
+  end
+
+  def rescue_encoding_error
+    yield
+  rescue ArgumentError => _e
+    # if parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
+    # and convert to utf-8
+    with_encoding "iso-8859-1:utf-8" do
+      yield
+    end
+  end
+
+  def to_utf_8 str, encoding="utf-8", force=false
+    if force
+      str.force_encoding("iso-8859-1").encode("utf-8")
+    else
+      str.encode encoding
+    end
+   rescue Encoding::UndefinedConversionError => _e
+     # If parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
+     # and convert to utf-8.
+     # If that failed to force it to iso-8859-1 before converting it.
+     to_utf_8 str, "iso-8859-1", encoding == "iso-8859-1"
+   end
+
+  def csv_options
+    { col_sep: @col_sep, encoding: @encoding }
+  end
+
+  def with_encoding encoding
+    enc = @encoding
+    @encoding = encoding
+    yield
+  ensure
+    @encoding = enc
+  end
+
+  def all_rows
+    @rows.each.with_index do |row, i|
+      next if row.compact.empty?
+      yield row_to_hash(row), i
+    end
+  end
+
+  def selected_rows rows
+    rows.each do |index|
+      yield row_to_hash(@rows[index]), index
+    end
+  end
+
+  def map_headers
+    @col_map = {}
+    headers = @rows.shift.map { |h| h.to_name.key.to_sym }
+    @row_class.columns.each do |key|
+      @col_map[key] = headers.index key
+      raise StandardError, "column #{key} is missing" unless @col_map[key]
+    end
+  end
+
+  def header_row?
+    return unless first_row = @rows.first.map { |h| h.to_name.key.to_sym }
+    @row_class.columns.all? do |item|
+      first_row.include? item
+    end
+  end
+
+  def row_to_hash row
+    @col_map.each_with_object({}) do |(k, v), h|
+      h[k] = row[v]
+      h[k] &&= h[k].strip
+    end
+  end
+
+  def initialize_column_map header_line
+    if (header_line == :detect && header_row?) || header_line == true
+      map_headers
+    else
+      @col_map = @row_class.columns.zip((0..@row_class.columns.size)).to_h
+    end
+  end
+end

data/lib/csv_row.rb

@@ -0,0 +1,167 @@
+# Inherit from CSVRow to describe and process a csv row.
+# CSVFile creates an instance of CSVRow for every row and calls #execute_import on it
+class CSVRow
+  include ::Card::Model::SaveHelper
+  include Normalizer
+
+  @columns = []
+  @required = [] # array of required fields or :all
+
+  # Use column names as keys and method names as values to define normalization
+  # and validation methods.
+  # The normalization methods get the original field value as
+  # argument. The validation methods get the normalized value as argument.
+  # The return value of normalize methods replaces the field value.
+  # If a validate method returns false then the import fails.
+  @normalize = {}
+  @validate = {}
+
+  class << self
+    attr_reader :columns, :required
+
+    def normalize key
+      @normalize && @normalize[key]
+    end
+
+    def validate key
+      @validate && @validate[key]
+    end
+  end
+
+  attr_reader :errors, :row_index, :import_manager
+  attr_accessor :status, :name
+
+  delegate :add_card, :import_card, :override?, :pick_up_card_errors, to: :import_manager
+
+  def initialize row, index, import_manager=nil
+    @row = row
+    @import_manager = import_manager || ImportManager.new(nil)
+    @extra_data = @import_manager.extra_data(index)
+    @abort_on_error = true
+    @row_index = index # 0-based, not counting the header line
+    merge_corrections
+  end
+
+  def original_row
+    @row.merge @before_corrected
+  end
+
+  def label
+    label = "##{@row_index + 1}"
+    label += ": #{@name}" if @name
+    label
+  end
+
+  def merge_corrections
+    @corrections = @extra_data[:corrections]
+    @corrections = {} unless @corrections.is_a? Hash
+    @before_corrected = {}
+    @corrections.each do |k, v|
+      next unless v.present?
+      @before_corrected[k] = @row[k]
+      @row[k] = v
+    end
+  end
+
+  def execute_import
+    @import_manager.handle_import(self) do
+      prepare_import
+      ImportLog.debug "start import"
+      import
+    end
+  rescue => e
+    ImportLog.debug "import failed: #{e.message}"
+    ImportLog.debug e.backtrace
+    raise e
+  end
+
+  def prepare_import
+    collect_errors { check_required_fields }
+    normalize
+    collect_errors { validate }
+  end
+
+  def check_required_fields
+    required.each do |key|
+      error "value for #{key} missing" unless @row[key].present?
+    end
+  end
+
+  def collect_errors
+    @abort_on_error = false
+    yield
+    skip :failed if errors?
+  ensure
+    @abort_on_error = true
+  end
+
+  def skip status=:skipped
+    throw :skip_row, status
+  end
+
+  def errors?
+    @import_manager.errors? self
+  end
+
+  def errors
+    @import_manager.errors self
+  end
+
+  def error msg
+    @import_manager.report_error msg
+    skip :failed if @abort_on_error
+  end
+
+  def required
+    self.class.required == :all ? columns : self.class.required
+  end
+
+  def columns
+    self.class.columns
+  end
+
+  def normalize
+    @row.each do |k, v|
+      normalize_field k, v
+    end
+  end
+
+  def validate
+    @row.each do |k, v|
+      validate_field k, v
+    end
+  end
+
+  def normalize_field field, value
+    return unless (method_name = method_name(field, :normalize))
+    @row[field] = send method_name, value
+  end
+
+  def validate_field field, value
+    return unless (method_name = method_name(field, :validate))
+    return if send method_name, value
+    error "row #{@row_index + 1}: invalid value for #{field}: #{value}"
+  end
+
+  # @param type [:normalize, :validate]
+  def method_name field, type
+    method_name = "#{type}_#{field}".to_sym
+    respond_to?(method_name) ? method_name : self.class.send(type, field)
+  end
+
+  def [] key
+    @row[key]
+  end
+
+  def fields
+    @row
+  end
+
+  def method_missing method_name, *args
+    respond_to_missing?(method_name) ? @row[method_name.to_sym] : super
+  end
+
+  def respond_to_missing? method_name, _include_private=false
+    @row.keys.include? method_name
+  end
+end

data/lib/csv_row/normalizer.rb

@@ -0,0 +1,12 @@
+class CSVRow
+  # common methods to be used to normalize values
+  module Normalizer
+    def comma_list_to_pointer str
+      str.split(",").map(&:strip).to_pointer_content
+    end
+
+    def to_html value
+      value.gsub "\n", "<br\>"
+    end
+  end
+end

data/lib/import_log.rb

@@ -0,0 +1,7 @@
+class ImportLog
+  LogFile = Rails.root.join("log", "import.log")
+  class << self
+    cattr_accessor :logger
+    delegate :debug, :info, :warn, :error, :fatal, to: :logger
+  end
+end

data/lib/import_manager.rb

@@ -0,0 +1,73 @@
+# ImportManager coordinates the import of a CSVFile. It defines the conflict and error
+# policy. It collects all errors and provides extra data like corrections for row fields.
+class ImportManager
+  require_dependency "import_manager/status"
+  include StatusLog
+  include Conflicts
+
+  attr_reader :conflict_strategy
+
+  def initialize csv_file, conflict_strategy=:skip, extra_data={}
+    @csv_file = csv_file
+    @conflict_strategy = conflict_strategy
+    @extra_data = integerfy_keys(extra_data || {})
+
+    @extra_data[:all] ||= {}
+    # init_import_status
+    @imported_keys = ::Set.new
+  end
+
+  def import row_indices=nil
+    import_rows row_indices
+  end
+
+  def import_rows row_indices
+    row_count = row_indices ? row_indices.size : @csv_file&.row_count
+    init_import_status row_count
+    @csv_file.each_row self, row_indices, &:execute_import
+  end
+
+  def extra_data index
+    (@extra_data[:all] || {}).deep_merge(@extra_data[index] || {})
+  end
+
+  def handle_import row
+    @current_row = row
+    status = catch(:skip_row) { yield }
+    status = specify_success_status status
+    @current_row.status = status
+    log_status
+    run_hook status
+  end
+
+  # used by csv rows to add additional cards
+  def add_card args
+    pick_up_card_errors do
+      Card.create args
+    end
+  end
+
+  def add_extra_data index, data
+    @extra_data[index].deep_merge! data
+  end
+
+  # add the final import card
+  def import_card card_args
+    @current_row.name = card_args[:name]
+    check_for_duplicates card_args[:name]
+    add_card card_args
+  end
+
+  private
+
+  # methods like row_imported, row_failed, etc. can be used to add additional logic
+  def run_hook status
+    row_finished @current_row if respond_to? :row_finished
+    hook_name = "row_#{status}".to_sym
+    send hook_name, @current_row if respond_to? hook_name
+  end
+
+  def integerfy_keys hash
+    hash.transform_keys { |key| key == :all ? :all : key.to_s.to_i }
+  end
+end

data/lib/import_manager/act_import_manager.rb

@@ -0,0 +1,43 @@
+# ActImportManager puts all creates and update actions that are part of the import
+# under one act of a import card
+class ActImportManager < ImportManager
+  def initialize act_card, csv_file, conflict_strategy=:skip, extra_row_data={}
+    @act_card = act_card
+    super(csv_file, conflict_strategy, extra_row_data)
+  end
+
+  def add_card args
+    handle_conflict args[:name] do |existing_card|
+      subcard =
+        if existing_card
+          existing_card.tap { |card| card.update_attributes args }
+        else
+          Card.create args
+        end
+      # subcard = @act_card&.add_subcard args.delete(:name), args
+      # subcard.director.catch_up_to_stage :validate
+      pick_up_card_errors { subcard }
+    end
+  end
+
+  def duplicate name
+    Card[name] || @act_card&.subcards&.at(name)
+  end
+
+  def log_status
+    super
+    @act_card&.import_status_card&.update_attributes content: @import_status.to_json
+  end
+
+  private
+
+  def init_import_status row_count=nil
+    isc = @act_card&.try :import_status_card
+    @import_status = isc&.real? ? isc.status : super
+  end
+
+  def row_finished row
+    return if row.status == :failed
+    @act_card&.mark_as_imported row.row_index
+  end
+end