card-mod-csv_import 0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 25acd866e5b9397247a10490a13ecda10c0abc89f77b8e99bd817aa71f16901b
4
+ data.tar.gz: a4362aeffc395a82e8eab678cfa3f559a58bf0f57c60f6a8f8cf463bc606907b
5
+ SHA512:
6
+ metadata.gz: 887b48faecdd473f0234e1ee6942d442646618b22c4efad4786040148e271ac4e2286454009fb2bf8da47410fb96d19db2f1395313bcd782e1d57b2960ddc990
7
+ data.tar.gz: d7762714d04d219e0387adba9964ac8ccb083474f0ddbade53017d9c0e7099f1ba0efb737b972b7a03fc31a67d5633ff1fc86c52fa39a6c940a36a040b5cb6be
@@ -0,0 +1,2 @@
1
+ ImportLog.logger = Logger.new(ImportLog::LogFile)
2
+ ImportLog.logger.level = "debug"
data/lib/csv_file.rb ADDED
@@ -0,0 +1,137 @@
1
+ # CSVFile loads csv data from a given path or file handle and provides methods
2
+ # to iterate over the data.
3
+ class CSVFile
4
+ # @param headers [true, false, :detect] (false) if true the import raises an error
5
+ # if the csv file has no or wrong headers
6
+ def initialize path_or_file, row_class, col_sep: ",", encoding: "utf-8", headers: false
7
+ raise ArgumentError, "no row class given" unless row_class.is_a?(Class)
8
+ raise ArgumentError, "#{row_class} must inherit from CSVRow" unless row_class < CSVRow
9
+ @row_class = row_class
10
+ @col_sep = col_sep
11
+ @encoding = encoding
12
+
13
+ read_csv path_or_file
14
+ initialize_column_map headers
15
+ end
16
+
17
+ # yields the rows of the csv file as CSVRow objects
18
+ def each_row import_manager, rows=nil & block
19
+ each_row_hash rows do |row_hash, index|
20
+ yield @row_class.new(row_hash, index, import_manager)
21
+ end
22
+ end
23
+
24
+ # yields the rows of the csv file as simple hashes
25
+ def each_row_hash rows=nil, &block
26
+ if rows
27
+ selected_rows rows, &block
28
+ else
29
+ all_rows &block
30
+ end
31
+ end
32
+
33
+ def row_count
34
+ @rows.size
35
+ end
36
+
37
+ private
38
+
39
+ def read_csv path_or_file
40
+ @rows =
41
+ if path_or_file.respond_to?(:read)
42
+ read_csv_from_file_handle path_or_file
43
+ else
44
+ read_csv_from_path path_or_file
45
+ end
46
+ end
47
+
48
+ def read_csv_from_path path
49
+ raise StandardError, "file does not exist: #{path}" unless File.exist? path
50
+ rescue_encoding_error do
51
+ CSV.read path, csv_options
52
+ end
53
+ end
54
+
55
+ def read_csv_from_file_handle file
56
+ CSV.parse to_utf_8(file.read), csv_options
57
+ end
58
+
59
+ def rescue_encoding_error
60
+ yield
61
+ rescue ArgumentError => _e
62
+ # if parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
63
+ # and convert to utf-8
64
+ with_encoding "iso-8859-1:utf-8" do
65
+ yield
66
+ end
67
+ end
68
+
69
+ def to_utf_8 str, encoding="utf-8", force=false
70
+ if force
71
+ str.force_encoding("iso-8859-1").encode("utf-8")
72
+ else
73
+ str.encode encoding
74
+ end
75
+ rescue Encoding::UndefinedConversionError => _e
76
+ # If parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
77
+ # and convert to utf-8.
78
+ # If that failed to force it to iso-8859-1 before converting it.
79
+ to_utf_8 str, "iso-8859-1", encoding == "iso-8859-1"
80
+ end
81
+
82
+ def csv_options
83
+ { col_sep: @col_sep, encoding: @encoding }
84
+ end
85
+
86
+ def with_encoding encoding
87
+ enc = @encoding
88
+ @encoding = encoding
89
+ yield
90
+ ensure
91
+ @encoding = enc
92
+ end
93
+
94
+ def all_rows
95
+ @rows.each.with_index do |row, i|
96
+ next if row.compact.empty?
97
+ yield row_to_hash(row), i
98
+ end
99
+ end
100
+
101
+ def selected_rows rows
102
+ rows.each do |index|
103
+ yield row_to_hash(@rows[index]), index
104
+ end
105
+ end
106
+
107
+ def map_headers
108
+ @col_map = {}
109
+ headers = @rows.shift.map { |h| h.to_name.key.to_sym }
110
+ @row_class.columns.each do |key|
111
+ @col_map[key] = headers.index key
112
+ raise StandardError, "column #{key} is missing" unless @col_map[key]
113
+ end
114
+ end
115
+
116
+ def header_row?
117
+ return unless first_row = @rows.first.map { |h| h.to_name.key.to_sym }
118
+ @row_class.columns.all? do |item|
119
+ first_row.include? item
120
+ end
121
+ end
122
+
123
+ def row_to_hash row
124
+ @col_map.each_with_object({}) do |(k, v), h|
125
+ h[k] = row[v]
126
+ h[k] &&= h[k].strip
127
+ end
128
+ end
129
+
130
+ def initialize_column_map header_line
131
+ if (header_line == :detect && header_row?) || header_line == true
132
+ map_headers
133
+ else
134
+ @col_map = @row_class.columns.zip((0..@row_class.columns.size)).to_h
135
+ end
136
+ end
137
+ end
data/lib/csv_row.rb ADDED
@@ -0,0 +1,167 @@
1
+ # Inherit from CSVRow to describe and process a csv row.
2
+ # CSVFile creates an instance of CSVRow for every row and calls #execute_import on it
3
+ class CSVRow
4
+ include ::Card::Model::SaveHelper
5
+ include Normalizer
6
+
7
+ @columns = []
8
+ @required = [] # array of required fields or :all
9
+
10
+ # Use column names as keys and method names as values to define normalization
11
+ # and validation methods.
12
+ # The normalization methods get the original field value as
13
+ # argument. The validation methods get the normalized value as argument.
14
+ # The return value of normalize methods replaces the field value.
15
+ # If a validate method returns false then the import fails.
16
+ @normalize = {}
17
+ @validate = {}
18
+
19
+ class << self
20
+ attr_reader :columns, :required
21
+
22
+ def normalize key
23
+ @normalize && @normalize[key]
24
+ end
25
+
26
+ def validate key
27
+ @validate && @validate[key]
28
+ end
29
+ end
30
+
31
+ attr_reader :errors, :row_index, :import_manager
32
+ attr_accessor :status, :name
33
+
34
+ delegate :add_card, :import_card, :override?, :pick_up_card_errors, to: :import_manager
35
+
36
+ def initialize row, index, import_manager=nil
37
+ @row = row
38
+ @import_manager = import_manager || ImportManager.new(nil)
39
+ @extra_data = @import_manager.extra_data(index)
40
+ @abort_on_error = true
41
+ @row_index = index # 0-based, not counting the header line
42
+ merge_corrections
43
+ end
44
+
45
+ def original_row
46
+ @row.merge @before_corrected
47
+ end
48
+
49
+ def label
50
+ label = "##{@row_index + 1}"
51
+ label += ": #{@name}" if @name
52
+ label
53
+ end
54
+
55
+ def merge_corrections
56
+ @corrections = @extra_data[:corrections]
57
+ @corrections = {} unless @corrections.is_a? Hash
58
+ @before_corrected = {}
59
+ @corrections.each do |k, v|
60
+ next unless v.present?
61
+ @before_corrected[k] = @row[k]
62
+ @row[k] = v
63
+ end
64
+ end
65
+
66
+ def execute_import
67
+ @import_manager.handle_import(self) do
68
+ prepare_import
69
+ ImportLog.debug "start import"
70
+ import
71
+ end
72
+ rescue => e
73
+ ImportLog.debug "import failed: #{e.message}"
74
+ ImportLog.debug e.backtrace
75
+ raise e
76
+ end
77
+
78
+ def prepare_import
79
+ collect_errors { check_required_fields }
80
+ normalize
81
+ collect_errors { validate }
82
+ end
83
+
84
+ def check_required_fields
85
+ required.each do |key|
86
+ error "value for #{key} missing" unless @row[key].present?
87
+ end
88
+ end
89
+
90
+ def collect_errors
91
+ @abort_on_error = false
92
+ yield
93
+ skip :failed if errors?
94
+ ensure
95
+ @abort_on_error = true
96
+ end
97
+
98
+ def skip status=:skipped
99
+ throw :skip_row, status
100
+ end
101
+
102
+ def errors?
103
+ @import_manager.errors? self
104
+ end
105
+
106
+ def errors
107
+ @import_manager.errors self
108
+ end
109
+
110
+ def error msg
111
+ @import_manager.report_error msg
112
+ skip :failed if @abort_on_error
113
+ end
114
+
115
+ def required
116
+ self.class.required == :all ? columns : self.class.required
117
+ end
118
+
119
+ def columns
120
+ self.class.columns
121
+ end
122
+
123
+ def normalize
124
+ @row.each do |k, v|
125
+ normalize_field k, v
126
+ end
127
+ end
128
+
129
+ def validate
130
+ @row.each do |k, v|
131
+ validate_field k, v
132
+ end
133
+ end
134
+
135
+ def normalize_field field, value
136
+ return unless (method_name = method_name(field, :normalize))
137
+ @row[field] = send method_name, value
138
+ end
139
+
140
+ def validate_field field, value
141
+ return unless (method_name = method_name(field, :validate))
142
+ return if send method_name, value
143
+ error "row #{@row_index + 1}: invalid value for #{field}: #{value}"
144
+ end
145
+
146
+ # @param type [:normalize, :validate]
147
+ def method_name field, type
148
+ method_name = "#{type}_#{field}".to_sym
149
+ respond_to?(method_name) ? method_name : self.class.send(type, field)
150
+ end
151
+
152
+ def [] key
153
+ @row[key]
154
+ end
155
+
156
+ def fields
157
+ @row
158
+ end
159
+
160
+ def method_missing method_name, *args
161
+ respond_to_missing?(method_name) ? @row[method_name.to_sym] : super
162
+ end
163
+
164
+ def respond_to_missing? method_name, _include_private=false
165
+ @row.keys.include? method_name
166
+ end
167
+ end
@@ -0,0 +1,12 @@
1
+ class CSVRow
2
+ # common methods to be used to normalize values
3
+ module Normalizer
4
+ def comma_list_to_pointer str
5
+ str.split(",").map(&:strip).to_pointer_content
6
+ end
7
+
8
+ def to_html value
9
+ value.gsub "\n", "<br\>"
10
+ end
11
+ end
12
+ end
data/lib/import_log.rb ADDED
@@ -0,0 +1,7 @@
1
+ class ImportLog
2
+ LogFile = Rails.root.join("log", "import.log")
3
+ class << self
4
+ cattr_accessor :logger
5
+ delegate :debug, :info, :warn, :error, :fatal, to: :logger
6
+ end
7
+ end
@@ -0,0 +1,73 @@
1
+ # ImportManager coordinates the import of a CSVFile. It defines the conflict and error
2
+ # policy. It collects all errors and provides extra data like corrections for row fields.
3
+ class ImportManager
4
+ require_dependency "import_manager/status"
5
+ include StatusLog
6
+ include Conflicts
7
+
8
+ attr_reader :conflict_strategy
9
+
10
+ def initialize csv_file, conflict_strategy=:skip, extra_data={}
11
+ @csv_file = csv_file
12
+ @conflict_strategy = conflict_strategy
13
+ @extra_data = integerfy_keys(extra_data || {})
14
+
15
+ @extra_data[:all] ||= {}
16
+ # init_import_status
17
+ @imported_keys = ::Set.new
18
+ end
19
+
20
+ def import row_indices=nil
21
+ import_rows row_indices
22
+ end
23
+
24
+ def import_rows row_indices
25
+ row_count = row_indices ? row_indices.size : @csv_file&.row_count
26
+ init_import_status row_count
27
+ @csv_file.each_row self, row_indices, &:execute_import
28
+ end
29
+
30
+ def extra_data index
31
+ (@extra_data[:all] || {}).deep_merge(@extra_data[index] || {})
32
+ end
33
+
34
+ def handle_import row
35
+ @current_row = row
36
+ status = catch(:skip_row) { yield }
37
+ status = specify_success_status status
38
+ @current_row.status = status
39
+ log_status
40
+ run_hook status
41
+ end
42
+
43
+ # used by csv rows to add additional cards
44
+ def add_card args
45
+ pick_up_card_errors do
46
+ Card.create args
47
+ end
48
+ end
49
+
50
+ def add_extra_data index, data
51
+ @extra_data[index].deep_merge! data
52
+ end
53
+
54
+ # add the final import card
55
+ def import_card card_args
56
+ @current_row.name = card_args[:name]
57
+ check_for_duplicates card_args[:name]
58
+ add_card card_args
59
+ end
60
+
61
+ private
62
+
63
+ # methods like row_imported, row_failed, etc. can be used to add additional logic
64
+ def run_hook status
65
+ row_finished @current_row if respond_to? :row_finished
66
+ hook_name = "row_#{status}".to_sym
67
+ send hook_name, @current_row if respond_to? hook_name
68
+ end
69
+
70
+ def integerfy_keys hash
71
+ hash.transform_keys { |key| key == :all ? :all : key.to_s.to_i }
72
+ end
73
+ end
@@ -0,0 +1,43 @@
1
+ # ActImportManager puts all creates and update actions that are part of the import
2
+ # under one act of a import card
3
+ class ActImportManager < ImportManager
4
+ def initialize act_card, csv_file, conflict_strategy=:skip, extra_row_data={}
5
+ @act_card = act_card
6
+ super(csv_file, conflict_strategy, extra_row_data)
7
+ end
8
+
9
+ def add_card args
10
+ handle_conflict args[:name] do |existing_card|
11
+ subcard =
12
+ if existing_card
13
+ existing_card.tap { |card| card.update_attributes args }
14
+ else
15
+ Card.create args
16
+ end
17
+ # subcard = @act_card&.add_subcard args.delete(:name), args
18
+ # subcard.director.catch_up_to_stage :validate
19
+ pick_up_card_errors { subcard }
20
+ end
21
+ end
22
+
23
+ def duplicate name
24
+ Card[name] || @act_card&.subcards&.at(name)
25
+ end
26
+
27
+ def log_status
28
+ super
29
+ @act_card&.import_status_card&.update_attributes content: @import_status.to_json
30
+ end
31
+
32
+ private
33
+
34
+ def init_import_status row_count=nil
35
+ isc = @act_card&.try :import_status_card
36
+ @import_status = isc&.real? ? isc.status : super
37
+ end
38
+
39
+ def row_finished row
40
+ return if row.status == :failed
41
+ @act_card&.mark_as_imported row.row_index
42
+ end
43
+ end