card-mod-csv_import 0.8

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 25acd866e5b9397247a10490a13ecda10c0abc89f77b8e99bd817aa71f16901b
4
+ data.tar.gz: a4362aeffc395a82e8eab678cfa3f559a58bf0f57c60f6a8f8cf463bc606907b
5
+ SHA512:
6
+ metadata.gz: 887b48faecdd473f0234e1ee6942d442646618b22c4efad4786040148e271ac4e2286454009fb2bf8da47410fb96d19db2f1395313bcd782e1d57b2960ddc990
7
+ data.tar.gz: d7762714d04d219e0387adba9964ac8ccb083474f0ddbade53017d9c0e7099f1ba0efb737b972b7a03fc31a67d5633ff1fc86c52fa39a6c940a36a040b5cb6be
@@ -0,0 +1,2 @@
1
+ ImportLog.logger = Logger.new(ImportLog::LogFile)
2
+ ImportLog.logger.level = "debug"
data/lib/csv_file.rb ADDED
@@ -0,0 +1,137 @@
1
+ # CSVFile loads csv data from a given path or file handle and provides methods
2
+ # to iterate over the data.
3
+ class CSVFile
4
+ # @param headers [true, false, :detect] (false) if true the import raises an error
5
+ # if the csv file has no or wrong headers
6
+ def initialize path_or_file, row_class, col_sep: ",", encoding: "utf-8", headers: false
7
+ raise ArgumentError, "no row class given" unless row_class.is_a?(Class)
8
+ raise ArgumentError, "#{row_class} must inherit from CSVRow" unless row_class < CSVRow
9
+ @row_class = row_class
10
+ @col_sep = col_sep
11
+ @encoding = encoding
12
+
13
+ read_csv path_or_file
14
+ initialize_column_map headers
15
+ end
16
+
17
+ # yields the rows of the csv file as CSVRow objects
18
+ def each_row import_manager, rows=nil & block
19
+ each_row_hash rows do |row_hash, index|
20
+ yield @row_class.new(row_hash, index, import_manager)
21
+ end
22
+ end
23
+
24
+ # yields the rows of the csv file as simple hashes
25
+ def each_row_hash rows=nil, &block
26
+ if rows
27
+ selected_rows rows, &block
28
+ else
29
+ all_rows &block
30
+ end
31
+ end
32
+
33
+ def row_count
34
+ @rows.size
35
+ end
36
+
37
+ private
38
+
39
+ def read_csv path_or_file
40
+ @rows =
41
+ if path_or_file.respond_to?(:read)
42
+ read_csv_from_file_handle path_or_file
43
+ else
44
+ read_csv_from_path path_or_file
45
+ end
46
+ end
47
+
48
+ def read_csv_from_path path
49
+ raise StandardError, "file does not exist: #{path}" unless File.exist? path
50
+ rescue_encoding_error do
51
+ CSV.read path, csv_options
52
+ end
53
+ end
54
+
55
+ def read_csv_from_file_handle file
56
+ CSV.parse to_utf_8(file.read), csv_options
57
+ end
58
+
59
+ def rescue_encoding_error
60
+ yield
61
+ rescue ArgumentError => _e
62
+ # if parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
63
+ # and convert to utf-8
64
+ with_encoding "iso-8859-1:utf-8" do
65
+ yield
66
+ end
67
+ end
68
+
69
+ def to_utf_8 str, encoding="utf-8", force=false
70
+ if force
71
+ str.force_encoding("iso-8859-1").encode("utf-8")
72
+ else
73
+ str.encode encoding
74
+ end
75
+ rescue Encoding::UndefinedConversionError => _e
76
+ # If parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
77
+ # and convert to utf-8.
78
+ # If that failed to force it to iso-8859-1 before converting it.
79
+ to_utf_8 str, "iso-8859-1", encoding == "iso-8859-1"
80
+ end
81
+
82
+ def csv_options
83
+ { col_sep: @col_sep, encoding: @encoding }
84
+ end
85
+
86
+ def with_encoding encoding
87
+ enc = @encoding
88
+ @encoding = encoding
89
+ yield
90
+ ensure
91
+ @encoding = enc
92
+ end
93
+
94
+ def all_rows
95
+ @rows.each.with_index do |row, i|
96
+ next if row.compact.empty?
97
+ yield row_to_hash(row), i
98
+ end
99
+ end
100
+
101
+ def selected_rows rows
102
+ rows.each do |index|
103
+ yield row_to_hash(@rows[index]), index
104
+ end
105
+ end
106
+
107
+ def map_headers
108
+ @col_map = {}
109
+ headers = @rows.shift.map { |h| h.to_name.key.to_sym }
110
+ @row_class.columns.each do |key|
111
+ @col_map[key] = headers.index key
112
+ raise StandardError, "column #{key} is missing" unless @col_map[key]
113
+ end
114
+ end
115
+
116
+ def header_row?
117
+ return unless first_row = @rows.first.map { |h| h.to_name.key.to_sym }
118
+ @row_class.columns.all? do |item|
119
+ first_row.include? item
120
+ end
121
+ end
122
+
123
+ def row_to_hash row
124
+ @col_map.each_with_object({}) do |(k, v), h|
125
+ h[k] = row[v]
126
+ h[k] &&= h[k].strip
127
+ end
128
+ end
129
+
130
+ def initialize_column_map header_line
131
+ if (header_line == :detect && header_row?) || header_line == true
132
+ map_headers
133
+ else
134
+ @col_map = @row_class.columns.zip((0..@row_class.columns.size)).to_h
135
+ end
136
+ end
137
+ end
data/lib/csv_row.rb ADDED
@@ -0,0 +1,167 @@
1
+ # Inherit from CSVRow to describe and process a csv row.
2
+ # CSVFile creates an instance of CSVRow for every row and calls #execute_import on it
3
+ class CSVRow
4
+ include ::Card::Model::SaveHelper
5
+ include Normalizer
6
+
7
+ @columns = []
8
+ @required = [] # array of required fields or :all
9
+
10
+ # Use column names as keys and method names as values to define normalization
11
+ # and validation methods.
12
+ # The normalization methods get the original field value as
13
+ # argument. The validation methods get the normalized value as argument.
14
+ # The return value of normalize methods replaces the field value.
15
+ # If a validate method returns false then the import fails.
16
+ @normalize = {}
17
+ @validate = {}
18
+
19
+ class << self
20
+ attr_reader :columns, :required
21
+
22
+ def normalize key
23
+ @normalize && @normalize[key]
24
+ end
25
+
26
+ def validate key
27
+ @validate && @validate[key]
28
+ end
29
+ end
30
+
31
+ attr_reader :errors, :row_index, :import_manager
32
+ attr_accessor :status, :name
33
+
34
+ delegate :add_card, :import_card, :override?, :pick_up_card_errors, to: :import_manager
35
+
36
+ def initialize row, index, import_manager=nil
37
+ @row = row
38
+ @import_manager = import_manager || ImportManager.new(nil)
39
+ @extra_data = @import_manager.extra_data(index)
40
+ @abort_on_error = true
41
+ @row_index = index # 0-based, not counting the header line
42
+ merge_corrections
43
+ end
44
+
45
+ def original_row
46
+ @row.merge @before_corrected
47
+ end
48
+
49
+ def label
50
+ label = "##{@row_index + 1}"
51
+ label += ": #{@name}" if @name
52
+ label
53
+ end
54
+
55
+ def merge_corrections
56
+ @corrections = @extra_data[:corrections]
57
+ @corrections = {} unless @corrections.is_a? Hash
58
+ @before_corrected = {}
59
+ @corrections.each do |k, v|
60
+ next unless v.present?
61
+ @before_corrected[k] = @row[k]
62
+ @row[k] = v
63
+ end
64
+ end
65
+
66
+ def execute_import
67
+ @import_manager.handle_import(self) do
68
+ prepare_import
69
+ ImportLog.debug "start import"
70
+ import
71
+ end
72
+ rescue => e
73
+ ImportLog.debug "import failed: #{e.message}"
74
+ ImportLog.debug e.backtrace
75
+ raise e
76
+ end
77
+
78
+ def prepare_import
79
+ collect_errors { check_required_fields }
80
+ normalize
81
+ collect_errors { validate }
82
+ end
83
+
84
+ def check_required_fields
85
+ required.each do |key|
86
+ error "value for #{key} missing" unless @row[key].present?
87
+ end
88
+ end
89
+
90
+ def collect_errors
91
+ @abort_on_error = false
92
+ yield
93
+ skip :failed if errors?
94
+ ensure
95
+ @abort_on_error = true
96
+ end
97
+
98
+ def skip status=:skipped
99
+ throw :skip_row, status
100
+ end
101
+
102
+ def errors?
103
+ @import_manager.errors? self
104
+ end
105
+
106
+ def errors
107
+ @import_manager.errors self
108
+ end
109
+
110
+ def error msg
111
+ @import_manager.report_error msg
112
+ skip :failed if @abort_on_error
113
+ end
114
+
115
+ def required
116
+ self.class.required == :all ? columns : self.class.required
117
+ end
118
+
119
+ def columns
120
+ self.class.columns
121
+ end
122
+
123
+ def normalize
124
+ @row.each do |k, v|
125
+ normalize_field k, v
126
+ end
127
+ end
128
+
129
+ def validate
130
+ @row.each do |k, v|
131
+ validate_field k, v
132
+ end
133
+ end
134
+
135
+ def normalize_field field, value
136
+ return unless (method_name = method_name(field, :normalize))
137
+ @row[field] = send method_name, value
138
+ end
139
+
140
+ def validate_field field, value
141
+ return unless (method_name = method_name(field, :validate))
142
+ return if send method_name, value
143
+ error "row #{@row_index + 1}: invalid value for #{field}: #{value}"
144
+ end
145
+
146
+ # @param type [:normalize, :validate]
147
+ def method_name field, type
148
+ method_name = "#{type}_#{field}".to_sym
149
+ respond_to?(method_name) ? method_name : self.class.send(type, field)
150
+ end
151
+
152
+ def [] key
153
+ @row[key]
154
+ end
155
+
156
+ def fields
157
+ @row
158
+ end
159
+
160
+ def method_missing method_name, *args
161
+ respond_to_missing?(method_name) ? @row[method_name.to_sym] : super
162
+ end
163
+
164
+ def respond_to_missing? method_name, _include_private=false
165
+ @row.keys.include? method_name
166
+ end
167
+ end
@@ -0,0 +1,12 @@
1
+ class CSVRow
2
+ # common methods to be used to normalize values
3
+ module Normalizer
4
+ def comma_list_to_pointer str
5
+ str.split(",").map(&:strip).to_pointer_content
6
+ end
7
+
8
+ def to_html value
9
+ value.gsub "\n", "<br\>"
10
+ end
11
+ end
12
+ end
data/lib/import_log.rb ADDED
@@ -0,0 +1,7 @@
1
+ class ImportLog
2
+ LogFile = Rails.root.join("log", "import.log")
3
+ class << self
4
+ cattr_accessor :logger
5
+ delegate :debug, :info, :warn, :error, :fatal, to: :logger
6
+ end
7
+ end
@@ -0,0 +1,73 @@
1
+ # ImportManager coordinates the import of a CSVFile. It defines the conflict and error
2
+ # policy. It collects all errors and provides extra data like corrections for row fields.
3
+ class ImportManager
4
+ require_dependency "import_manager/status"
5
+ include StatusLog
6
+ include Conflicts
7
+
8
+ attr_reader :conflict_strategy
9
+
10
+ def initialize csv_file, conflict_strategy=:skip, extra_data={}
11
+ @csv_file = csv_file
12
+ @conflict_strategy = conflict_strategy
13
+ @extra_data = integerfy_keys(extra_data || {})
14
+
15
+ @extra_data[:all] ||= {}
16
+ # init_import_status
17
+ @imported_keys = ::Set.new
18
+ end
19
+
20
+ def import row_indices=nil
21
+ import_rows row_indices
22
+ end
23
+
24
+ def import_rows row_indices
25
+ row_count = row_indices ? row_indices.size : @csv_file&.row_count
26
+ init_import_status row_count
27
+ @csv_file.each_row self, row_indices, &:execute_import
28
+ end
29
+
30
+ def extra_data index
31
+ (@extra_data[:all] || {}).deep_merge(@extra_data[index] || {})
32
+ end
33
+
34
+ def handle_import row
35
+ @current_row = row
36
+ status = catch(:skip_row) { yield }
37
+ status = specify_success_status status
38
+ @current_row.status = status
39
+ log_status
40
+ run_hook status
41
+ end
42
+
43
+ # used by csv rows to add additional cards
44
+ def add_card args
45
+ pick_up_card_errors do
46
+ Card.create args
47
+ end
48
+ end
49
+
50
+ def add_extra_data index, data
51
+ @extra_data[index].deep_merge! data
52
+ end
53
+
54
+ # add the final import card
55
+ def import_card card_args
56
+ @current_row.name = card_args[:name]
57
+ check_for_duplicates card_args[:name]
58
+ add_card card_args
59
+ end
60
+
61
+ private
62
+
63
+ # methods like row_imported, row_failed, etc. can be used to add additional logic
64
+ def run_hook status
65
+ row_finished @current_row if respond_to? :row_finished
66
+ hook_name = "row_#{status}".to_sym
67
+ send hook_name, @current_row if respond_to? hook_name
68
+ end
69
+
70
+ def integerfy_keys hash
71
+ hash.transform_keys { |key| key == :all ? :all : key.to_s.to_i }
72
+ end
73
+ end
@@ -0,0 +1,43 @@
1
+ # ActImportManager puts all creates and update actions that are part of the import
2
+ # under one act of a import card
3
+ class ActImportManager < ImportManager
4
+ def initialize act_card, csv_file, conflict_strategy=:skip, extra_row_data={}
5
+ @act_card = act_card
6
+ super(csv_file, conflict_strategy, extra_row_data)
7
+ end
8
+
9
+ def add_card args
10
+ handle_conflict args[:name] do |existing_card|
11
+ subcard =
12
+ if existing_card
13
+ existing_card.tap { |card| card.update_attributes args }
14
+ else
15
+ Card.create args
16
+ end
17
+ # subcard = @act_card&.add_subcard args.delete(:name), args
18
+ # subcard.director.catch_up_to_stage :validate
19
+ pick_up_card_errors { subcard }
20
+ end
21
+ end
22
+
23
+ def duplicate name
24
+ Card[name] || @act_card&.subcards&.at(name)
25
+ end
26
+
27
+ def log_status
28
+ super
29
+ @act_card&.import_status_card&.update_attributes content: @import_status.to_json
30
+ end
31
+
32
+ private
33
+
34
+ def init_import_status row_count=nil
35
+ isc = @act_card&.try :import_status_card
36
+ @import_status = isc&.real? ? isc.status : super
37
+ end
38
+
39
+ def row_finished row
40
+ return if row.status == :failed
41
+ @act_card&.mark_as_imported row.row_index
42
+ end
43
+ end