card-mod-csv_import 0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/config/initializers/import_logger.rb +2 -0
- data/lib/csv_file.rb +137 -0
- data/lib/csv_row.rb +167 -0
- data/lib/csv_row/normalizer.rb +12 -0
- data/lib/import_log.rb +7 -0
- data/lib/import_manager.rb +73 -0
- data/lib/import_manager/act_import_manager.rb +43 -0
- data/lib/import_manager/conflicts.rb +50 -0
- data/lib/import_manager/script_import_manager.rb +48 -0
- data/lib/import_manager/status.rb +50 -0
- data/lib/import_manager/status/counts.rb +31 -0
- data/lib/import_manager/status_log.rb +80 -0
- data/lib/import_manager/validation_manager.rb +40 -0
- data/set/abstract/import.rb +89 -0
- data/set/abstract/import/01_table_row.rb +112 -0
- data/set/abstract/import/execute_import.rb +59 -0
- data/set/abstract/import/import_page.rb +82 -0
- data/set/abstract/import/table.rb +76 -0
- data/set/all/import_act.rb +8 -0
- data/set/right/import_status.rb +163 -0
- data/set/right/imported_rows.rb +20 -0
- data/set/type/file.rb +10 -0
- data/template/self/import_tool.haml +3 -0
- metadata +83 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 25acd866e5b9397247a10490a13ecda10c0abc89f77b8e99bd817aa71f16901b
|
4
|
+
data.tar.gz: a4362aeffc395a82e8eab678cfa3f559a58bf0f57c60f6a8f8cf463bc606907b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 887b48faecdd473f0234e1ee6942d442646618b22c4efad4786040148e271ac4e2286454009fb2bf8da47410fb96d19db2f1395313bcd782e1d57b2960ddc990
|
7
|
+
data.tar.gz: d7762714d04d219e0387adba9964ac8ccb083474f0ddbade53017d9c0e7099f1ba0efb737b972b7a03fc31a67d5633ff1fc86c52fa39a6c940a36a040b5cb6be
|
data/lib/csv_file.rb
ADDED
@@ -0,0 +1,137 @@
|
|
1
|
+
# CSVFile loads csv data from a given path or file handle and provides methods
|
2
|
+
# to iterate over the data.
|
3
|
+
class CSVFile
|
4
|
+
# @param headers [true, false, :detect] (false) if true the import raises an error
|
5
|
+
# if the csv file has no or wrong headers
|
6
|
+
def initialize path_or_file, row_class, col_sep: ",", encoding: "utf-8", headers: false
|
7
|
+
raise ArgumentError, "no row class given" unless row_class.is_a?(Class)
|
8
|
+
raise ArgumentError, "#{row_class} must inherit from CSVRow" unless row_class < CSVRow
|
9
|
+
@row_class = row_class
|
10
|
+
@col_sep = col_sep
|
11
|
+
@encoding = encoding
|
12
|
+
|
13
|
+
read_csv path_or_file
|
14
|
+
initialize_column_map headers
|
15
|
+
end
|
16
|
+
|
17
|
+
# yields the rows of the csv file as CSVRow objects
|
18
|
+
def each_row import_manager, rows=nil & block
|
19
|
+
each_row_hash rows do |row_hash, index|
|
20
|
+
yield @row_class.new(row_hash, index, import_manager)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# yields the rows of the csv file as simple hashes
|
25
|
+
def each_row_hash rows=nil, &block
|
26
|
+
if rows
|
27
|
+
selected_rows rows, &block
|
28
|
+
else
|
29
|
+
all_rows &block
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def row_count
|
34
|
+
@rows.size
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def read_csv path_or_file
|
40
|
+
@rows =
|
41
|
+
if path_or_file.respond_to?(:read)
|
42
|
+
read_csv_from_file_handle path_or_file
|
43
|
+
else
|
44
|
+
read_csv_from_path path_or_file
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def read_csv_from_path path
|
49
|
+
raise StandardError, "file does not exist: #{path}" unless File.exist? path
|
50
|
+
rescue_encoding_error do
|
51
|
+
CSV.read path, csv_options
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def read_csv_from_file_handle file
|
56
|
+
CSV.parse to_utf_8(file.read), csv_options
|
57
|
+
end
|
58
|
+
|
59
|
+
def rescue_encoding_error
|
60
|
+
yield
|
61
|
+
rescue ArgumentError => _e
|
62
|
+
# if parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
|
63
|
+
# and convert to utf-8
|
64
|
+
with_encoding "iso-8859-1:utf-8" do
|
65
|
+
yield
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def to_utf_8 str, encoding="utf-8", force=false
|
70
|
+
if force
|
71
|
+
str.force_encoding("iso-8859-1").encode("utf-8")
|
72
|
+
else
|
73
|
+
str.encode encoding
|
74
|
+
end
|
75
|
+
rescue Encoding::UndefinedConversionError => _e
|
76
|
+
# If parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
|
77
|
+
# and convert to utf-8.
|
78
|
+
# If that failed to force it to iso-8859-1 before converting it.
|
79
|
+
to_utf_8 str, "iso-8859-1", encoding == "iso-8859-1"
|
80
|
+
end
|
81
|
+
|
82
|
+
def csv_options
|
83
|
+
{ col_sep: @col_sep, encoding: @encoding }
|
84
|
+
end
|
85
|
+
|
86
|
+
def with_encoding encoding
|
87
|
+
enc = @encoding
|
88
|
+
@encoding = encoding
|
89
|
+
yield
|
90
|
+
ensure
|
91
|
+
@encoding = enc
|
92
|
+
end
|
93
|
+
|
94
|
+
def all_rows
|
95
|
+
@rows.each.with_index do |row, i|
|
96
|
+
next if row.compact.empty?
|
97
|
+
yield row_to_hash(row), i
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def selected_rows rows
|
102
|
+
rows.each do |index|
|
103
|
+
yield row_to_hash(@rows[index]), index
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def map_headers
|
108
|
+
@col_map = {}
|
109
|
+
headers = @rows.shift.map { |h| h.to_name.key.to_sym }
|
110
|
+
@row_class.columns.each do |key|
|
111
|
+
@col_map[key] = headers.index key
|
112
|
+
raise StandardError, "column #{key} is missing" unless @col_map[key]
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def header_row?
|
117
|
+
return unless first_row = @rows.first.map { |h| h.to_name.key.to_sym }
|
118
|
+
@row_class.columns.all? do |item|
|
119
|
+
first_row.include? item
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def row_to_hash row
|
124
|
+
@col_map.each_with_object({}) do |(k, v), h|
|
125
|
+
h[k] = row[v]
|
126
|
+
h[k] &&= h[k].strip
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def initialize_column_map header_line
|
131
|
+
if (header_line == :detect && header_row?) || header_line == true
|
132
|
+
map_headers
|
133
|
+
else
|
134
|
+
@col_map = @row_class.columns.zip((0..@row_class.columns.size)).to_h
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
data/lib/csv_row.rb
ADDED
@@ -0,0 +1,167 @@
|
|
1
|
+
# Inherit from CSVRow to describe and process a csv row.
|
2
|
+
# CSVFile creates an instance of CSVRow for every row and calls #execute_import on it
|
3
|
+
class CSVRow
|
4
|
+
include ::Card::Model::SaveHelper
|
5
|
+
include Normalizer
|
6
|
+
|
7
|
+
@columns = []
|
8
|
+
@required = [] # array of required fields or :all
|
9
|
+
|
10
|
+
# Use column names as keys and method names as values to define normalization
|
11
|
+
# and validation methods.
|
12
|
+
# The normalization methods get the original field value as
|
13
|
+
# argument. The validation methods get the normalized value as argument.
|
14
|
+
# The return value of normalize methods replaces the field value.
|
15
|
+
# If a validate method returns false then the import fails.
|
16
|
+
@normalize = {}
|
17
|
+
@validate = {}
|
18
|
+
|
19
|
+
class << self
|
20
|
+
attr_reader :columns, :required
|
21
|
+
|
22
|
+
def normalize key
|
23
|
+
@normalize && @normalize[key]
|
24
|
+
end
|
25
|
+
|
26
|
+
def validate key
|
27
|
+
@validate && @validate[key]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
attr_reader :errors, :row_index, :import_manager
|
32
|
+
attr_accessor :status, :name
|
33
|
+
|
34
|
+
delegate :add_card, :import_card, :override?, :pick_up_card_errors, to: :import_manager
|
35
|
+
|
36
|
+
def initialize row, index, import_manager=nil
|
37
|
+
@row = row
|
38
|
+
@import_manager = import_manager || ImportManager.new(nil)
|
39
|
+
@extra_data = @import_manager.extra_data(index)
|
40
|
+
@abort_on_error = true
|
41
|
+
@row_index = index # 0-based, not counting the header line
|
42
|
+
merge_corrections
|
43
|
+
end
|
44
|
+
|
45
|
+
def original_row
|
46
|
+
@row.merge @before_corrected
|
47
|
+
end
|
48
|
+
|
49
|
+
def label
|
50
|
+
label = "##{@row_index + 1}"
|
51
|
+
label += ": #{@name}" if @name
|
52
|
+
label
|
53
|
+
end
|
54
|
+
|
55
|
+
def merge_corrections
|
56
|
+
@corrections = @extra_data[:corrections]
|
57
|
+
@corrections = {} unless @corrections.is_a? Hash
|
58
|
+
@before_corrected = {}
|
59
|
+
@corrections.each do |k, v|
|
60
|
+
next unless v.present?
|
61
|
+
@before_corrected[k] = @row[k]
|
62
|
+
@row[k] = v
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def execute_import
|
67
|
+
@import_manager.handle_import(self) do
|
68
|
+
prepare_import
|
69
|
+
ImportLog.debug "start import"
|
70
|
+
import
|
71
|
+
end
|
72
|
+
rescue => e
|
73
|
+
ImportLog.debug "import failed: #{e.message}"
|
74
|
+
ImportLog.debug e.backtrace
|
75
|
+
raise e
|
76
|
+
end
|
77
|
+
|
78
|
+
def prepare_import
|
79
|
+
collect_errors { check_required_fields }
|
80
|
+
normalize
|
81
|
+
collect_errors { validate }
|
82
|
+
end
|
83
|
+
|
84
|
+
def check_required_fields
|
85
|
+
required.each do |key|
|
86
|
+
error "value for #{key} missing" unless @row[key].present?
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def collect_errors
|
91
|
+
@abort_on_error = false
|
92
|
+
yield
|
93
|
+
skip :failed if errors?
|
94
|
+
ensure
|
95
|
+
@abort_on_error = true
|
96
|
+
end
|
97
|
+
|
98
|
+
def skip status=:skipped
|
99
|
+
throw :skip_row, status
|
100
|
+
end
|
101
|
+
|
102
|
+
def errors?
|
103
|
+
@import_manager.errors? self
|
104
|
+
end
|
105
|
+
|
106
|
+
def errors
|
107
|
+
@import_manager.errors self
|
108
|
+
end
|
109
|
+
|
110
|
+
def error msg
|
111
|
+
@import_manager.report_error msg
|
112
|
+
skip :failed if @abort_on_error
|
113
|
+
end
|
114
|
+
|
115
|
+
def required
|
116
|
+
self.class.required == :all ? columns : self.class.required
|
117
|
+
end
|
118
|
+
|
119
|
+
def columns
|
120
|
+
self.class.columns
|
121
|
+
end
|
122
|
+
|
123
|
+
def normalize
|
124
|
+
@row.each do |k, v|
|
125
|
+
normalize_field k, v
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def validate
|
130
|
+
@row.each do |k, v|
|
131
|
+
validate_field k, v
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def normalize_field field, value
|
136
|
+
return unless (method_name = method_name(field, :normalize))
|
137
|
+
@row[field] = send method_name, value
|
138
|
+
end
|
139
|
+
|
140
|
+
def validate_field field, value
|
141
|
+
return unless (method_name = method_name(field, :validate))
|
142
|
+
return if send method_name, value
|
143
|
+
error "row #{@row_index + 1}: invalid value for #{field}: #{value}"
|
144
|
+
end
|
145
|
+
|
146
|
+
# @param type [:normalize, :validate]
|
147
|
+
def method_name field, type
|
148
|
+
method_name = "#{type}_#{field}".to_sym
|
149
|
+
respond_to?(method_name) ? method_name : self.class.send(type, field)
|
150
|
+
end
|
151
|
+
|
152
|
+
def [] key
|
153
|
+
@row[key]
|
154
|
+
end
|
155
|
+
|
156
|
+
def fields
|
157
|
+
@row
|
158
|
+
end
|
159
|
+
|
160
|
+
def method_missing method_name, *args
|
161
|
+
respond_to_missing?(method_name) ? @row[method_name.to_sym] : super
|
162
|
+
end
|
163
|
+
|
164
|
+
def respond_to_missing? method_name, _include_private=false
|
165
|
+
@row.keys.include? method_name
|
166
|
+
end
|
167
|
+
end
|
data/lib/import_log.rb
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
# ImportManager coordinates the import of a CSVFile. It defines the conflict and error
|
2
|
+
# policy. It collects all errors and provides extra data like corrections for row fields.
|
3
|
+
class ImportManager
|
4
|
+
require_dependency "import_manager/status"
|
5
|
+
include StatusLog
|
6
|
+
include Conflicts
|
7
|
+
|
8
|
+
attr_reader :conflict_strategy
|
9
|
+
|
10
|
+
def initialize csv_file, conflict_strategy=:skip, extra_data={}
|
11
|
+
@csv_file = csv_file
|
12
|
+
@conflict_strategy = conflict_strategy
|
13
|
+
@extra_data = integerfy_keys(extra_data || {})
|
14
|
+
|
15
|
+
@extra_data[:all] ||= {}
|
16
|
+
# init_import_status
|
17
|
+
@imported_keys = ::Set.new
|
18
|
+
end
|
19
|
+
|
20
|
+
def import row_indices=nil
|
21
|
+
import_rows row_indices
|
22
|
+
end
|
23
|
+
|
24
|
+
def import_rows row_indices
|
25
|
+
row_count = row_indices ? row_indices.size : @csv_file&.row_count
|
26
|
+
init_import_status row_count
|
27
|
+
@csv_file.each_row self, row_indices, &:execute_import
|
28
|
+
end
|
29
|
+
|
30
|
+
def extra_data index
|
31
|
+
(@extra_data[:all] || {}).deep_merge(@extra_data[index] || {})
|
32
|
+
end
|
33
|
+
|
34
|
+
def handle_import row
|
35
|
+
@current_row = row
|
36
|
+
status = catch(:skip_row) { yield }
|
37
|
+
status = specify_success_status status
|
38
|
+
@current_row.status = status
|
39
|
+
log_status
|
40
|
+
run_hook status
|
41
|
+
end
|
42
|
+
|
43
|
+
# used by csv rows to add additional cards
|
44
|
+
def add_card args
|
45
|
+
pick_up_card_errors do
|
46
|
+
Card.create args
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def add_extra_data index, data
|
51
|
+
@extra_data[index].deep_merge! data
|
52
|
+
end
|
53
|
+
|
54
|
+
# add the final import card
|
55
|
+
def import_card card_args
|
56
|
+
@current_row.name = card_args[:name]
|
57
|
+
check_for_duplicates card_args[:name]
|
58
|
+
add_card card_args
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
# methods like row_imported, row_failed, etc. can be used to add additional logic
|
64
|
+
def run_hook status
|
65
|
+
row_finished @current_row if respond_to? :row_finished
|
66
|
+
hook_name = "row_#{status}".to_sym
|
67
|
+
send hook_name, @current_row if respond_to? hook_name
|
68
|
+
end
|
69
|
+
|
70
|
+
def integerfy_keys hash
|
71
|
+
hash.transform_keys { |key| key == :all ? :all : key.to_s.to_i }
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# ActImportManager puts all creates and update actions that are part of the import
|
2
|
+
# under one act of a import card
|
3
|
+
class ActImportManager < ImportManager
|
4
|
+
def initialize act_card, csv_file, conflict_strategy=:skip, extra_row_data={}
|
5
|
+
@act_card = act_card
|
6
|
+
super(csv_file, conflict_strategy, extra_row_data)
|
7
|
+
end
|
8
|
+
|
9
|
+
def add_card args
|
10
|
+
handle_conflict args[:name] do |existing_card|
|
11
|
+
subcard =
|
12
|
+
if existing_card
|
13
|
+
existing_card.tap { |card| card.update_attributes args }
|
14
|
+
else
|
15
|
+
Card.create args
|
16
|
+
end
|
17
|
+
# subcard = @act_card&.add_subcard args.delete(:name), args
|
18
|
+
# subcard.director.catch_up_to_stage :validate
|
19
|
+
pick_up_card_errors { subcard }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def duplicate name
|
24
|
+
Card[name] || @act_card&.subcards&.at(name)
|
25
|
+
end
|
26
|
+
|
27
|
+
def log_status
|
28
|
+
super
|
29
|
+
@act_card&.import_status_card&.update_attributes content: @import_status.to_json
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def init_import_status row_count=nil
|
35
|
+
isc = @act_card&.try :import_status_card
|
36
|
+
@import_status = isc&.real? ? isc.status : super
|
37
|
+
end
|
38
|
+
|
39
|
+
def row_finished row
|
40
|
+
return if row.status == :failed
|
41
|
+
@act_card&.mark_as_imported row.row_index
|
42
|
+
end
|
43
|
+
end
|