card-mod-csv_import 0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/config/initializers/import_logger.rb +2 -0
- data/lib/csv_file.rb +137 -0
- data/lib/csv_row.rb +167 -0
- data/lib/csv_row/normalizer.rb +12 -0
- data/lib/import_log.rb +7 -0
- data/lib/import_manager.rb +73 -0
- data/lib/import_manager/act_import_manager.rb +43 -0
- data/lib/import_manager/conflicts.rb +50 -0
- data/lib/import_manager/script_import_manager.rb +48 -0
- data/lib/import_manager/status.rb +50 -0
- data/lib/import_manager/status/counts.rb +31 -0
- data/lib/import_manager/status_log.rb +80 -0
- data/lib/import_manager/validation_manager.rb +40 -0
- data/set/abstract/import.rb +89 -0
- data/set/abstract/import/01_table_row.rb +112 -0
- data/set/abstract/import/execute_import.rb +59 -0
- data/set/abstract/import/import_page.rb +82 -0
- data/set/abstract/import/table.rb +76 -0
- data/set/all/import_act.rb +8 -0
- data/set/right/import_status.rb +163 -0
- data/set/right/imported_rows.rb +20 -0
- data/set/type/file.rb +10 -0
- data/template/self/import_tool.haml +3 -0
- metadata +83 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 25acd866e5b9397247a10490a13ecda10c0abc89f77b8e99bd817aa71f16901b
|
4
|
+
data.tar.gz: a4362aeffc395a82e8eab678cfa3f559a58bf0f57c60f6a8f8cf463bc606907b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 887b48faecdd473f0234e1ee6942d442646618b22c4efad4786040148e271ac4e2286454009fb2bf8da47410fb96d19db2f1395313bcd782e1d57b2960ddc990
|
7
|
+
data.tar.gz: d7762714d04d219e0387adba9964ac8ccb083474f0ddbade53017d9c0e7099f1ba0efb737b972b7a03fc31a67d5633ff1fc86c52fa39a6c940a36a040b5cb6be
|
data/lib/csv_file.rb
ADDED
@@ -0,0 +1,137 @@
|
|
1
|
+
# CSVFile loads csv data from a given path or file handle and provides methods
|
2
|
+
# to iterate over the data.
|
3
|
+
class CSVFile
|
4
|
+
# @param headers [true, false, :detect] (false) if true the import raises an error
|
5
|
+
# if the csv file has no or wrong headers
|
6
|
+
def initialize path_or_file, row_class, col_sep: ",", encoding: "utf-8", headers: false
|
7
|
+
raise ArgumentError, "no row class given" unless row_class.is_a?(Class)
|
8
|
+
raise ArgumentError, "#{row_class} must inherit from CSVRow" unless row_class < CSVRow
|
9
|
+
@row_class = row_class
|
10
|
+
@col_sep = col_sep
|
11
|
+
@encoding = encoding
|
12
|
+
|
13
|
+
read_csv path_or_file
|
14
|
+
initialize_column_map headers
|
15
|
+
end
|
16
|
+
|
17
|
+
# yields the rows of the csv file as CSVRow objects
|
18
|
+
def each_row import_manager, rows=nil & block
|
19
|
+
each_row_hash rows do |row_hash, index|
|
20
|
+
yield @row_class.new(row_hash, index, import_manager)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# yields the rows of the csv file as simple hashes
|
25
|
+
def each_row_hash rows=nil, &block
|
26
|
+
if rows
|
27
|
+
selected_rows rows, &block
|
28
|
+
else
|
29
|
+
all_rows &block
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def row_count
|
34
|
+
@rows.size
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def read_csv path_or_file
|
40
|
+
@rows =
|
41
|
+
if path_or_file.respond_to?(:read)
|
42
|
+
read_csv_from_file_handle path_or_file
|
43
|
+
else
|
44
|
+
read_csv_from_path path_or_file
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def read_csv_from_path path
|
49
|
+
raise StandardError, "file does not exist: #{path}" unless File.exist? path
|
50
|
+
rescue_encoding_error do
|
51
|
+
CSV.read path, csv_options
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def read_csv_from_file_handle file
|
56
|
+
CSV.parse to_utf_8(file.read), csv_options
|
57
|
+
end
|
58
|
+
|
59
|
+
def rescue_encoding_error
|
60
|
+
yield
|
61
|
+
rescue ArgumentError => _e
|
62
|
+
# if parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
|
63
|
+
# and convert to utf-8
|
64
|
+
with_encoding "iso-8859-1:utf-8" do
|
65
|
+
yield
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def to_utf_8 str, encoding="utf-8", force=false
|
70
|
+
if force
|
71
|
+
str.force_encoding("iso-8859-1").encode("utf-8")
|
72
|
+
else
|
73
|
+
str.encode encoding
|
74
|
+
end
|
75
|
+
rescue Encoding::UndefinedConversionError => _e
|
76
|
+
# If parsing with utf-8 encoding fails, assume it's iso-8859-1 encoding
|
77
|
+
# and convert to utf-8.
|
78
|
+
# If that failed to force it to iso-8859-1 before converting it.
|
79
|
+
to_utf_8 str, "iso-8859-1", encoding == "iso-8859-1"
|
80
|
+
end
|
81
|
+
|
82
|
+
def csv_options
|
83
|
+
{ col_sep: @col_sep, encoding: @encoding }
|
84
|
+
end
|
85
|
+
|
86
|
+
def with_encoding encoding
|
87
|
+
enc = @encoding
|
88
|
+
@encoding = encoding
|
89
|
+
yield
|
90
|
+
ensure
|
91
|
+
@encoding = enc
|
92
|
+
end
|
93
|
+
|
94
|
+
def all_rows
|
95
|
+
@rows.each.with_index do |row, i|
|
96
|
+
next if row.compact.empty?
|
97
|
+
yield row_to_hash(row), i
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def selected_rows rows
|
102
|
+
rows.each do |index|
|
103
|
+
yield row_to_hash(@rows[index]), index
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def map_headers
|
108
|
+
@col_map = {}
|
109
|
+
headers = @rows.shift.map { |h| h.to_name.key.to_sym }
|
110
|
+
@row_class.columns.each do |key|
|
111
|
+
@col_map[key] = headers.index key
|
112
|
+
raise StandardError, "column #{key} is missing" unless @col_map[key]
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def header_row?
|
117
|
+
return unless first_row = @rows.first.map { |h| h.to_name.key.to_sym }
|
118
|
+
@row_class.columns.all? do |item|
|
119
|
+
first_row.include? item
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def row_to_hash row
|
124
|
+
@col_map.each_with_object({}) do |(k, v), h|
|
125
|
+
h[k] = row[v]
|
126
|
+
h[k] &&= h[k].strip
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def initialize_column_map header_line
|
131
|
+
if (header_line == :detect && header_row?) || header_line == true
|
132
|
+
map_headers
|
133
|
+
else
|
134
|
+
@col_map = @row_class.columns.zip((0..@row_class.columns.size)).to_h
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
data/lib/csv_row.rb
ADDED
@@ -0,0 +1,167 @@
|
|
1
|
+
# Inherit from CSVRow to describe and process a csv row.
|
2
|
+
# CSVFile creates an instance of CSVRow for every row and calls #execute_import on it
|
3
|
+
class CSVRow
|
4
|
+
include ::Card::Model::SaveHelper
|
5
|
+
include Normalizer
|
6
|
+
|
7
|
+
@columns = []
|
8
|
+
@required = [] # array of required fields or :all
|
9
|
+
|
10
|
+
# Use column names as keys and method names as values to define normalization
|
11
|
+
# and validation methods.
|
12
|
+
# The normalization methods get the original field value as
|
13
|
+
# argument. The validation methods get the normalized value as argument.
|
14
|
+
# The return value of normalize methods replaces the field value.
|
15
|
+
# If a validate method returns false then the import fails.
|
16
|
+
@normalize = {}
|
17
|
+
@validate = {}
|
18
|
+
|
19
|
+
class << self
|
20
|
+
attr_reader :columns, :required
|
21
|
+
|
22
|
+
def normalize key
|
23
|
+
@normalize && @normalize[key]
|
24
|
+
end
|
25
|
+
|
26
|
+
def validate key
|
27
|
+
@validate && @validate[key]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
attr_reader :errors, :row_index, :import_manager
|
32
|
+
attr_accessor :status, :name
|
33
|
+
|
34
|
+
delegate :add_card, :import_card, :override?, :pick_up_card_errors, to: :import_manager
|
35
|
+
|
36
|
+
def initialize row, index, import_manager=nil
|
37
|
+
@row = row
|
38
|
+
@import_manager = import_manager || ImportManager.new(nil)
|
39
|
+
@extra_data = @import_manager.extra_data(index)
|
40
|
+
@abort_on_error = true
|
41
|
+
@row_index = index # 0-based, not counting the header line
|
42
|
+
merge_corrections
|
43
|
+
end
|
44
|
+
|
45
|
+
def original_row
|
46
|
+
@row.merge @before_corrected
|
47
|
+
end
|
48
|
+
|
49
|
+
def label
|
50
|
+
label = "##{@row_index + 1}"
|
51
|
+
label += ": #{@name}" if @name
|
52
|
+
label
|
53
|
+
end
|
54
|
+
|
55
|
+
def merge_corrections
|
56
|
+
@corrections = @extra_data[:corrections]
|
57
|
+
@corrections = {} unless @corrections.is_a? Hash
|
58
|
+
@before_corrected = {}
|
59
|
+
@corrections.each do |k, v|
|
60
|
+
next unless v.present?
|
61
|
+
@before_corrected[k] = @row[k]
|
62
|
+
@row[k] = v
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def execute_import
|
67
|
+
@import_manager.handle_import(self) do
|
68
|
+
prepare_import
|
69
|
+
ImportLog.debug "start import"
|
70
|
+
import
|
71
|
+
end
|
72
|
+
rescue => e
|
73
|
+
ImportLog.debug "import failed: #{e.message}"
|
74
|
+
ImportLog.debug e.backtrace
|
75
|
+
raise e
|
76
|
+
end
|
77
|
+
|
78
|
+
def prepare_import
|
79
|
+
collect_errors { check_required_fields }
|
80
|
+
normalize
|
81
|
+
collect_errors { validate }
|
82
|
+
end
|
83
|
+
|
84
|
+
def check_required_fields
|
85
|
+
required.each do |key|
|
86
|
+
error "value for #{key} missing" unless @row[key].present?
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def collect_errors
|
91
|
+
@abort_on_error = false
|
92
|
+
yield
|
93
|
+
skip :failed if errors?
|
94
|
+
ensure
|
95
|
+
@abort_on_error = true
|
96
|
+
end
|
97
|
+
|
98
|
+
def skip status=:skipped
|
99
|
+
throw :skip_row, status
|
100
|
+
end
|
101
|
+
|
102
|
+
def errors?
|
103
|
+
@import_manager.errors? self
|
104
|
+
end
|
105
|
+
|
106
|
+
def errors
|
107
|
+
@import_manager.errors self
|
108
|
+
end
|
109
|
+
|
110
|
+
def error msg
|
111
|
+
@import_manager.report_error msg
|
112
|
+
skip :failed if @abort_on_error
|
113
|
+
end
|
114
|
+
|
115
|
+
def required
|
116
|
+
self.class.required == :all ? columns : self.class.required
|
117
|
+
end
|
118
|
+
|
119
|
+
def columns
|
120
|
+
self.class.columns
|
121
|
+
end
|
122
|
+
|
123
|
+
def normalize
|
124
|
+
@row.each do |k, v|
|
125
|
+
normalize_field k, v
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def validate
|
130
|
+
@row.each do |k, v|
|
131
|
+
validate_field k, v
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def normalize_field field, value
|
136
|
+
return unless (method_name = method_name(field, :normalize))
|
137
|
+
@row[field] = send method_name, value
|
138
|
+
end
|
139
|
+
|
140
|
+
def validate_field field, value
|
141
|
+
return unless (method_name = method_name(field, :validate))
|
142
|
+
return if send method_name, value
|
143
|
+
error "row #{@row_index + 1}: invalid value for #{field}: #{value}"
|
144
|
+
end
|
145
|
+
|
146
|
+
# @param type [:normalize, :validate]
|
147
|
+
def method_name field, type
|
148
|
+
method_name = "#{type}_#{field}".to_sym
|
149
|
+
respond_to?(method_name) ? method_name : self.class.send(type, field)
|
150
|
+
end
|
151
|
+
|
152
|
+
def [] key
|
153
|
+
@row[key]
|
154
|
+
end
|
155
|
+
|
156
|
+
def fields
|
157
|
+
@row
|
158
|
+
end
|
159
|
+
|
160
|
+
def method_missing method_name, *args
|
161
|
+
respond_to_missing?(method_name) ? @row[method_name.to_sym] : super
|
162
|
+
end
|
163
|
+
|
164
|
+
def respond_to_missing? method_name, _include_private=false
|
165
|
+
@row.keys.include? method_name
|
166
|
+
end
|
167
|
+
end
|
data/lib/import_log.rb
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
# ImportManager coordinates the import of a CSVFile. It defines the conflict and error
|
2
|
+
# policy. It collects all errors and provides extra data like corrections for row fields.
|
3
|
+
class ImportManager
|
4
|
+
require_dependency "import_manager/status"
|
5
|
+
include StatusLog
|
6
|
+
include Conflicts
|
7
|
+
|
8
|
+
attr_reader :conflict_strategy
|
9
|
+
|
10
|
+
def initialize csv_file, conflict_strategy=:skip, extra_data={}
|
11
|
+
@csv_file = csv_file
|
12
|
+
@conflict_strategy = conflict_strategy
|
13
|
+
@extra_data = integerfy_keys(extra_data || {})
|
14
|
+
|
15
|
+
@extra_data[:all] ||= {}
|
16
|
+
# init_import_status
|
17
|
+
@imported_keys = ::Set.new
|
18
|
+
end
|
19
|
+
|
20
|
+
def import row_indices=nil
|
21
|
+
import_rows row_indices
|
22
|
+
end
|
23
|
+
|
24
|
+
def import_rows row_indices
|
25
|
+
row_count = row_indices ? row_indices.size : @csv_file&.row_count
|
26
|
+
init_import_status row_count
|
27
|
+
@csv_file.each_row self, row_indices, &:execute_import
|
28
|
+
end
|
29
|
+
|
30
|
+
def extra_data index
|
31
|
+
(@extra_data[:all] || {}).deep_merge(@extra_data[index] || {})
|
32
|
+
end
|
33
|
+
|
34
|
+
def handle_import row
|
35
|
+
@current_row = row
|
36
|
+
status = catch(:skip_row) { yield }
|
37
|
+
status = specify_success_status status
|
38
|
+
@current_row.status = status
|
39
|
+
log_status
|
40
|
+
run_hook status
|
41
|
+
end
|
42
|
+
|
43
|
+
# used by csv rows to add additional cards
|
44
|
+
def add_card args
|
45
|
+
pick_up_card_errors do
|
46
|
+
Card.create args
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def add_extra_data index, data
|
51
|
+
@extra_data[index].deep_merge! data
|
52
|
+
end
|
53
|
+
|
54
|
+
# add the final import card
|
55
|
+
def import_card card_args
|
56
|
+
@current_row.name = card_args[:name]
|
57
|
+
check_for_duplicates card_args[:name]
|
58
|
+
add_card card_args
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
# methods like row_imported, row_failed, etc. can be used to add additional logic
|
64
|
+
def run_hook status
|
65
|
+
row_finished @current_row if respond_to? :row_finished
|
66
|
+
hook_name = "row_#{status}".to_sym
|
67
|
+
send hook_name, @current_row if respond_to? hook_name
|
68
|
+
end
|
69
|
+
|
70
|
+
def integerfy_keys hash
|
71
|
+
hash.transform_keys { |key| key == :all ? :all : key.to_s.to_i }
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# ActImportManager puts all creates and update actions that are part of the import
|
2
|
+
# under one act of a import card
|
3
|
+
class ActImportManager < ImportManager
|
4
|
+
def initialize act_card, csv_file, conflict_strategy=:skip, extra_row_data={}
|
5
|
+
@act_card = act_card
|
6
|
+
super(csv_file, conflict_strategy, extra_row_data)
|
7
|
+
end
|
8
|
+
|
9
|
+
def add_card args
|
10
|
+
handle_conflict args[:name] do |existing_card|
|
11
|
+
subcard =
|
12
|
+
if existing_card
|
13
|
+
existing_card.tap { |card| card.update_attributes args }
|
14
|
+
else
|
15
|
+
Card.create args
|
16
|
+
end
|
17
|
+
# subcard = @act_card&.add_subcard args.delete(:name), args
|
18
|
+
# subcard.director.catch_up_to_stage :validate
|
19
|
+
pick_up_card_errors { subcard }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def duplicate name
|
24
|
+
Card[name] || @act_card&.subcards&.at(name)
|
25
|
+
end
|
26
|
+
|
27
|
+
def log_status
|
28
|
+
super
|
29
|
+
@act_card&.import_status_card&.update_attributes content: @import_status.to_json
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def init_import_status row_count=nil
|
35
|
+
isc = @act_card&.try :import_status_card
|
36
|
+
@import_status = isc&.real? ? isc.status : super
|
37
|
+
end
|
38
|
+
|
39
|
+
def row_finished row
|
40
|
+
return if row.status == :failed
|
41
|
+
@act_card&.mark_as_imported row.row_index
|
42
|
+
end
|
43
|
+
end
|