master_data_tool 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/master_data_tool +3 -65
- data/lib/master_data_tool/act_as_master_data.rb +13 -0
- data/lib/master_data_tool/command/dump.rb +43 -0
- data/lib/master_data_tool/command/import.rb +86 -0
- data/lib/master_data_tool/command/verify.rb +64 -0
- data/lib/master_data_tool/command.rb +5 -0
- data/lib/master_data_tool/config.rb +2 -2
- data/lib/master_data_tool/dump/config.rb +42 -0
- data/lib/master_data_tool/dump/executor.rb +9 -15
- data/lib/master_data_tool/dump.rb +1 -0
- data/lib/master_data_tool/import/config.rb +35 -0
- data/lib/master_data_tool/import/executor.rb +49 -76
- data/lib/master_data_tool/import.rb +1 -0
- data/lib/master_data_tool/master_data.rb +48 -36
- data/lib/master_data_tool/master_data_collection.rb +1 -1
- data/lib/master_data_tool/master_data_file.rb +4 -4
- data/lib/master_data_tool/master_data_file_collection.rb +10 -8
- data/lib/master_data_tool/master_data_status.rb +18 -9
- data/lib/master_data_tool/report/core.rb +2 -2
- data/lib/master_data_tool/report/default_printer.rb +1 -1
- data/lib/master_data_tool/report/import_report.rb +13 -15
- data/lib/master_data_tool/report/print_affected_table_report.rb +2 -2
- data/lib/master_data_tool/report/printer.rb +2 -2
- data/lib/master_data_tool/report/verify_report.rb +8 -8
- data/lib/master_data_tool/spec_config.rb +8 -10
- data/lib/master_data_tool/verify/config.rb +39 -0
- data/lib/master_data_tool/verify/executor.rb +39 -0
- data/lib/master_data_tool/verify.rb +4 -0
- data/lib/master_data_tool/version.rb +1 -1
- data/lib/master_data_tool.rb +21 -3
- data/sig/master_data_tool.rbs +323 -171
- metadata +38 -15
- data/LICENSE +0 -21
- data/README.md +0 -242
@@ -3,53 +3,41 @@
|
|
3
3
|
module MasterDataTool
|
4
4
|
module Import
|
5
5
|
class Executor
|
6
|
-
def initialize(spec_config:,
|
7
|
-
dry_run: true,
|
8
|
-
|
9
|
-
only_import_tables: [],
|
10
|
-
except_import_tables: [],
|
11
|
-
only_verify_tables: [],
|
12
|
-
except_verify_tables: [],
|
13
|
-
skip_no_change: true,
|
14
|
-
silent: false,
|
15
|
-
delete_all_ignore_foreign_key: false,
|
16
|
-
override_identifier: nil,
|
17
|
-
report_printer: nil)
|
6
|
+
def initialize(spec_config:, import_config: nil, verify_config: nil,
|
7
|
+
dry_run: true, verify: true, silent: false,
|
8
|
+
override_identifier: nil, report_printer: nil)
|
18
9
|
|
19
10
|
@spec_config = spec_config
|
11
|
+
@import_config = import_config || MasterDataTool::Import::Config.default_config
|
12
|
+
@verify_config = verify_config || MasterDataTool::Verify::Config.default_config
|
13
|
+
|
20
14
|
@dry_run = dry_run
|
21
15
|
@verify = verify
|
22
|
-
@only_import_tables = Array(only_import_tables)
|
23
|
-
@except_import_tables = Array(except_import_tables)
|
24
|
-
@only_verify_tables = Array(only_verify_tables)
|
25
|
-
@except_verify_tables = Array(except_verify_tables)
|
26
|
-
@skip_no_change = skip_no_change
|
27
16
|
@silent = silent
|
28
|
-
|
17
|
+
|
29
18
|
@override_identifier = override_identifier
|
30
|
-
@report_printer = report_printer || MasterDataTool::Report::DefaultPrinter.new(spec_config)
|
19
|
+
@report_printer = report_printer || MasterDataTool::Report::DefaultPrinter.new(spec_config: spec_config)
|
31
20
|
@report_printer.silent = silent
|
32
|
-
|
21
|
+
|
22
|
+
@master_data_statuses_by_name = {}
|
33
23
|
end
|
34
24
|
|
35
25
|
def execute
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
load_master_data_statuses
|
26
|
+
transaction do
|
27
|
+
print_execute_options
|
28
|
+
load_master_data_statuses
|
40
29
|
|
41
|
-
|
30
|
+
master_data_collection = build_master_data_collection
|
42
31
|
|
43
|
-
|
44
|
-
|
45
|
-
|
32
|
+
import_all!(master_data_collection)
|
33
|
+
verify_all!(master_data_collection) if verify
|
34
|
+
save_master_data_statuses!(master_data_collection)
|
46
35
|
|
47
|
-
|
36
|
+
print_affected_tables(master_data_collection)
|
48
37
|
|
49
|
-
|
38
|
+
raise DryRunError if dry_run
|
50
39
|
|
51
|
-
|
52
|
-
end
|
40
|
+
master_data_collection
|
53
41
|
end
|
54
42
|
rescue DryRunError
|
55
43
|
puts "[DryRun] end"
|
@@ -57,25 +45,33 @@ module MasterDataTool
|
|
57
45
|
|
58
46
|
private
|
59
47
|
|
60
|
-
attr_reader :
|
61
|
-
:
|
48
|
+
attr_reader :master_data_statuses_by_name, :spec_config, :import_config, :verify_config,
|
49
|
+
:dry_run, :verify, :silent, :override_identifier, :report_printer
|
50
|
+
|
51
|
+
def transaction
|
52
|
+
spec_config.application_record_class.transaction do
|
53
|
+
MasterDataTool::MasterDataStatus.transaction do
|
54
|
+
yield
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
62
58
|
|
63
59
|
def print_execute_options
|
64
60
|
return if silent
|
65
61
|
|
66
62
|
puts "==== execute ===="
|
67
63
|
instance_variables.each do |k|
|
68
|
-
puts "#{k}: #{instance_variable_get(k)}"
|
64
|
+
puts "#{k}: #{instance_variable_get(k).inspect}"
|
69
65
|
end
|
70
66
|
puts "================="
|
71
67
|
end
|
72
68
|
|
73
69
|
def build_master_data_collection
|
74
70
|
MasterDataCollection.new.tap do |collection|
|
75
|
-
MasterDataTool::MasterDataFileCollection.new(spec_config.spec_name, override_identifier: override_identifier).each do |master_data_file|
|
71
|
+
MasterDataTool::MasterDataFileCollection.new(spec_name: spec_config.spec_name, override_identifier: override_identifier).each do |master_data_file|
|
76
72
|
load_skip = load_skip_table?(master_data_file)
|
77
|
-
master_data = MasterData.build(spec_config, master_data_file, load: !load_skip)
|
78
|
-
collection.append(master_data)
|
73
|
+
master_data = MasterData.build(spec_config: spec_config, master_data_file: master_data_file, load: !load_skip)
|
74
|
+
collection.append(master_data: master_data)
|
79
75
|
end
|
80
76
|
end
|
81
77
|
end
|
@@ -83,19 +79,19 @@ module MasterDataTool
|
|
83
79
|
def import_all!(master_data_collection)
|
84
80
|
master_data_collection.each do |master_data|
|
85
81
|
next unless master_data.loaded?
|
86
|
-
next if
|
82
|
+
next if import_config.skip_table?(master_data.table_name)
|
87
83
|
|
88
|
-
report = master_data.import!(
|
89
|
-
report.print(report_printer)
|
84
|
+
report = master_data.import!(import_config: import_config, dry_run: dry_run)
|
85
|
+
report.print(printer: report_printer)
|
90
86
|
end
|
91
87
|
end
|
92
88
|
|
93
89
|
def verify_all!(master_data_collection)
|
94
90
|
master_data_collection.each do |master_data|
|
95
|
-
next if
|
91
|
+
next if verify_config.skip_table?(master_data.table_name)
|
96
92
|
|
97
|
-
report = master_data.verify!(ignore_fail: dry_run)
|
98
|
-
report.print(report_printer)
|
93
|
+
report = master_data.verify!(verify_config: verify_config, ignore_fail: dry_run)
|
94
|
+
report.print(printer: report_printer)
|
99
95
|
end
|
100
96
|
end
|
101
97
|
|
@@ -104,10 +100,10 @@ module MasterDataTool
|
|
104
100
|
master_data_collection.each do |master_data|
|
105
101
|
next unless master_data.loaded?
|
106
102
|
|
107
|
-
records << MasterDataTool::MasterDataStatus.build(spec_config.spec_name, master_data.master_data_file)
|
103
|
+
records << MasterDataTool::MasterDataStatus.build(spec_name: spec_config.spec_name, master_data_file: master_data.master_data_file)
|
108
104
|
end
|
109
105
|
|
110
|
-
MasterDataTool::MasterDataStatus.import_records!(records, dry_run: dry_run)
|
106
|
+
MasterDataTool::MasterDataStatus.import_records!(records: records, dry_run: dry_run)
|
111
107
|
end
|
112
108
|
|
113
109
|
def print_affected_tables(master_data_collection)
|
@@ -116,57 +112,34 @@ module MasterDataTool
|
|
116
112
|
next unless master_data.affected?
|
117
113
|
|
118
114
|
report = master_data.print_affected_table
|
119
|
-
report&.print(report_printer)
|
115
|
+
report&.print(printer: report_printer)
|
120
116
|
end
|
121
117
|
end
|
122
118
|
|
123
119
|
def load_skip_table?(master_data_file)
|
124
|
-
return true if
|
125
|
-
return false unless skip_no_change
|
120
|
+
return true if import_config.skip_table?(master_data_file.table_name)
|
121
|
+
return false unless import_config.skip_no_change
|
126
122
|
|
127
|
-
master_data_status =
|
123
|
+
master_data_status = master_data_statuses_by_name[master_data_file.table_name]
|
128
124
|
return false unless master_data_status
|
129
125
|
|
130
126
|
!master_data_status.will_change?(master_data_file)
|
131
127
|
end
|
132
128
|
|
133
|
-
def import_skip_table?(table_name)
|
134
|
-
need_skip_table?(table_name, only_import_tables, except_import_tables)
|
135
|
-
end
|
136
|
-
|
137
|
-
def verify_skip_table?(table_name)
|
138
|
-
need_skip_table?(table_name, only_verify_tables, except_verify_tables)
|
139
|
-
end
|
140
|
-
|
141
|
-
# 1. onlyを指定した時点でそのリストに含まれるものだけになるべき
|
142
|
-
# 2. exceptのリストはどんな状況でも除外されるべき
|
143
|
-
# 3. それ以外はすべて実行する
|
144
|
-
def need_skip_table?(table_name, only, except)
|
145
|
-
only_result = only.presence&.include?(table_name)
|
146
|
-
except_result = except.presence&.include?(table_name)
|
147
|
-
|
148
|
-
# onlyが指定された時点でデフォルトはskipとする
|
149
|
-
default = only_result.nil? ? false : true
|
150
|
-
return true if except_result == true
|
151
|
-
return false if only_result == true
|
152
|
-
|
153
|
-
default
|
154
|
-
end
|
155
|
-
|
156
129
|
def extract_master_data_csv_paths
|
157
|
-
pattern = MasterDataTool.config.csv_dir_for(spec_config.spec_name).join('*.csv').to_s
|
130
|
+
pattern = MasterDataTool.config.csv_dir_for(spec_name: spec_config.spec_name).join('*.csv').to_s
|
158
131
|
Pathname.glob(pattern).select(&:file?)
|
159
132
|
end
|
160
133
|
|
161
134
|
def overridden_master_data_csv_paths
|
162
135
|
return [] unless override_identifier
|
163
136
|
|
164
|
-
pattern = MasterDataTool.config.csv_dir_for(spec_config.spec_name, override_identifier).join('*.csv').to_s
|
137
|
+
pattern = MasterDataTool.config.csv_dir_for(spec_name: spec_config.spec_name, override_identifier: override_identifier).join('*.csv').to_s
|
165
138
|
Pathname.glob(pattern).select(&:file?)
|
166
139
|
end
|
167
140
|
|
168
141
|
def load_master_data_statuses
|
169
|
-
@
|
142
|
+
@master_data_statuses_by_name = MasterDataTool::MasterDataStatus.all.index_by(&:name)
|
170
143
|
end
|
171
144
|
end
|
172
145
|
end
|
@@ -2,17 +2,16 @@
|
|
2
2
|
|
3
3
|
module MasterDataTool
|
4
4
|
class MasterData
|
5
|
-
|
6
|
-
:before_count, :after_count, :spec_config
|
5
|
+
BULK_INSERT_SIZE = 1000
|
7
6
|
|
8
|
-
|
9
|
-
|
7
|
+
attr_reader :master_data_file, :model_klass, :columns, :spec_config
|
8
|
+
|
9
|
+
def initialize(spec_config:, master_data_file:, model_klass:)
|
10
10
|
@spec_config = spec_config
|
11
11
|
@master_data_file = master_data_file
|
12
12
|
@model_klass = model_klass
|
13
13
|
|
14
14
|
@loaded = false
|
15
|
-
|
16
15
|
@columns = []
|
17
16
|
@new_records = []
|
18
17
|
@updated_records = []
|
@@ -21,26 +20,26 @@ module MasterDataTool
|
|
21
20
|
end
|
22
21
|
|
23
22
|
class << self
|
24
|
-
def build(spec_config
|
23
|
+
def build(spec_config:, master_data_file:, load: false)
|
25
24
|
model_klass = Object.const_get(master_data_file.table_name.classify)
|
26
|
-
new(spec_config, master_data_file, model_klass).tap do |record|
|
25
|
+
new(spec_config: spec_config, master_data_file: master_data_file, model_klass: model_klass).tap do |record|
|
27
26
|
record.load if load
|
28
27
|
end
|
29
28
|
end
|
30
29
|
end
|
31
30
|
|
32
31
|
def basename
|
33
|
-
|
32
|
+
master_data_file.basename
|
34
33
|
end
|
35
34
|
|
36
35
|
def load
|
37
|
-
csv = CSV.read(
|
38
|
-
old_records_by_id =
|
36
|
+
csv = CSV.read(master_data_file.path, headers: true, skip_blanks: true)
|
37
|
+
old_records_by_id = model_klass.all.index_by(&:id)
|
39
38
|
|
40
39
|
csv_records_by_id = build_records_from_csv(csv, old_records_by_id)
|
41
40
|
deleted_ids = old_records_by_id.keys - csv_records_by_id.keys
|
42
41
|
|
43
|
-
|
42
|
+
self.columns = csv.headers
|
44
43
|
|
45
44
|
csv_records_by_id.each do |_, record|
|
46
45
|
if record.new_record?
|
@@ -74,25 +73,25 @@ module MasterDataTool
|
|
74
73
|
end
|
75
74
|
|
76
75
|
def new_records
|
77
|
-
raise MasterDataTool::NotLoadedError unless
|
76
|
+
raise MasterDataTool::NotLoadedError unless loaded?
|
78
77
|
|
79
78
|
@new_records
|
80
79
|
end
|
81
80
|
|
82
81
|
def updated_records
|
83
|
-
raise MasterDataTool::NotLoadedError unless
|
82
|
+
raise MasterDataTool::NotLoadedError unless loaded?
|
84
83
|
|
85
84
|
@updated_records
|
86
85
|
end
|
87
86
|
|
88
87
|
def no_change_records
|
89
|
-
raise MasterDataTool::NotLoadedError unless
|
88
|
+
raise MasterDataTool::NotLoadedError unless loaded?
|
90
89
|
|
91
90
|
@no_change_records
|
92
91
|
end
|
93
92
|
|
94
93
|
def deleted_records
|
95
|
-
raise MasterDataTool::NotLoadedError unless
|
94
|
+
raise MasterDataTool::NotLoadedError unless loaded?
|
96
95
|
|
97
96
|
@deleted_records
|
98
97
|
end
|
@@ -115,34 +114,42 @@ module MasterDataTool
|
|
115
114
|
end
|
116
115
|
|
117
116
|
def table_name
|
118
|
-
|
117
|
+
model_klass.table_name
|
119
118
|
end
|
120
119
|
|
121
|
-
def import!(dry_run: true
|
122
|
-
raise MasterDataTool::NotLoadedError unless
|
120
|
+
def import!(import_config:, dry_run: true)
|
121
|
+
raise MasterDataTool::NotLoadedError unless loaded?
|
123
122
|
|
124
|
-
|
123
|
+
ignore_foreign_key_when_delete = import_config.ignore_foreign_key_when_delete
|
124
|
+
|
125
|
+
MasterDataTool::Report::ImportReport.new(master_data: self).tap do |report|
|
125
126
|
return report if dry_run
|
126
127
|
return report unless affected?
|
127
128
|
|
128
|
-
disable_foreign_key_checks if
|
129
|
-
|
130
|
-
enable_foreign_key_checks if
|
129
|
+
disable_foreign_key_checks if ignore_foreign_key_when_delete
|
130
|
+
model_klass.delete_all
|
131
|
+
enable_foreign_key_checks if ignore_foreign_key_when_delete
|
131
132
|
|
132
|
-
|
133
|
-
|
133
|
+
import_records.each_slice(BULK_INSERT_SIZE) do |chunked_import_records|
|
134
|
+
records = chunked_import_records.map { |obj| obj.attributes.slice(*columns) }
|
135
|
+
model_klass.insert_all(records)
|
136
|
+
end
|
134
137
|
end
|
135
138
|
end
|
136
139
|
|
137
|
-
def verify!(ignore_fail: false)
|
138
|
-
MasterDataTool::Report::VerifyReport.new(self).tap do |report|
|
139
|
-
|
140
|
-
|
141
|
-
|
140
|
+
def verify!(verify_config:, ignore_fail: false)
|
141
|
+
MasterDataTool::Report::VerifyReport.new(master_data: self).tap do |report|
|
142
|
+
preload_associations = decide_preload_associations(verify_config)
|
143
|
+
eager_load_associations = decide_eager_load_associations(verify_config)
|
144
|
+
|
145
|
+
scoped = model_klass.all
|
146
|
+
scoped = scoped.preload(preload_associations) unless preload_associations.empty?
|
147
|
+
scoped = scoped.eager_load(eager_load_associations) unless eager_load_associations.empty?
|
142
148
|
|
143
149
|
scoped.find_each do |record|
|
144
150
|
valid = record.valid?
|
145
|
-
report.append(MasterDataTool::Report::VerifyReport.build_verify_record_report(self, record, valid))
|
151
|
+
report.append(report: MasterDataTool::Report::VerifyReport.build_verify_record_report(master_data: self, record: record, valid: valid))
|
152
|
+
|
146
153
|
next if valid
|
147
154
|
next if ignore_fail
|
148
155
|
|
@@ -158,24 +165,29 @@ module MasterDataTool
|
|
158
165
|
return unless loaded?
|
159
166
|
return unless affected?
|
160
167
|
|
161
|
-
MasterDataTool::Report::PrintAffectedTableReport.new(self)
|
168
|
+
MasterDataTool::Report::PrintAffectedTableReport.new(master_data: self)
|
162
169
|
end
|
163
170
|
|
164
171
|
private
|
165
172
|
|
166
|
-
|
167
|
-
|
173
|
+
attr_writer :loaded, :columns, :new_records, :updated_records,
|
174
|
+
:no_change_records, :deleted_records
|
175
|
+
|
176
|
+
def decide_preload_associations(verify_config)
|
177
|
+
preload_associations = model_klass.reflections.values.select(&:belongs_to?).map(&:name).map(&:to_sym) if verify_config.preload_belongs_to_associations
|
178
|
+
preload_associations += verify_config.preload_associations.dig(model_klass.to_s.to_sym)&.map(&:to_sym) || []
|
179
|
+
preload_associations.uniq
|
168
180
|
end
|
169
181
|
|
170
|
-
def
|
171
|
-
|
182
|
+
def decide_eager_load_associations(verify_config)
|
183
|
+
verify_config.eager_load_associations.dig(model_klass.to_s.to_sym)&.map(&:to_sym) || []
|
172
184
|
end
|
173
185
|
|
174
186
|
def build_records_from_csv(csv, old_records_by_id)
|
175
187
|
{}.tap do |records|
|
176
188
|
csv.each do |row|
|
177
189
|
id = row['id'].to_i
|
178
|
-
record = old_records_by_id[id] ||
|
190
|
+
record = old_records_by_id[id] || model_klass.new(id: id)
|
179
191
|
|
180
192
|
csv.headers.each do |key|
|
181
193
|
record[key.to_s] = row[key]
|
@@ -3,7 +3,7 @@ module MasterDataTool
|
|
3
3
|
class MasterDataFile
|
4
4
|
attr_reader :spec_name, :table_name, :path, :override_identifier
|
5
5
|
|
6
|
-
def initialize(spec_name
|
6
|
+
def initialize(spec_name:, table_name:, path:, override_identifier:)
|
7
7
|
@spec_name = spec_name
|
8
8
|
@table_name = table_name
|
9
9
|
@path = path
|
@@ -12,9 +12,9 @@ module MasterDataTool
|
|
12
12
|
end
|
13
13
|
|
14
14
|
class << self
|
15
|
-
def build(spec_name
|
16
|
-
table_name = MasterDataTool.resolve_table_name(spec_name, path, override_identifier)
|
17
|
-
new(spec_name, table_name, path, override_identifier)
|
15
|
+
def build(spec_name:, path:, override_identifier:)
|
16
|
+
table_name = MasterDataTool.resolve_table_name(spec_name: spec_name, csv_path: path, override_identifier: override_identifier)
|
17
|
+
new(spec_name: spec_name, table_name: table_name, path: path, override_identifier: override_identifier)
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module MasterDataTool
|
4
4
|
class MasterDataFileCollection
|
5
|
-
def initialize(spec_name
|
5
|
+
def initialize(spec_name:, override_identifier: nil)
|
6
6
|
@spec_name = spec_name
|
7
7
|
@override_identifier = override_identifier
|
8
8
|
@collection = build
|
@@ -13,7 +13,7 @@ module MasterDataTool
|
|
13
13
|
def each
|
14
14
|
return enum_for(:each) unless block_given?
|
15
15
|
|
16
|
-
|
16
|
+
collection.each do |file|
|
17
17
|
yield file
|
18
18
|
end
|
19
19
|
end
|
@@ -24,8 +24,10 @@ module MasterDataTool
|
|
24
24
|
|
25
25
|
private
|
26
26
|
|
27
|
+
attr_reader :spec_name, :override_identifier, :collection
|
28
|
+
|
27
29
|
def build
|
28
|
-
files = extract_master_data_csv_paths.presence&.index_by(&:table_name)
|
30
|
+
files = extract_master_data_csv_paths.presence&.index_by(&:table_name) || {}
|
29
31
|
overridden_files = overridden_master_data_csv_paths.presence&.index_by(&:table_name) || {}
|
30
32
|
|
31
33
|
table_names = (files.keys + overridden_files.keys).uniq
|
@@ -35,18 +37,18 @@ module MasterDataTool
|
|
35
37
|
end
|
36
38
|
|
37
39
|
def extract_master_data_csv_paths
|
38
|
-
pattern = MasterDataTool.config.csv_dir_for(
|
40
|
+
pattern = MasterDataTool.config.csv_dir_for(spec_name: spec_name).join('*.csv').to_s
|
39
41
|
Pathname.glob(pattern).select(&:file?).map do |path|
|
40
|
-
MasterDataFile.build(
|
42
|
+
MasterDataFile.build(spec_name: spec_name, path: path, override_identifier: nil)
|
41
43
|
end
|
42
44
|
end
|
43
45
|
|
44
46
|
def overridden_master_data_csv_paths
|
45
|
-
return [] if
|
47
|
+
return [] if override_identifier.blank?
|
46
48
|
|
47
|
-
pattern = MasterDataTool.config.csv_dir_for(
|
49
|
+
pattern = MasterDataTool.config.csv_dir_for(spec_name: spec_name, override_identifier: override_identifier).join('*.csv').to_s
|
48
50
|
Pathname.glob(pattern).select(&:file?).map do |path|
|
49
|
-
MasterDataFile.build(
|
51
|
+
MasterDataFile.build(spec_name: spec_name, path: path, override_identifier: override_identifier)
|
50
52
|
end
|
51
53
|
end
|
52
54
|
end
|
@@ -1,7 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'active_record'
|
4
|
-
require 'activerecord-import'
|
5
4
|
require 'openssl'
|
6
5
|
|
7
6
|
module MasterDataTool
|
@@ -20,28 +19,38 @@ module MasterDataTool
|
|
20
19
|
version != self.class.decide_version(master_data_file.path)
|
21
20
|
end
|
22
21
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
end
|
22
|
+
def model_klass
|
23
|
+
Object.const_get(name.classify)
|
24
|
+
end
|
27
25
|
|
28
|
-
|
26
|
+
class << self
|
27
|
+
def build(spec_name:, master_data_file:)
|
29
28
|
version = decide_version(master_data_file.path)
|
30
|
-
|
29
|
+
name = MasterDataTool.resolve_table_name(spec_name: spec_name, csv_path: master_data_file.path, override_identifier: master_data_file.override_identifier)
|
30
|
+
new(spec_name: spec_name, name: name, version: version)
|
31
31
|
end
|
32
32
|
|
33
|
-
def import_records!(records
|
33
|
+
def import_records!(records:, dry_run: true)
|
34
34
|
if dry_run
|
35
35
|
pp records
|
36
36
|
return
|
37
37
|
end
|
38
38
|
|
39
|
-
|
39
|
+
return if records.empty?
|
40
|
+
|
41
|
+
import_records = records.map { |obj| obj.attributes.slice(*import_columns) }
|
42
|
+
upsert_all(import_records, update_only: %w[version])
|
40
43
|
end
|
41
44
|
|
42
45
|
def decide_version(csv_path)
|
43
46
|
OpenSSL::Digest::SHA256.hexdigest(File.open(csv_path).read)
|
44
47
|
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def import_columns
|
52
|
+
%w[spec_name name version]
|
53
|
+
end
|
45
54
|
end
|
46
55
|
end
|
47
56
|
end
|
@@ -5,14 +5,12 @@ module MasterDataTool
|
|
5
5
|
class ImportReport
|
6
6
|
include Core
|
7
7
|
|
8
|
-
|
9
|
-
|
10
|
-
def print(printer)
|
8
|
+
def print(printer:)
|
11
9
|
reports.each do |_, report|
|
12
10
|
if report.is_a?(Array)
|
13
|
-
report.each { |r| printer.print(convert_to_ltsv(r)) }
|
11
|
+
report.each { |r| printer.print(message: convert_to_ltsv(r)) }
|
14
12
|
else
|
15
|
-
printer.print(convert_to_ltsv(report))
|
13
|
+
printer.print(message: convert_to_ltsv(report))
|
16
14
|
end
|
17
15
|
end
|
18
16
|
end
|
@@ -27,12 +25,12 @@ module MasterDataTool
|
|
27
25
|
label = :count_report
|
28
26
|
{}.tap do |report|
|
29
27
|
report[label] = []
|
30
|
-
report[label] << {operation: :import, label: :count, table_name: master_data.table_name, before: master_data.before_count, after: master_data.after_count}
|
31
|
-
report[label] << {operation: :import, label: :affected, table_name: master_data.table_name, affected: master_data.affected?}
|
32
|
-
report[label] << {operation: :import, label: :new_count, table_name: master_data.table_name, count: master_data.new_records.count}
|
33
|
-
report[label] << {operation: :import, label: :updated_count, table_name: master_data.table_name, count: master_data.updated_records.count}
|
34
|
-
report[label] << {operation: :import, label: :no_change_count, table_name: master_data.table_name, count: master_data.no_change_records.count}
|
35
|
-
report[label] << {operation: :import, label: :deleted_count, table_name: master_data.table_name, count: master_data.deleted_records.count}
|
28
|
+
report[label] << { operation: :import, label: :count, table_name: master_data.table_name, before: master_data.before_count, after: master_data.after_count }
|
29
|
+
report[label] << { operation: :import, label: :affected, table_name: master_data.table_name, affected: master_data.affected? }
|
30
|
+
report[label] << { operation: :import, label: :new_count, table_name: master_data.table_name, count: master_data.new_records.count }
|
31
|
+
report[label] << { operation: :import, label: :updated_count, table_name: master_data.table_name, count: master_data.updated_records.count }
|
32
|
+
report[label] << { operation: :import, label: :no_change_count, table_name: master_data.table_name, count: master_data.no_change_records.count }
|
33
|
+
report[label] << { operation: :import, label: :deleted_count, table_name: master_data.table_name, count: master_data.deleted_records.count }
|
36
34
|
end
|
37
35
|
end
|
38
36
|
|
@@ -41,7 +39,7 @@ module MasterDataTool
|
|
41
39
|
{}.tap do |report|
|
42
40
|
report[label] = []
|
43
41
|
master_data.new_records.each do |record|
|
44
|
-
report[label] << {operation: :import, label: :detail, table_name: master_data.table_name, status: :new, id: record.id}
|
42
|
+
report[label] << { operation: :import, label: :detail, table_name: master_data.table_name, status: :new, id: record.id }
|
45
43
|
end
|
46
44
|
end
|
47
45
|
end
|
@@ -51,7 +49,7 @@ module MasterDataTool
|
|
51
49
|
{}.tap do |report|
|
52
50
|
report[label] = []
|
53
51
|
master_data.updated_records.each do |record|
|
54
|
-
report[label] << {operation: :import, label: :detail, table_name: master_data.table_name, status: :updated, id: record.id, detail: record.changes_to_save}
|
52
|
+
report[label] << { operation: :import, label: :detail, table_name: master_data.table_name, status: :updated, id: record.id, detail: record.changes_to_save }
|
55
53
|
end
|
56
54
|
end
|
57
55
|
end
|
@@ -61,7 +59,7 @@ module MasterDataTool
|
|
61
59
|
{}.tap do |report|
|
62
60
|
report[label] = []
|
63
61
|
master_data.no_change_records.each do |record|
|
64
|
-
report[label] << {operation: :import, label: :detail, table_name: master_data.table_name, status: :no_change, id: record.id}
|
62
|
+
report[label] << { operation: :import, label: :detail, table_name: master_data.table_name, status: :no_change, id: record.id }
|
65
63
|
end
|
66
64
|
end
|
67
65
|
end
|
@@ -71,7 +69,7 @@ module MasterDataTool
|
|
71
69
|
{}.tap do |report|
|
72
70
|
report[label] = []
|
73
71
|
master_data.deleted_records.each do |record|
|
74
|
-
report[label] << {operation: :import, label: :detail, table_name: master_data.table_name, status: :deleted, id: record.id}
|
72
|
+
report[label] << { operation: :import, label: :detail, table_name: master_data.table_name, status: :deleted, id: record.id }
|
75
73
|
end
|
76
74
|
end
|
77
75
|
end
|
@@ -5,8 +5,8 @@ module MasterDataTool
|
|
5
5
|
class PrintAffectedTableReport
|
6
6
|
include Core
|
7
7
|
|
8
|
-
def print(printer)
|
9
|
-
printer.print(convert_to_ltsv({operation: :affected_table, table_name: master_data.table_name}))
|
8
|
+
def print(printer:)
|
9
|
+
printer.print(message: convert_to_ltsv({ operation: :affected_table, table_name: master_data.table_name }))
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
@@ -6,12 +6,12 @@ module MasterDataTool
|
|
6
6
|
attr_reader :spec_config
|
7
7
|
attr_accessor :silent
|
8
8
|
|
9
|
-
def initialize(spec_config
|
9
|
+
def initialize(spec_config:, silent: false)
|
10
10
|
@spec_config = spec_config
|
11
11
|
@silent = silent
|
12
12
|
end
|
13
13
|
|
14
|
-
def print(message)
|
14
|
+
def print(message:)
|
15
15
|
return if silent
|
16
16
|
|
17
17
|
raise NotImplementedError
|