master_data_tool 0.1.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Appraisals +13 -0
- data/README.md +35 -0
- data/docker-compose.yml +19 -0
- data/exe/master_data_tool +6 -1
- data/gemfiles/.bundle/config +2 -0
- data/gemfiles/rails52.gemfile +9 -0
- data/gemfiles/rails61.gemfile +9 -0
- data/gemfiles/rails70.gemfile +9 -0
- data/lib/master_data_tool/master_data.rb +104 -106
- data/lib/master_data_tool/report/import_report.rb +0 -5
- data/lib/master_data_tool/version.rb +1 -1
- data/log/test.log +0 -0
- metadata +23 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e86a46f55a42684795554c0bc95b7ae735cb8c05f35a160b53c2499e2117971c
|
4
|
+
data.tar.gz: b76e406b2373ef43ada14c05bab2f7b79a94fd9ac37c959911dfc71470106061
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5d9c803172e97e3b3e607ce19f770de15ccf8dc42736b557abce41d90ad112c3b8e6a3d67e2a38871b2cd683492ae610f3ddc774bdfda6f0c05ced893ea3fc06
|
7
|
+
data.tar.gz: fe620d8afd70d5950d2f96a84d6db18255fa5ccc2b1fcf0c583e5dad3d52057e48507ecff0aa6e0c94ee026a38e868caf1b30e11bfec48557ca4f0fd94393d63
|
data/Appraisals
ADDED
data/README.md
CHANGED
@@ -141,6 +141,41 @@ grep 'operation:import' /tmp/dry-run.txt | grep 'label:detail' | grep 'status:ne
|
|
141
141
|
|
142
142
|
- upsert_allに移行する
|
143
143
|
|
144
|
+
## Test
|
145
|
+
|
146
|
+
docker-composeでMySQLを立ち上げてテストを実行する。
|
147
|
+
|
148
|
+
```
|
149
|
+
docker-compose up -d
|
150
|
+
```
|
151
|
+
|
152
|
+
以下のENVを設定すること。
|
153
|
+
|
154
|
+
```
|
155
|
+
export DB_HOST=127.0.0.1
|
156
|
+
export DB_PORT=`docker port master_data_tool_mysql57 3306 | cut -f 2 -d ':'`
|
157
|
+
export DB_USERNAME=root
|
158
|
+
export DB_PASSWORD=
|
159
|
+
```
|
160
|
+
|
161
|
+
- dockerでMySQLを立ち上げるたびにポートは変わるのでDB_PORTは都度設定する
|
162
|
+
- direnvを使っているならば `direnv reload` すればいい
|
163
|
+
|
164
|
+
```
|
165
|
+
cd spec/dummy-rails52/
|
166
|
+
RAILS_ENV=test bundle exec rake db:create
|
167
|
+
RAILS_ENV=test bundle exec rake db:migrate
|
168
|
+
```
|
169
|
+
|
170
|
+
## rspec
|
171
|
+
|
172
|
+
```
|
173
|
+
bundle exec appraisal rails52 rspec
|
174
|
+
bundle exec appraisal rails61 rspec
|
175
|
+
bundle exec appraisal rails70 rspec
|
176
|
+
```
|
177
|
+
|
178
|
+
|
144
179
|
## Contributing
|
145
180
|
|
146
181
|
Bug reports and pull requests are welcome on GitHub at https://github.com/taka0125/master_data_tool.
|
data/docker-compose.yml
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
version: '3.9'
|
2
|
+
|
3
|
+
x-mysql: &mysql
|
4
|
+
image: mysql:5.7
|
5
|
+
container_name: master_data_tool_mysql57
|
6
|
+
environment:
|
7
|
+
MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
|
8
|
+
platform: linux/x86_64
|
9
|
+
ports:
|
10
|
+
- 127.0.0.1::3306
|
11
|
+
volumes:
|
12
|
+
- mysql:/var/lib/mysql
|
13
|
+
|
14
|
+
services:
|
15
|
+
mysql:
|
16
|
+
<<: *mysql
|
17
|
+
|
18
|
+
volumes:
|
19
|
+
mysql:
|
data/exe/master_data_tool
CHANGED
@@ -49,7 +49,12 @@ module MasterDataTool
|
|
49
49
|
ignore_column_names: ignore_column_names,
|
50
50
|
verbose: verbose
|
51
51
|
)
|
52
|
-
executor.execute
|
52
|
+
errors = executor.execute
|
53
|
+
|
54
|
+
return if errors.empty?
|
55
|
+
|
56
|
+
message = errors.map { |error| "table:#{error.table}\tmessage:#{error.exception.message}" }.join("\n")
|
57
|
+
raise message
|
53
58
|
end
|
54
59
|
end
|
55
60
|
end
|
@@ -1,156 +1,154 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module MasterDataTool
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
attr_reader :before_count, :after_count
|
8
|
-
|
9
|
-
def initialize(csv_path, model_klass)
|
10
|
-
@csv_path = csv_path
|
11
|
-
@model_klass = model_klass
|
12
|
-
|
13
|
-
@loaded = false
|
14
|
-
|
15
|
-
@columns = []
|
16
|
-
@new_records = []
|
17
|
-
@updated_records = []
|
18
|
-
@no_change_records = []
|
19
|
-
@deleted_records = []
|
20
|
-
end
|
4
|
+
class MasterData
|
5
|
+
attr_reader :csv_path, :model_klass, :columns, :new_records, :updated_records, :no_change_records, :deleted_records
|
6
|
+
attr_reader :before_count, :after_count
|
21
7
|
|
22
|
-
|
23
|
-
|
24
|
-
|
8
|
+
def initialize(csv_path, model_klass)
|
9
|
+
@csv_path = csv_path
|
10
|
+
@model_klass = model_klass
|
25
11
|
|
26
|
-
|
27
|
-
deleted_ids = old_records_by_id.keys - csv_records_by_id.keys
|
12
|
+
@loaded = false
|
28
13
|
|
29
|
-
|
14
|
+
@columns = []
|
15
|
+
@new_records = []
|
16
|
+
@updated_records = []
|
17
|
+
@no_change_records = []
|
18
|
+
@deleted_records = []
|
19
|
+
end
|
30
20
|
|
31
|
-
|
32
|
-
|
33
|
-
|
21
|
+
def load
|
22
|
+
csv = CSV.read(@csv_path, headers: true, skip_blanks: true)
|
23
|
+
old_records_by_id = @model_klass.all.index_by(&:id)
|
34
24
|
|
35
|
-
|
36
|
-
|
25
|
+
csv_records_by_id = build_records_from_csv(csv, old_records_by_id)
|
26
|
+
deleted_ids = old_records_by_id.keys - csv_records_by_id.keys
|
37
27
|
|
38
|
-
|
39
|
-
@updated_records << record
|
28
|
+
@columns = csv.headers
|
40
29
|
|
41
|
-
|
42
|
-
|
30
|
+
csv_records_by_id.each do |_, record|
|
31
|
+
if record.new_record?
|
32
|
+
@new_records << record
|
43
33
|
|
44
|
-
|
34
|
+
next
|
45
35
|
end
|
46
36
|
|
47
|
-
|
48
|
-
@
|
37
|
+
if record.has_changes_to_save?
|
38
|
+
@updated_records << record
|
39
|
+
|
40
|
+
next
|
49
41
|
end
|
50
42
|
|
51
|
-
@
|
43
|
+
@no_change_records << record
|
52
44
|
end
|
53
45
|
|
54
|
-
|
55
|
-
|
46
|
+
deleted_ids.each do |id|
|
47
|
+
@deleted_records << old_records_by_id[id]
|
56
48
|
end
|
57
49
|
|
58
|
-
|
59
|
-
|
60
|
-
end
|
50
|
+
@loaded = true
|
51
|
+
end
|
61
52
|
|
62
|
-
|
63
|
-
|
53
|
+
def import_records
|
54
|
+
new_records + updated_records + no_change_records
|
55
|
+
end
|
64
56
|
|
65
|
-
|
66
|
-
|
57
|
+
def affected_records
|
58
|
+
new_records + updated_records + deleted_records
|
59
|
+
end
|
67
60
|
|
68
|
-
|
69
|
-
|
61
|
+
def new_records
|
62
|
+
raise MasterDataTool::NotLoadedError unless @loaded
|
70
63
|
|
71
|
-
|
72
|
-
|
64
|
+
@new_records
|
65
|
+
end
|
73
66
|
|
74
|
-
|
75
|
-
|
67
|
+
def updated_records
|
68
|
+
raise MasterDataTool::NotLoadedError unless @loaded
|
76
69
|
|
77
|
-
|
78
|
-
|
70
|
+
@updated_records
|
71
|
+
end
|
79
72
|
|
80
|
-
|
81
|
-
|
73
|
+
def no_change_records
|
74
|
+
raise MasterDataTool::NotLoadedError unless @loaded
|
82
75
|
|
83
|
-
|
84
|
-
|
76
|
+
@no_change_records
|
77
|
+
end
|
85
78
|
|
86
|
-
|
87
|
-
|
88
|
-
end
|
79
|
+
def deleted_records
|
80
|
+
raise MasterDataTool::NotLoadedError unless @loaded
|
89
81
|
|
90
|
-
|
91
|
-
|
92
|
-
@affected = affected_records.any?
|
93
|
-
end
|
82
|
+
@deleted_records
|
83
|
+
end
|
94
84
|
|
95
|
-
|
96
|
-
|
97
|
-
|
85
|
+
def loaded?
|
86
|
+
@loaded
|
87
|
+
end
|
98
88
|
|
99
|
-
|
100
|
-
|
101
|
-
|
89
|
+
def affected?
|
90
|
+
return @affected if instance_variable_defined?(:@affected)
|
91
|
+
@affected = affected_records.any?
|
92
|
+
end
|
102
93
|
|
103
|
-
|
104
|
-
|
105
|
-
|
94
|
+
def before_count
|
95
|
+
@before_count ||= updated_records.count + no_change_records.count + deleted_records.count
|
96
|
+
end
|
106
97
|
|
107
|
-
|
108
|
-
|
98
|
+
def after_count
|
99
|
+
@after_count ||= updated_records.count + no_change_records.count + new_records.count
|
100
|
+
end
|
109
101
|
|
110
|
-
|
111
|
-
|
112
|
-
|
102
|
+
def table_name
|
103
|
+
@model_klass.table_name
|
104
|
+
end
|
113
105
|
|
114
|
-
|
106
|
+
def import!(dry_run: true)
|
107
|
+
raise MasterDataTool::NotLoadedError unless @loaded
|
115
108
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
end
|
109
|
+
MasterDataTool::Report::ImportReport.new(self).tap do |report|
|
110
|
+
return report if dry_run
|
111
|
+
return report unless affected?
|
120
112
|
|
121
|
-
|
122
|
-
MasterDataTool::Report::VerifyReport.new(self).tap do |report|
|
123
|
-
@model_klass.all.find_each do |record|
|
124
|
-
valid = record.valid?
|
125
|
-
report.append(MasterDataTool::Report::VerifyReport.build_verify_record_report(self, record, valid))
|
126
|
-
next if dry_run
|
113
|
+
@model_klass.delete_all
|
127
114
|
|
128
|
-
|
129
|
-
|
130
|
-
end
|
115
|
+
# マスターデータ間の依存がある場合に投入順制御するのは大変なのでこのタイミングでのバリデーションはしない
|
116
|
+
@model_klass.import(import_records, validate: false, on_duplicate_key_update: @columns, timestamps: true)
|
131
117
|
end
|
118
|
+
end
|
132
119
|
|
133
|
-
|
134
|
-
|
135
|
-
|
120
|
+
def verify!(dry_run: true)
|
121
|
+
MasterDataTool::Report::VerifyReport.new(self).tap do |report|
|
122
|
+
@model_klass.all.find_each do |record|
|
123
|
+
valid = record.valid?
|
124
|
+
report.append(MasterDataTool::Report::VerifyReport.build_verify_record_report(self, record, valid))
|
125
|
+
next if dry_run
|
136
126
|
|
137
|
-
|
127
|
+
raise MasterDataTool::VerifyFailed.new("[#{table_name}] id = #{record.id} is invalid") unless valid
|
128
|
+
end
|
138
129
|
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def print_affected_table
|
133
|
+
return unless loaded?
|
134
|
+
return unless affected?
|
139
135
|
|
140
|
-
|
136
|
+
MasterDataTool::Report::PrintAffectedTableReport.new(self)
|
137
|
+
end
|
141
138
|
|
142
|
-
|
143
|
-
{}.tap do |records|
|
144
|
-
csv.each do |row|
|
145
|
-
id = row['id'].to_i
|
146
|
-
record = old_records_by_id[id] || @model_klass.new(id: id)
|
139
|
+
private
|
147
140
|
|
148
|
-
|
149
|
-
|
150
|
-
|
141
|
+
def build_records_from_csv(csv, old_records_by_id)
|
142
|
+
{}.tap do |records|
|
143
|
+
csv.each do |row|
|
144
|
+
id = row['id'].to_i
|
145
|
+
record = old_records_by_id[id] || @model_klass.new(id: id)
|
151
146
|
|
152
|
-
|
147
|
+
csv.headers.each do |key|
|
148
|
+
record[key.to_s] = row[key]
|
153
149
|
end
|
150
|
+
|
151
|
+
records[id] = record
|
154
152
|
end
|
155
153
|
end
|
156
154
|
end
|
data/log/test.log
ADDED
File without changes
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: master_data_tool
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takahiro Ooishi
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-02-
|
11
|
+
date: 2022-02-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec-rails
|
@@ -24,20 +24,6 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rails
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - '='
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: 5.2.6.2
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - '='
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: 5.2.6.2
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: mysql2
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,6 +52,20 @@ dependencies:
|
|
66
52
|
- - "~>"
|
67
53
|
- !ruby/object:Gem::Version
|
68
54
|
version: '3.1'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: appraisal
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rails
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -117,12 +117,18 @@ extensions: []
|
|
117
117
|
extra_rdoc_files: []
|
118
118
|
files:
|
119
119
|
- ".rspec"
|
120
|
+
- Appraisals
|
120
121
|
- Gemfile
|
121
122
|
- README.md
|
122
123
|
- Rakefile
|
123
124
|
- bin/console
|
124
125
|
- bin/setup
|
126
|
+
- docker-compose.yml
|
125
127
|
- exe/master_data_tool
|
128
|
+
- gemfiles/.bundle/config
|
129
|
+
- gemfiles/rails52.gemfile
|
130
|
+
- gemfiles/rails61.gemfile
|
131
|
+
- gemfiles/rails70.gemfile
|
126
132
|
- lib/generators/master_data_tool/install/install_generator.rb
|
127
133
|
- lib/generators/master_data_tool/install/templates/create_master_data_statuses.rb.erb
|
128
134
|
- lib/master_data_tool.rb
|
@@ -141,6 +147,7 @@ files:
|
|
141
147
|
- lib/master_data_tool/report/printer.rb
|
142
148
|
- lib/master_data_tool/report/verify_report.rb
|
143
149
|
- lib/master_data_tool/version.rb
|
150
|
+
- log/test.log
|
144
151
|
- sig/master_data_tool.rbs
|
145
152
|
homepage: https://github.com/taka0125/master_data_tool
|
146
153
|
licenses: []
|