master_data_tool 0.22.0 → 0.23.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/exe/master_data_tool +3 -65
  3. data/lib/master_data_tool/act_as_master_data.rb +13 -0
  4. data/lib/master_data_tool/command/dump.rb +43 -0
  5. data/lib/master_data_tool/command/import.rb +86 -0
  6. data/lib/master_data_tool/command/verify.rb +64 -0
  7. data/lib/master_data_tool/command.rb +5 -0
  8. data/lib/master_data_tool/config.rb +1 -1
  9. data/lib/master_data_tool/dump/config.rb +42 -0
  10. data/lib/master_data_tool/dump/executor.rb +9 -15
  11. data/lib/master_data_tool/dump.rb +1 -0
  12. data/lib/master_data_tool/import/config.rb +35 -0
  13. data/lib/master_data_tool/import/executor.rb +49 -76
  14. data/lib/master_data_tool/import.rb +1 -0
  15. data/lib/master_data_tool/master_data.rb +48 -36
  16. data/lib/master_data_tool/master_data_collection.rb +1 -1
  17. data/lib/master_data_tool/master_data_file.rb +4 -4
  18. data/lib/master_data_tool/master_data_file_collection.rb +10 -8
  19. data/lib/master_data_tool/master_data_status.rb +18 -9
  20. data/lib/master_data_tool/report/core.rb +2 -2
  21. data/lib/master_data_tool/report/default_printer.rb +1 -1
  22. data/lib/master_data_tool/report/import_report.rb +13 -15
  23. data/lib/master_data_tool/report/print_affected_table_report.rb +2 -2
  24. data/lib/master_data_tool/report/printer.rb +2 -2
  25. data/lib/master_data_tool/report/verify_report.rb +8 -8
  26. data/lib/master_data_tool/spec_config.rb +8 -10
  27. data/lib/master_data_tool/verify/config.rb +39 -0
  28. data/lib/master_data_tool/verify/executor.rb +39 -0
  29. data/lib/master_data_tool/verify.rb +4 -0
  30. data/lib/master_data_tool/version.rb +1 -1
  31. data/lib/master_data_tool.rb +21 -3
  32. data/sig/master_data_tool.rbs +323 -171
  33. metadata +38 -15
  34. data/LICENSE +0 -21
  35. data/README.md +0 -242
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: master_data_tool
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.22.0
4
+ version: 0.23.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Takahiro Ooishi
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-01-09 00:00:00.000000000 Z
11
+ date: 2024-08-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rspec
@@ -123,27 +123,27 @@ dependencies:
123
123
  - !ruby/object:Gem::Version
124
124
  version: '0'
125
125
  - !ruby/object:Gem::Dependency
126
- name: activerecord
126
+ name: rbs
127
127
  requirement: !ruby/object:Gem::Requirement
128
128
  requirements:
129
129
  - - ">="
130
130
  - !ruby/object:Gem::Version
131
- version: 5.1.7
132
- type: :runtime
131
+ version: '0'
132
+ type: :development
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - ">="
137
137
  - !ruby/object:Gem::Version
138
- version: 5.1.7
138
+ version: '0'
139
139
  - !ruby/object:Gem::Dependency
140
- name: activesupport
140
+ name: typeprof
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
143
  - - ">="
144
144
  - !ruby/object:Gem::Version
145
145
  version: '0'
146
- type: :runtime
146
+ type: :development
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
@@ -151,7 +151,21 @@ dependencies:
151
151
  - !ruby/object:Gem::Version
152
152
  version: '0'
153
153
  - !ruby/object:Gem::Dependency
154
- name: thor
154
+ name: activerecord
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: 6.0.0
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: 6.0.0
167
+ - !ruby/object:Gem::Dependency
168
+ name: activesupport
155
169
  requirement: !ruby/object:Gem::Requirement
156
170
  requirements:
157
171
  - - ">="
@@ -165,7 +179,7 @@ dependencies:
165
179
  - !ruby/object:Gem::Version
166
180
  version: '0'
167
181
  - !ruby/object:Gem::Dependency
168
- name: activerecord-import
182
+ name: thor
169
183
  requirement: !ruby/object:Gem::Requirement
170
184
  requirements:
171
185
  - - ">="
@@ -186,16 +200,21 @@ executables:
186
200
  extensions: []
187
201
  extra_rdoc_files: []
188
202
  files:
189
- - LICENSE
190
- - README.md
191
203
  - exe/master_data_tool
192
204
  - lib/generators/master_data_tool/install/install_generator.rb
193
205
  - lib/generators/master_data_tool/install/templates/create_master_data_statuses.rb.erb
194
206
  - lib/master_data_tool.rb
207
+ - lib/master_data_tool/act_as_master_data.rb
208
+ - lib/master_data_tool/command.rb
209
+ - lib/master_data_tool/command/dump.rb
210
+ - lib/master_data_tool/command/import.rb
211
+ - lib/master_data_tool/command/verify.rb
195
212
  - lib/master_data_tool/config.rb
196
213
  - lib/master_data_tool/dump.rb
214
+ - lib/master_data_tool/dump/config.rb
197
215
  - lib/master_data_tool/dump/executor.rb
198
216
  - lib/master_data_tool/import.rb
217
+ - lib/master_data_tool/import/config.rb
199
218
  - lib/master_data_tool/import/executor.rb
200
219
  - lib/master_data_tool/master_data.rb
201
220
  - lib/master_data_tool/master_data_collection.rb
@@ -210,10 +229,14 @@ files:
210
229
  - lib/master_data_tool/report/printer.rb
211
230
  - lib/master_data_tool/report/verify_report.rb
212
231
  - lib/master_data_tool/spec_config.rb
232
+ - lib/master_data_tool/verify.rb
233
+ - lib/master_data_tool/verify/config.rb
234
+ - lib/master_data_tool/verify/executor.rb
213
235
  - lib/master_data_tool/version.rb
214
236
  - sig/master_data_tool.rbs
215
237
  homepage: https://github.com/taka0125/master_data_tool
216
- licenses: []
238
+ licenses:
239
+ - MIT
217
240
  metadata:
218
241
  homepage_uri: https://github.com/taka0125/master_data_tool
219
242
  source_code_uri: https://github.com/taka0125/master_data_tool
@@ -225,14 +248,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
225
248
  requirements:
226
249
  - - ">="
227
250
  - !ruby/object:Gem::Version
228
- version: 2.6.0
251
+ version: 3.1.0
229
252
  required_rubygems_version: !ruby/object:Gem::Requirement
230
253
  requirements:
231
254
  - - ">="
232
255
  - !ruby/object:Gem::Version
233
256
  version: '0'
234
257
  requirements: []
235
- rubygems_version: 3.4.10
258
+ rubygems_version: 3.4.19
236
259
  signing_key:
237
260
  specification_version: 4
238
261
  summary: マスタデータの管理ツール
data/LICENSE DELETED
@@ -1,21 +0,0 @@
1
- MIT License
2
-
3
- Copyright (c) 2022 Takahiro Ooishi
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all
13
- copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
- SOFTWARE.
data/README.md DELETED
@@ -1,242 +0,0 @@
1
- # MasterDataTool
2
-
3
- [![Build Status](https://github.com/taka0125/master_data_tool/workflows/Ruby/badge.svg?branch=main)](https://github.com/taka0125/master_data_tool/actions)
4
- [![Gem Version](https://badge.fury.io/rb/master_data_tool.svg)](https://badge.fury.io/rb/master_data_tool)
5
- [![Maintainability](https://api.codeclimate.com/v1/badges/5fc8420c4fe83a2e6c92/maintainability)](https://codeclimate.com/github/taka0125/master_data_tool/maintainability)
6
- [![Test Coverage](https://api.codeclimate.com/v1/badges/5fc8420c4fe83a2e6c92/test_coverage)](https://codeclimate.com/github/taka0125/master_data_tool/test_coverage)
7
-
8
- システムが稼働する上で最初から必要なデータ(マスタデータ)を管理するツール
9
-
10
- 以下の機能を提供する
11
-
12
- - CSVからテーブルにデータを入れる
13
- - dry-runができる
14
- - 新規・更新・変更なし・削除がわかる
15
- - CSVのハッシュ値をDBに記録し差分があったテーブルのみ取り込みを実行する
16
- - 既存DBからCSVとしてデータをダンプする
17
-
18
- ## 前提条件
19
-
20
- - マスタデータの更新は同時並行で実行されない
21
- - `db/fixtures/#{spec_name}/#{table_name}.csv` の命名規則
22
- - 1DBの場合は `db/fixtures/#{table_name}.csv`
23
-
24
- ## インストール
25
-
26
- ```ruby
27
- gem 'master_data_tool'
28
- ```
29
-
30
- And then execute:
31
-
32
- $ bundle install
33
-
34
- Or install it yourself as:
35
-
36
- $ gem install master_data_tool
37
-
38
- ## 初期設定
39
-
40
- `config/initializers/master_data_tool.rb`
41
-
42
- ### 複数DB
43
-
44
- ```ruby
45
- Rails.application.reloader.to_prepare do
46
- MasterDataTool.configure do |config|
47
- primary_config = MasterDataTool::SpecConfig.new(
48
- spec_name: :primary,
49
- master_data_dir: Rails.root.join('db/fixtures/primary'),
50
- application_record_class: ::ApplicationRecord
51
- )
52
-
53
- animals_config = MasterDataTool::SpecConfig.new(
54
- spec_name: :animals,
55
- master_data_dir: Rails.root.join('db/fixtures/animals'),
56
- application_record_class: ::AnimalsRecord
57
- )
58
-
59
- config.spec_configs = [
60
- primary_config, animals_config
61
- ]
62
- end
63
-
64
- end
65
- ```
66
-
67
- ### 単一DB
68
-
69
- ```ruby
70
- Rails.application.reloader.to_prepare do
71
- MasterDataTool.configure do |config|
72
- primary_config = MasterDataTool::SpecConfig.new(
73
- spec_name: '',
74
- master_data_dir: Rails.root.join('db/fixtures'),
75
- application_record_class: ::ApplicationRecord
76
- )
77
-
78
- config.spec_configs = [
79
- primary_config
80
- ]
81
- end
82
- end
83
- ```
84
-
85
- ## Usage
86
-
87
- ### マスタデータの投入
88
-
89
- | option | default | 内容 |
90
- |---------------------------------|---------|-----------------------------------------------------------------|
91
- | --dry-run | true | dry-runモードで実行する(データ変更は行わない) |
92
- | --verify | true | データ投入後に全テーブル・全レコードのバリデーションチェックを行う |
93
- | --spec-name | nil | 対象となるDBのspec name |
94
- | --only-import-tables | [] | 指定したテーブルのみデータ投入を行う |
95
- | --except-import-tables | [] | 指定したテーブルのデータ投入を行わない |
96
- | --only-verify-tables | [] | 指定したテーブルのみ投入後のバリデーションチェックを行う |
97
- | --except-verify-tables | [] | 指定したテーブルのバリデーションチェックを行わない |
98
- | --skip-no-change | true | CSVファイルに更新がないテーブルをスキップする |
99
- | --silent | false | 結果の出力をやめる |
100
- | --delete-all-ignore-foreign-key | false | 外部キー制約を無視してレコードを消すかどうか |
101
- | --override-identifier | nil | fixtures/#{override_identifier} のディレクトリにある内容でfixturesを上書きして投入する |
102
-
103
- ```bash
104
- bundle exec master_data_tool import
105
- ```
106
-
107
- は以下のオプションを指定したものと一緒
108
-
109
- ```bash
110
- bundle exec thor master_data_tool import \
111
- --dry-run=true \
112
- --verify=true \
113
- --only-import-tables="" \
114
- --except-import-tables="" \
115
- --only-verify-tables="" \
116
- --except-verify-tables="" \
117
- --skip-no-change=true \
118
- --silent=false \
119
- --delete-all-ignore-foreign-key=false
120
- ```
121
-
122
- ### ダンプ
123
-
124
- | option | default | 内容 |
125
- |-----------------------|---------|-----------------|
126
- | --ignore-empty-table | true | 空のテーブルを無視する |
127
- | --ignore-tables | [] | 指定したテーブルを無視する |
128
- | --ignore-column-names | [] | 指定したカラムを無視する |
129
- | --only-tables | nil | 指定したテーブルのみダンプする |
130
- | --verbose | false | 詳細表示 |
131
-
132
- ```bash
133
- bundle exec master_data_tool dump
134
- ```
135
-
136
- は以下のオプションを指定したものと一緒
137
-
138
- ```bash
139
- bundle exec master_data_tool dump \
140
- --ignore-empty-table=true \
141
- --ignore-tables="" \
142
- --ignore-column-names="" \
143
- --only-tables="" \
144
- --verbose=false
145
- ```
146
-
147
- ## マイグレーション
148
-
149
- `master_data_statuses` というテーブルにCSVファイルのハッシュ値を記録し差分更新に利用する
150
-
151
- ```
152
- bundle exec rails generate master_data_tool:install
153
- ```
154
-
155
- を実行するとマイグレーションファイルが生成される。
156
-
157
- ridgepoleの場合は以下のような定義で実行する
158
-
159
- ```
160
- create_table 'master_data_statuses', id: :bigint, unsigned: true, force: :cascade, comment: "マスタデータの状態管理用テーブル" do |t|
161
- t.string "name", limit: 255, null: false, comment: 'テーブル名'
162
- t.string "version", limit: 255, null: false, comment: 'ハッシュ値'
163
- t.datetime "created_at", null: false, comment: '作成日時'
164
- t.datetime "updated_at", null: false, comment: '更新日時'
165
- end
166
-
167
- add_index 'master_data_statuses', ["name"], name: "idx_master_data_statuses_1", unique: true, using: :btree
168
- add_index 'master_data_statuses', ["name", "version"], name: "idx_master_data_statuses_2", using: :btree
169
- ```
170
-
171
- ## Tips
172
-
173
- ### マスタデータ投入でどうなるか?を調べる
174
-
175
- ```
176
- RAILS_ENV=development bundle exec master_data_tool import > /tmp/dry-run.txt
177
- ```
178
-
179
- - 影響を受けるテーブル
180
-
181
- ```
182
- grep 'operation:affected_table' /tmp/dry-run.txt
183
- ```
184
-
185
- - 更新されるレコード
186
-
187
- ```
188
- grep 'operation:import' /tmp/dry-run.txt | grep 'label:detail' | grep 'status:updated'
189
- ```
190
-
191
- - 削除されるレコード
192
-
193
- ```
194
- grep 'operation:import' /tmp/dry-run.txt | grep 'label:detail' | grep 'status:deleted'
195
- ```
196
-
197
- - 追加されるレコード
198
-
199
- ```
200
- grep 'operation:import' /tmp/dry-run.txt | grep 'label:detail' | grep 'status:new'
201
- ```
202
-
203
- ## TODO
204
-
205
- - upsert_allに移行する
206
-
207
- ## Test
208
-
209
- docker-composeでMySQLを立ち上げてテストを実行する。
210
-
211
- ```
212
- docker-compose up -d
213
- ```
214
-
215
- 以下のENVを設定すること。
216
-
217
- ```
218
- export DB_HOST=127.0.0.1
219
- export DB_PORT=`docker port master_data_tool_mysql57 3306 | cut -f 2 -d ':'`
220
- export DB_USERNAME=root
221
- export DB_PASSWORD=f3WpxNreVT2NgQry
222
- export DB_NAME=master_data_tool_test
223
- ```
224
-
225
- - dockerでMySQLを立ち上げるたびにポートは変わるのでDB_PORTは都度設定する
226
- - direnvを使っているならば `direnv reload` すればいい
227
-
228
- ```
229
- ./scripts/setup.sh
230
- ```
231
-
232
- ## rspec
233
-
234
- ```
235
- bundle exec appraisal activerecord52 rspec
236
- bundle exec appraisal activerecord61 rspec
237
- bundle exec appraisal activerecord70 rspec
238
- ```
239
-
240
- ## Contributing
241
-
242
- Bug reports and pull requests are welcome on GitHub at https://github.com/taka0125/master_data_tool.