master_data_tool 0.19.1 → 0.21.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +63 -15
- data/exe/master_data_tool +22 -1
- data/lib/generators/master_data_tool/install/templates/create_master_data_statuses.rb.erb +3 -2
- data/lib/master_data_tool/config.rb +13 -12
- data/lib/master_data_tool/dump/executor.rb +16 -10
- data/lib/master_data_tool/import/executor.rb +35 -30
- data/lib/master_data_tool/master_data.rb +10 -9
- data/lib/master_data_tool/master_data_file.rb +7 -6
- data/lib/master_data_tool/master_data_file_collection.rb +7 -5
- data/lib/master_data_tool/master_data_status.rb +2 -2
- data/lib/master_data_tool/report/core.rb +2 -0
- data/lib/master_data_tool/report/default_printer.rb +2 -2
- data/lib/master_data_tool/report/import_report.rb +14 -14
- data/lib/master_data_tool/report/print_affected_table_report.rb +1 -1
- data/lib/master_data_tool/report/printer.rb +4 -2
- data/lib/master_data_tool/spec_config.rb +21 -0
- data/lib/master_data_tool/version.rb +1 -1
- data/lib/master_data_tool.rb +3 -3
- data/scripts/drop_db.sh +1 -1
- data/sig/master_data_tool.rbs +32 -12
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 68cf2a10239a17aee6eacda8e54cbf2820adbb8202f5fb53ef458bcaf1acb9a6
|
4
|
+
data.tar.gz: 23464ebf12f17ff06fe1dea8acc7e465510d00bbde75ba5baf00fd2a82362d4c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a0713903b340a14a89c174beabf12df1a7ba29212b7004ae9056c28fd123a885e069ef0f907635da12be1cce4a9d59527c6c11a22a365aa249e27094577391b3
|
7
|
+
data.tar.gz: 7c395995163c9035fe5930415736a8d8bc5126e649a41490546d9c4483d0943f8d48a8f6f5c6296507b5514640a63c37b35f42a003bda595f2401a1ebe6b1443
|
data/README.md
CHANGED
@@ -18,7 +18,8 @@
|
|
18
18
|
## 前提条件
|
19
19
|
|
20
20
|
- マスタデータの更新は同時並行で実行されない
|
21
|
-
- `db/fixtures/#{table_name}.csv` の命名規則
|
21
|
+
- `db/fixtures/#{spec_name}/#{table_name}.csv` の命名規則
|
22
|
+
- 1DBの場合は `db/fixtures/#{table_name}.csv`
|
22
23
|
|
23
24
|
## インストール
|
24
25
|
|
@@ -34,22 +35,70 @@ Or install it yourself as:
|
|
34
35
|
|
35
36
|
$ gem install master_data_tool
|
36
37
|
|
38
|
+
## 初期設定
|
39
|
+
|
40
|
+
`config/initializers/master_data_tool.rb`
|
41
|
+
|
42
|
+
### 複数DB
|
43
|
+
|
44
|
+
```ruby
|
45
|
+
Rails.application.reloader.to_prepare do
|
46
|
+
MasterDataTool.configure do |config|
|
47
|
+
primary_config = MasterDataTool::SpecConfig.new(
|
48
|
+
spec_name: :primary,
|
49
|
+
master_data_dir: Rails.root.join('db/fixtures/primary'),
|
50
|
+
application_record_class: ::ApplicationRecord
|
51
|
+
)
|
52
|
+
|
53
|
+
animals_config = MasterDataTool::SpecConfig.new(
|
54
|
+
spec_name: :animals,
|
55
|
+
master_data_dir: Rails.root.join('db/fixtures/animals'),
|
56
|
+
application_record_class: ::AnimalsRecord
|
57
|
+
)
|
58
|
+
|
59
|
+
config.spec_configs = [
|
60
|
+
primary_config, animals_config
|
61
|
+
]
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
```
|
66
|
+
|
67
|
+
### 単一DB
|
68
|
+
|
69
|
+
```ruby
|
70
|
+
Rails.application.reloader.to_prepare do
|
71
|
+
MasterDataTool.configure do |config|
|
72
|
+
primary_config = MasterDataTool::SpecConfig.new(
|
73
|
+
spec_name: '',
|
74
|
+
master_data_dir: Rails.root.join('db/fixtures'),
|
75
|
+
application_record_class: ::ApplicationRecord
|
76
|
+
)
|
77
|
+
|
78
|
+
config.spec_configs = [
|
79
|
+
primary_config
|
80
|
+
]
|
81
|
+
end
|
82
|
+
end
|
83
|
+
```
|
84
|
+
|
37
85
|
## Usage
|
38
86
|
|
39
87
|
### マスタデータの投入
|
40
88
|
|
41
89
|
| option | default | 内容 |
|
42
|
-
|
43
|
-
| --dry-run | true
|
44
|
-
| --verify | true
|
45
|
-
| --
|
46
|
-
| --
|
47
|
-
| --
|
48
|
-
| --
|
49
|
-
| --
|
50
|
-
| --
|
51
|
-
| --
|
52
|
-
| --
|
90
|
+
|---------------------------------|---------|-----------------------------------------------------------------|
|
91
|
+
| --dry-run | true | dry-runモードで実行する(データ変更は行わない) |
|
92
|
+
| --verify | true | データ投入後に全テーブル・全レコードのバリデーションチェックを行う |
|
93
|
+
| --spec-name | nil | 対象となるDBのspec name |
|
94
|
+
| --only-import-tables | [] | 指定したテーブルのみデータ投入を行う |
|
95
|
+
| --except-import-tables | [] | 指定したテーブルのデータ投入を行わない |
|
96
|
+
| --only-verify-tables | [] | 指定したテーブルのみ投入後のバリデーションチェックを行う |
|
97
|
+
| --except-verify-tables | [] | 指定したテーブルのバリデーションチェックを行わない |
|
98
|
+
| --skip-no-change | true | CSVファイルに更新がないテーブルをスキップする |
|
99
|
+
| --silent | false | 結果の出力をやめる |
|
100
|
+
| --delete-all-ignore-foreign-key | false | 外部キー制約を無視してレコードを消すかどうか |
|
101
|
+
| --override-identifier | nil | fixtures/#{override_identifier} のディレクトリにある内容でfixturesを上書きして投入する |
|
53
102
|
|
54
103
|
```bash
|
55
104
|
bundle exec master_data_tool import
|
@@ -119,8 +168,8 @@ add_index 'master_data_statuses', ["name"], name: "idx_master_data_statuses_1",
|
|
119
168
|
add_index 'master_data_statuses', ["name", "version"], name: "idx_master_data_statuses_2", using: :btree
|
120
169
|
```
|
121
170
|
|
122
|
-
|
123
171
|
## Tips
|
172
|
+
|
124
173
|
### マスタデータ投入でどうなるか?を調べる
|
125
174
|
|
126
175
|
```
|
@@ -174,7 +223,7 @@ export DB_NAME=master_data_tool_test
|
|
174
223
|
```
|
175
224
|
|
176
225
|
- dockerでMySQLを立ち上げるたびにポートは変わるのでDB_PORTは都度設定する
|
177
|
-
|
226
|
+
- direnvを使っているならば `direnv reload` すればいい
|
178
227
|
|
179
228
|
```
|
180
229
|
./scripts/setup.sh
|
@@ -188,7 +237,6 @@ bundle exec appraisal activerecord61 rspec
|
|
188
237
|
bundle exec appraisal activerecord70 rspec
|
189
238
|
```
|
190
239
|
|
191
|
-
|
192
240
|
## Contributing
|
193
241
|
|
194
242
|
Bug reports and pull requests are welcome on GitHub at https://github.com/taka0125/master_data_tool.
|
data/exe/master_data_tool
CHANGED
@@ -11,6 +11,7 @@ module MasterDataTool
|
|
11
11
|
class CLI < Thor
|
12
12
|
option :dry_run, default: nil, type: :boolean
|
13
13
|
option :verify, default: nil, type: :boolean
|
14
|
+
option :spec_name, default: '', type: :string
|
14
15
|
option :only_import_tables, default: nil, type: :array
|
15
16
|
option :except_import_tables, default: nil, type: :array
|
16
17
|
option :only_verify_tables, default: nil, type: :array
|
@@ -21,11 +22,29 @@ module MasterDataTool
|
|
21
22
|
option :delete_all_ignore_foreign_key, default: nil, type: :boolean
|
22
23
|
desc 'import', 'import'
|
23
24
|
def import
|
24
|
-
|
25
|
+
spec_config = MasterDataTool.config.spec_config(options['spec_name'])
|
26
|
+
new_options = config.default_import_options.with_indifferent_access.merge(options)
|
27
|
+
new_options[:spec_config] = spec_config
|
28
|
+
|
25
29
|
executor = MasterDataTool::Import::Executor.new(**new_options.symbolize_keys)
|
26
30
|
executor.execute
|
27
31
|
end
|
28
32
|
|
33
|
+
option :dry_run, default: nil, type: :boolean
|
34
|
+
option :verify, default: nil, type: :boolean
|
35
|
+
option :skip_no_change, default: nil, type: :boolean
|
36
|
+
desc 'import_all', 'import all'
|
37
|
+
def import_all
|
38
|
+
MasterDataTool.config.spec_configs.each do |spec_config|
|
39
|
+
new_options = spec_config.default_import_options.with_indifferent_access.merge(options)
|
40
|
+
new_options[:spec_config] = spec_config
|
41
|
+
|
42
|
+
executor = MasterDataTool::Import::Executor.new(**new_options.symbolize_keys)
|
43
|
+
executor.execute
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
option :spec_name, default: nil, type: :string
|
29
48
|
option :ignore_empty_table, default: true, type: :boolean
|
30
49
|
option :ignore_tables, default: [], type: :array
|
31
50
|
option :ignore_column_names, default: [], type: :array
|
@@ -38,8 +57,10 @@ module MasterDataTool
|
|
38
57
|
ignore_column_names = options[:ignore_column_names]
|
39
58
|
only_tables = options[:only_tables]
|
40
59
|
verbose = options[:verbose]
|
60
|
+
spec_config = MasterDataTool.config.spec_config(options['spec_name'])
|
41
61
|
|
42
62
|
executor = MasterDataTool::Dump::Executor.new(
|
63
|
+
spec_config: spec_config,
|
43
64
|
ignore_empty_table: ignore_empty_table,
|
44
65
|
ignore_tables: ignore_tables,
|
45
66
|
ignore_column_names: ignore_column_names,
|
@@ -1,14 +1,15 @@
|
|
1
1
|
class CreateMasterDataStatuses < ActiveRecord::Migration<%= migration_version %>
|
2
2
|
def self.up
|
3
3
|
create_table :master_data_statuses do |t|
|
4
|
+
t.string :spec_name, limit: 255, null: false, default: '', comment: 'spec name'
|
4
5
|
t.string :name, limit: 255, null: false, comment: 'テーブル名'
|
5
6
|
t.string :version, limit: 255, null: false, comment: 'ハッシュ値'
|
6
7
|
t.datetime :created_at, null: false, comment: '作成日時'
|
7
8
|
t.datetime :updated_at, null: false, comment: '更新日時'
|
8
9
|
end
|
9
10
|
|
10
|
-
add_index :master_data_statuses, %i[name], name: 'idx_master_data_statuses_1', unique: true
|
11
|
-
add_index :master_data_statuses, %i[name version], name: 'idx_master_data_statuses_2'
|
11
|
+
add_index :master_data_statuses, %i[spec_name name], name: 'idx_master_data_statuses_1', unique: true
|
12
|
+
add_index :master_data_statuses, %i[spec_name name version], name: 'idx_master_data_statuses_2'
|
12
13
|
end
|
13
14
|
|
14
15
|
def self.down
|
@@ -5,21 +5,22 @@ module MasterDataTool
|
|
5
5
|
include ActiveSupport::Configurable
|
6
6
|
|
7
7
|
config_accessor :master_data_dir
|
8
|
-
config_accessor :
|
9
|
-
config_accessor :dump_ignore_columns
|
10
|
-
config_accessor :default_import_options
|
11
|
-
config_accessor :logger
|
12
|
-
config_accessor :preload_associations
|
13
|
-
config_accessor :eager_load_associations
|
8
|
+
config_accessor :spec_configs
|
14
9
|
|
15
10
|
def initialize
|
16
11
|
self.master_data_dir = nil
|
17
|
-
self.
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
12
|
+
self.spec_configs = []
|
13
|
+
end
|
14
|
+
|
15
|
+
def spec_config(spec_name)
|
16
|
+
spec_configs.detect { |c| c.spec_name == spec_name }
|
17
|
+
end
|
18
|
+
|
19
|
+
def csv_dir_for(spec_name, override_identifier = nil)
|
20
|
+
path = MasterDataTool.config.master_data_dir
|
21
|
+
path = path.join(spec_name.to_s) if spec_name.present?
|
22
|
+
path = path.join(override_identifier.to_s) if override_identifier.present?
|
23
|
+
path
|
23
24
|
end
|
24
25
|
end
|
25
26
|
end
|
@@ -8,23 +8,25 @@ module MasterDataTool
|
|
8
8
|
DEFAULT_IGNORE_TABLES = %w[ar_internal_metadata schema_migrations master_data_statuses]
|
9
9
|
DEFAULT_IGNORE_COLUMNS = %w[created_at updated_at]
|
10
10
|
|
11
|
-
def initialize(ignore_empty_table: true, ignore_tables: [], ignore_column_names: [], only_tables: [], verbose: false)
|
11
|
+
def initialize(spec_config:, ignore_empty_table: true, ignore_tables: [], ignore_column_names: [], only_tables: [], verbose: false)
|
12
|
+
@spec_config = spec_config
|
12
13
|
@ignore_empty_table = ignore_empty_table
|
13
|
-
@ignore_tables = DEFAULT_IGNORE_TABLES + Array(
|
14
|
-
@ignore_column_names = DEFAULT_IGNORE_COLUMNS + Array(
|
14
|
+
@ignore_tables = DEFAULT_IGNORE_TABLES + Array(spec_config.dump_ignore_tables) + ignore_tables
|
15
|
+
@ignore_column_names = DEFAULT_IGNORE_COLUMNS + Array(spec_config.dump_ignore_columns) + ignore_column_names
|
15
16
|
@only_tables = Array(only_tables)
|
17
|
+
@verbose = verbose
|
16
18
|
end
|
17
19
|
|
18
20
|
def execute
|
19
21
|
[].tap do |errors|
|
20
|
-
|
21
|
-
if
|
22
|
+
spec_config.application_record_class.connection.tables.each do |table|
|
23
|
+
if ignore_tables.include?(table)
|
22
24
|
print_message "[ignore] #{table}"
|
23
25
|
|
24
26
|
next
|
25
27
|
end
|
26
28
|
|
27
|
-
if
|
29
|
+
if only_tables.any? && !only_tables.include?(table)
|
28
30
|
print_message "[skip] #{table}"
|
29
31
|
next
|
30
32
|
end
|
@@ -38,8 +40,10 @@ module MasterDataTool
|
|
38
40
|
|
39
41
|
private
|
40
42
|
|
43
|
+
attr_reader :spec_config, :ignore_empty_table, :ignore_tables, :ignore_column_names, :only_tables, :verbose
|
44
|
+
|
41
45
|
def print_message(message)
|
42
|
-
return unless
|
46
|
+
return unless verbose
|
43
47
|
|
44
48
|
puts message
|
45
49
|
end
|
@@ -52,9 +56,11 @@ module MasterDataTool
|
|
52
56
|
return
|
53
57
|
end
|
54
58
|
|
55
|
-
csv_path =
|
59
|
+
csv_path = MasterDataTool.config.csv_dir_for(spec_config.spec_name).join("#{table}.csv")
|
60
|
+
FileUtils.mkdir_p(csv_path.dirname)
|
61
|
+
|
56
62
|
CSV.open(csv_path, 'w', force_quotes: true) do |csv|
|
57
|
-
headers = model_klass.column_names -
|
63
|
+
headers = model_klass.column_names - ignore_column_names
|
58
64
|
|
59
65
|
csv << headers
|
60
66
|
|
@@ -70,7 +76,7 @@ module MasterDataTool
|
|
70
76
|
end
|
71
77
|
|
72
78
|
def ignore?(model_klass)
|
73
|
-
return false unless
|
79
|
+
return false unless ignore_empty_table
|
74
80
|
|
75
81
|
model_klass.count < 1
|
76
82
|
end
|
@@ -3,7 +3,8 @@
|
|
3
3
|
module MasterDataTool
|
4
4
|
module Import
|
5
5
|
class Executor
|
6
|
-
def initialize(
|
6
|
+
def initialize(spec_config:,
|
7
|
+
dry_run: true,
|
7
8
|
verify: true,
|
8
9
|
only_import_tables: [],
|
9
10
|
except_import_tables: [],
|
@@ -13,8 +14,9 @@ module MasterDataTool
|
|
13
14
|
silent: false,
|
14
15
|
delete_all_ignore_foreign_key: false,
|
15
16
|
override_identifier: nil,
|
16
|
-
report_printer:
|
17
|
+
report_printer: nil)
|
17
18
|
|
19
|
+
@spec_config = spec_config
|
18
20
|
@dry_run = dry_run
|
19
21
|
@verify = verify
|
20
22
|
@only_import_tables = Array(only_import_tables)
|
@@ -25,27 +27,29 @@ module MasterDataTool
|
|
25
27
|
@silent = silent
|
26
28
|
@delete_all_ignore_foreign_key = delete_all_ignore_foreign_key
|
27
29
|
@override_identifier = override_identifier
|
28
|
-
@report_printer = report_printer
|
30
|
+
@report_printer = report_printer || MasterDataTool::Report::DefaultPrinter.new(spec_config)
|
29
31
|
@report_printer.silent = silent
|
30
32
|
@master_data_statuses = []
|
31
33
|
end
|
32
34
|
|
33
35
|
def execute
|
34
|
-
|
35
|
-
|
36
|
-
|
36
|
+
spec_config.application_record_class.transaction do
|
37
|
+
MasterDataTool::MasterDataStatus.transaction do
|
38
|
+
print_execute_options
|
39
|
+
load_master_data_statuses
|
37
40
|
|
38
|
-
|
41
|
+
master_data_collection = build_master_data_collection
|
39
42
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
+
import_all!(master_data_collection)
|
44
|
+
verify_all!(master_data_collection) if verify
|
45
|
+
save_master_data_statuses!(master_data_collection)
|
43
46
|
|
44
|
-
|
47
|
+
print_affected_tables(master_data_collection)
|
45
48
|
|
46
|
-
|
49
|
+
raise DryRunError if dry_run
|
47
50
|
|
48
|
-
|
51
|
+
master_data_collection
|
52
|
+
end
|
49
53
|
end
|
50
54
|
rescue DryRunError
|
51
55
|
puts "[DryRun] end"
|
@@ -53,10 +57,11 @@ module MasterDataTool
|
|
53
57
|
|
54
58
|
private
|
55
59
|
|
56
|
-
attr_reader :master_data_statuses
|
60
|
+
attr_reader :master_data_statuses, :spec_config, :dry_run, :verify, :only_import_tables, :except_import_tables, :only_verify_tables,
|
61
|
+
:except_verify_tables, :skip_no_change, :silent, :delete_all_ignore_foreign_key, :override_identifier, :report_printer
|
57
62
|
|
58
63
|
def print_execute_options
|
59
|
-
return if
|
64
|
+
return if silent
|
60
65
|
|
61
66
|
puts "==== execute ===="
|
62
67
|
instance_variables.each do |k|
|
@@ -67,9 +72,9 @@ module MasterDataTool
|
|
67
72
|
|
68
73
|
def build_master_data_collection
|
69
74
|
MasterDataCollection.new.tap do |collection|
|
70
|
-
MasterDataTool::MasterDataFileCollection.new(override_identifier:
|
75
|
+
MasterDataTool::MasterDataFileCollection.new(spec_config.spec_name, override_identifier: override_identifier).each do |master_data_file|
|
71
76
|
load_skip = load_skip_table?(master_data_file)
|
72
|
-
master_data = MasterData.build(master_data_file, load: !load_skip)
|
77
|
+
master_data = MasterData.build(spec_config, master_data_file, load: !load_skip)
|
73
78
|
collection.append(master_data)
|
74
79
|
end
|
75
80
|
end
|
@@ -80,8 +85,8 @@ module MasterDataTool
|
|
80
85
|
next unless master_data.loaded?
|
81
86
|
next if import_skip_table?(master_data.table_name)
|
82
87
|
|
83
|
-
report = master_data.import!(dry_run:
|
84
|
-
report.print(
|
88
|
+
report = master_data.import!(dry_run: dry_run, delete_all_ignore_foreign_key: delete_all_ignore_foreign_key)
|
89
|
+
report.print(report_printer)
|
85
90
|
end
|
86
91
|
end
|
87
92
|
|
@@ -89,8 +94,8 @@ module MasterDataTool
|
|
89
94
|
master_data_collection.each do |master_data|
|
90
95
|
next if verify_skip_table?(master_data.table_name)
|
91
96
|
|
92
|
-
report = master_data.verify!(ignore_fail:
|
93
|
-
report.print(
|
97
|
+
report = master_data.verify!(ignore_fail: dry_run)
|
98
|
+
report.print(report_printer)
|
94
99
|
end
|
95
100
|
end
|
96
101
|
|
@@ -99,10 +104,10 @@ module MasterDataTool
|
|
99
104
|
master_data_collection.each do |master_data|
|
100
105
|
next unless master_data.loaded?
|
101
106
|
|
102
|
-
records << MasterDataTool::MasterDataStatus.build(master_data.master_data_file)
|
107
|
+
records << MasterDataTool::MasterDataStatus.build(spec_config.spec_name, master_data.master_data_file)
|
103
108
|
end
|
104
109
|
|
105
|
-
MasterDataTool::MasterDataStatus.import_records!(records, dry_run:
|
110
|
+
MasterDataTool::MasterDataStatus.import_records!(records, dry_run: dry_run)
|
106
111
|
end
|
107
112
|
|
108
113
|
def print_affected_tables(master_data_collection)
|
@@ -111,13 +116,13 @@ module MasterDataTool
|
|
111
116
|
next unless master_data.affected?
|
112
117
|
|
113
118
|
report = master_data.print_affected_table
|
114
|
-
report&.print(
|
119
|
+
report&.print(report_printer)
|
115
120
|
end
|
116
121
|
end
|
117
122
|
|
118
123
|
def load_skip_table?(master_data_file)
|
119
124
|
return true if import_skip_table?(master_data_file.table_name)
|
120
|
-
return false unless
|
125
|
+
return false unless skip_no_change
|
121
126
|
|
122
127
|
master_data_status = master_data_statuses.dig(master_data_file.table_name)
|
123
128
|
return false unless master_data_status
|
@@ -126,11 +131,11 @@ module MasterDataTool
|
|
126
131
|
end
|
127
132
|
|
128
133
|
def import_skip_table?(table_name)
|
129
|
-
need_skip_table?(table_name,
|
134
|
+
need_skip_table?(table_name, only_import_tables, except_import_tables)
|
130
135
|
end
|
131
136
|
|
132
137
|
def verify_skip_table?(table_name)
|
133
|
-
need_skip_table?(table_name,
|
138
|
+
need_skip_table?(table_name, only_verify_tables, except_verify_tables)
|
134
139
|
end
|
135
140
|
|
136
141
|
# 1. onlyを指定した時点でそのリストに含まれるものだけになるべき
|
@@ -149,14 +154,14 @@ module MasterDataTool
|
|
149
154
|
end
|
150
155
|
|
151
156
|
def extract_master_data_csv_paths
|
152
|
-
pattern =
|
157
|
+
pattern = MasterDataTool.config.csv_dir_for(spec_config.spec_name).join('*.csv').to_s
|
153
158
|
Pathname.glob(pattern).select(&:file?)
|
154
159
|
end
|
155
160
|
|
156
161
|
def overridden_master_data_csv_paths
|
157
|
-
return [] unless
|
162
|
+
return [] unless override_identifier
|
158
163
|
|
159
|
-
pattern =
|
164
|
+
pattern = MasterDataTool.config.csv_dir_for(spec_config.spec_name, override_identifier).join('*.csv').to_s
|
160
165
|
Pathname.glob(pattern).select(&:file?)
|
161
166
|
end
|
162
167
|
|
@@ -2,11 +2,12 @@
|
|
2
2
|
|
3
3
|
module MasterDataTool
|
4
4
|
class MasterData
|
5
|
-
attr_reader :master_data_file, :model_klass, :columns, :new_records, :updated_records, :no_change_records, :deleted_records
|
6
|
-
|
5
|
+
attr_reader :master_data_file, :model_klass, :columns, :new_records, :updated_records, :no_change_records, :deleted_records,
|
6
|
+
:before_count, :after_count, :spec_config
|
7
7
|
|
8
8
|
# @param [MasterDataTool::MasterDataFile] master_data_file
|
9
|
-
def initialize(master_data_file, model_klass)
|
9
|
+
def initialize(spec_config, master_data_file, model_klass)
|
10
|
+
@spec_config = spec_config
|
10
11
|
@master_data_file = master_data_file
|
11
12
|
@model_klass = model_klass
|
12
13
|
|
@@ -20,9 +21,9 @@ module MasterDataTool
|
|
20
21
|
end
|
21
22
|
|
22
23
|
class << self
|
23
|
-
def build(master_data_file, load: false)
|
24
|
+
def build(spec_config, master_data_file, load: false)
|
24
25
|
model_klass = Object.const_get(master_data_file.table_name.classify)
|
25
|
-
new(master_data_file, model_klass).tap do |record|
|
26
|
+
new(spec_config, master_data_file, model_klass).tap do |record|
|
26
27
|
record.load if load
|
27
28
|
end
|
28
29
|
end
|
@@ -163,11 +164,11 @@ module MasterDataTool
|
|
163
164
|
private
|
164
165
|
|
165
166
|
def preload_associations
|
166
|
-
@preload_associations ||=
|
167
|
+
@preload_associations ||= spec_config.preload_associations.dig(@model_klass.to_s.to_sym)
|
167
168
|
end
|
168
169
|
|
169
170
|
def eager_load_associations
|
170
|
-
@eager_load_associations ||=
|
171
|
+
@eager_load_associations ||= spec_config.eager_load_associations.dig(@model_klass.to_s.to_sym)
|
171
172
|
end
|
172
173
|
|
173
174
|
def build_records_from_csv(csv, old_records_by_id)
|
@@ -186,11 +187,11 @@ module MasterDataTool
|
|
186
187
|
end
|
187
188
|
|
188
189
|
def enable_foreign_key_checks
|
189
|
-
|
190
|
+
spec_config.application_record_class.connection.execute('SET FOREIGN_KEY_CHECKS = 1')
|
190
191
|
end
|
191
192
|
|
192
193
|
def disable_foreign_key_checks
|
193
|
-
|
194
|
+
spec_config.application_record_class.connection.execute('SET FOREIGN_KEY_CHECKS = 0')
|
194
195
|
end
|
195
196
|
end
|
196
197
|
end
|
@@ -1,9 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
module MasterDataTool
|
3
3
|
class MasterDataFile
|
4
|
-
attr_reader :table_name, :path, :override_identifier
|
4
|
+
attr_reader :spec_name, :table_name, :path, :override_identifier
|
5
5
|
|
6
|
-
def initialize(table_name, path, override_identifier)
|
6
|
+
def initialize(spec_name, table_name, path, override_identifier)
|
7
|
+
@spec_name = spec_name
|
7
8
|
@table_name = table_name
|
8
9
|
@path = path
|
9
10
|
@override_identifier = override_identifier
|
@@ -11,9 +12,9 @@ module MasterDataTool
|
|
11
12
|
end
|
12
13
|
|
13
14
|
class << self
|
14
|
-
def build(path, override_identifier)
|
15
|
-
table_name = MasterDataTool.resolve_table_name(path, override_identifier)
|
16
|
-
new(table_name, path, override_identifier)
|
15
|
+
def build(spec_name, path, override_identifier)
|
16
|
+
table_name = MasterDataTool.resolve_table_name(spec_name, path, override_identifier)
|
17
|
+
new(spec_name, table_name, path, override_identifier)
|
17
18
|
end
|
18
19
|
end
|
19
20
|
|
@@ -29,7 +30,7 @@ module MasterDataTool
|
|
29
30
|
alias eql? ==
|
30
31
|
|
31
32
|
def hash
|
32
|
-
[@table_name, @path, @override_identifier].join.hash
|
33
|
+
[@spec_name, @table_name, @path, @override_identifier].join.hash
|
33
34
|
end
|
34
35
|
end
|
35
36
|
end
|
@@ -2,9 +2,11 @@
|
|
2
2
|
|
3
3
|
module MasterDataTool
|
4
4
|
class MasterDataFileCollection
|
5
|
-
def initialize(override_identifier: nil)
|
5
|
+
def initialize(spec_name, override_identifier: nil)
|
6
|
+
@spec_name = spec_name
|
6
7
|
@override_identifier = override_identifier
|
7
8
|
@collection = build
|
9
|
+
|
8
10
|
freeze
|
9
11
|
end
|
10
12
|
|
@@ -33,18 +35,18 @@ module MasterDataTool
|
|
33
35
|
end
|
34
36
|
|
35
37
|
def extract_master_data_csv_paths
|
36
|
-
pattern =
|
38
|
+
pattern = MasterDataTool.config.csv_dir_for(@spec_name).join('*.csv').to_s
|
37
39
|
Pathname.glob(pattern).select(&:file?).map do |path|
|
38
|
-
MasterDataFile.build(path, nil)
|
40
|
+
MasterDataFile.build(@spec_name, path, nil)
|
39
41
|
end
|
40
42
|
end
|
41
43
|
|
42
44
|
def overridden_master_data_csv_paths
|
43
45
|
return [] if @override_identifier.blank?
|
44
46
|
|
45
|
-
pattern =
|
47
|
+
pattern = MasterDataTool.config.csv_dir_for(@spec_name, @override_identifier).join('*.csv').to_s
|
46
48
|
Pathname.glob(pattern).select(&:file?).map do |path|
|
47
|
-
MasterDataFile.build(path, @override_identifier)
|
49
|
+
MasterDataFile.build(@spec_name, path, @override_identifier)
|
48
50
|
end
|
49
51
|
end
|
50
52
|
end
|
@@ -25,9 +25,9 @@ module MasterDataTool
|
|
25
25
|
all.index_by(&:name)
|
26
26
|
end
|
27
27
|
|
28
|
-
def build(master_data_file)
|
28
|
+
def build(spec_name, master_data_file)
|
29
29
|
version = decide_version(master_data_file.path)
|
30
|
-
new(name: MasterDataTool.resolve_table_name(master_data_file.path, master_data_file.override_identifier), version: version)
|
30
|
+
new(spec_name: spec_name, name: MasterDataTool.resolve_table_name(spec_name, master_data_file.path, master_data_file.override_identifier), version: version)
|
31
31
|
end
|
32
32
|
|
33
33
|
def import_records!(records, dry_run: true)
|
@@ -27,12 +27,12 @@ module MasterDataTool
|
|
27
27
|
label = :count_report
|
28
28
|
{}.tap do |report|
|
29
29
|
report[label] = []
|
30
|
-
report[label] << {operation: :import, label: :count, table_name:
|
31
|
-
report[label] << {operation: :import, label: :affected, table_name:
|
32
|
-
report[label] << {operation: :import, label: :new_count, table_name:
|
33
|
-
report[label] << {operation: :import, label: :updated_count, table_name:
|
34
|
-
report[label] << {operation: :import, label: :no_change_count, table_name:
|
35
|
-
report[label] << {operation: :import, label: :deleted_count, table_name:
|
30
|
+
report[label] << {operation: :import, label: :count, table_name: master_data.table_name, before: master_data.before_count, after: master_data.after_count}
|
31
|
+
report[label] << {operation: :import, label: :affected, table_name: master_data.table_name, affected: master_data.affected?}
|
32
|
+
report[label] << {operation: :import, label: :new_count, table_name: master_data.table_name, count: master_data.new_records.count}
|
33
|
+
report[label] << {operation: :import, label: :updated_count, table_name: master_data.table_name, count: master_data.updated_records.count}
|
34
|
+
report[label] << {operation: :import, label: :no_change_count, table_name: master_data.table_name, count: master_data.no_change_records.count}
|
35
|
+
report[label] << {operation: :import, label: :deleted_count, table_name: master_data.table_name, count: master_data.deleted_records.count}
|
36
36
|
end
|
37
37
|
end
|
38
38
|
|
@@ -40,8 +40,8 @@ module MasterDataTool
|
|
40
40
|
label = :new_records_report
|
41
41
|
{}.tap do |report|
|
42
42
|
report[label] = []
|
43
|
-
|
44
|
-
report[label] << {operation: :import, label: :detail, table_name:
|
43
|
+
master_data.new_records.each do |record|
|
44
|
+
report[label] << {operation: :import, label: :detail, table_name: master_data.table_name, status: :new, id: record.id}
|
45
45
|
end
|
46
46
|
end
|
47
47
|
end
|
@@ -50,8 +50,8 @@ module MasterDataTool
|
|
50
50
|
label = :updated_records_report
|
51
51
|
{}.tap do |report|
|
52
52
|
report[label] = []
|
53
|
-
|
54
|
-
report[label] << {operation: :import, label: :detail, table_name:
|
53
|
+
master_data.updated_records.each do |record|
|
54
|
+
report[label] << {operation: :import, label: :detail, table_name: master_data.table_name, status: :updated, id: record.id, detail: record.changes_to_save}
|
55
55
|
end
|
56
56
|
end
|
57
57
|
end
|
@@ -60,8 +60,8 @@ module MasterDataTool
|
|
60
60
|
label = :no_change_records_report
|
61
61
|
{}.tap do |report|
|
62
62
|
report[label] = []
|
63
|
-
|
64
|
-
report[label] << {operation: :import, label: :detail, table_name:
|
63
|
+
master_data.no_change_records.each do |record|
|
64
|
+
report[label] << {operation: :import, label: :detail, table_name: master_data.table_name, status: :no_change, id: record.id}
|
65
65
|
end
|
66
66
|
end
|
67
67
|
end
|
@@ -70,8 +70,8 @@ module MasterDataTool
|
|
70
70
|
label = :deleted_records_report
|
71
71
|
{}.tap do |report|
|
72
72
|
report[label] = []
|
73
|
-
|
74
|
-
report[label] << {operation: :import, label: :detail, table_name:
|
73
|
+
master_data.deleted_records.each do |record|
|
74
|
+
report[label] << {operation: :import, label: :detail, table_name: master_data.table_name, status: :deleted, id: record.id}
|
75
75
|
end
|
76
76
|
end
|
77
77
|
end
|
@@ -6,7 +6,7 @@ module MasterDataTool
|
|
6
6
|
include Core
|
7
7
|
|
8
8
|
def print(printer)
|
9
|
-
printer.print(convert_to_ltsv({operation: :affected_table, table_name:
|
9
|
+
printer.print(convert_to_ltsv({operation: :affected_table, table_name: master_data.table_name}))
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
@@ -3,14 +3,16 @@
|
|
3
3
|
module MasterDataTool
|
4
4
|
module Report
|
5
5
|
module Printer
|
6
|
+
attr_reader :spec_config
|
6
7
|
attr_accessor :silent
|
7
8
|
|
8
|
-
def initialize(silent: false)
|
9
|
+
def initialize(spec_config, silent: false)
|
10
|
+
@spec_config = spec_config
|
9
11
|
@silent = silent
|
10
12
|
end
|
11
13
|
|
12
14
|
def print(message)
|
13
|
-
return if
|
15
|
+
return if silent
|
14
16
|
|
15
17
|
raise NotImplementedError
|
16
18
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module MasterDataTool
|
2
|
+
class SpecConfig
|
3
|
+
attr_reader :spec_name, :application_record_class, :dump_ignore_tables, :dump_ignore_columns,
|
4
|
+
:default_import_options, :logger, :preload_associations, :eager_load_associations
|
5
|
+
|
6
|
+
def initialize(spec_name:, application_record_class:, dump_ignore_tables: [], dump_ignore_columns: [],
|
7
|
+
default_import_options: {}, logger: Logger.new(nil), preload_associations: {}, eager_load_associations: {})
|
8
|
+
|
9
|
+
@spec_name = spec_name.presence || ''
|
10
|
+
@application_record_class = application_record_class
|
11
|
+
@dump_ignore_tables = dump_ignore_tables
|
12
|
+
@dump_ignore_columns = dump_ignore_columns
|
13
|
+
@default_import_options = default_import_options
|
14
|
+
@logger = logger
|
15
|
+
@preload_associations = preload_associations # key: Class, value: associations
|
16
|
+
@eager_load_associations = eager_load_associations # key: Class, value: associations
|
17
|
+
|
18
|
+
freeze
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/master_data_tool.rb
CHANGED
@@ -4,6 +4,7 @@ require 'csv'
|
|
4
4
|
require 'socket'
|
5
5
|
require_relative "master_data_tool/version"
|
6
6
|
require_relative "master_data_tool/config"
|
7
|
+
require_relative "master_data_tool/spec_config"
|
7
8
|
require_relative "master_data_tool/master_data_status"
|
8
9
|
require_relative "master_data_tool/master_data_file"
|
9
10
|
require_relative "master_data_tool/master_data_file_collection"
|
@@ -32,10 +33,9 @@ module MasterDataTool
|
|
32
33
|
yield config
|
33
34
|
end
|
34
35
|
|
35
|
-
def resolve_table_name(csv_path, override_identifier)
|
36
|
+
def resolve_table_name(spec_name, csv_path, override_identifier)
|
36
37
|
# 0001_table_nameのように投入順序を制御可能にする
|
37
|
-
relative_path = config.
|
38
|
-
relative_path = "#{relative_path}/#{override_identifier}" if override_identifier.present?
|
38
|
+
relative_path = MasterDataTool.config.csv_dir_for(spec_name, override_identifier)
|
39
39
|
csv_path.relative_path_from(relative_path).to_s.gsub(/^\d+_/, '').delete_suffix('.csv')
|
40
40
|
end
|
41
41
|
end
|
data/scripts/drop_db.sh
CHANGED
data/sig/master_data_tool.rbs
CHANGED
@@ -7,7 +7,7 @@ module MasterDataTool
|
|
7
7
|
|
8
8
|
def self.config: -> Config
|
9
9
|
def self.configure: -> untyped
|
10
|
-
def self.resolve_table_name: (Pathname csv_path, String? override_identifier) -> String
|
10
|
+
def self.resolve_table_name: (String spec_name, Pathname csv_path, String? override_identifier) -> String
|
11
11
|
|
12
12
|
class Config
|
13
13
|
def initialize: -> void
|
@@ -16,18 +16,19 @@ module MasterDataTool
|
|
16
16
|
class MasterDataStatus
|
17
17
|
def will_change?: (MasterDataFile master_data_file) -> bool
|
18
18
|
def self.fetch_all: -> Hash[String,MasterDataStatus]
|
19
|
-
def self.build: (MasterDataFile master_data_file) -> MasterDataStatus
|
19
|
+
def self.build: (String spec_name, MasterDataFile master_data_file) -> MasterDataStatus
|
20
20
|
def self.import_records!: (Array[MasterDataStatus] records, dry_run: bool) -> Array[MasterDataStatus]
|
21
21
|
def self.master_data_will_change?: (MasterDataFile master_data_file) -> bool
|
22
22
|
def self.decide_version: (Pathname csv_path) -> String
|
23
23
|
end
|
24
24
|
|
25
25
|
class MasterDataFile
|
26
|
+
attr_reader spec_name: String
|
26
27
|
attr_reader table_name: String
|
27
28
|
attr_reader path: Pathname
|
28
29
|
attr_reader override_identifier: String?
|
29
|
-
def initialize: (String table_name, Pathname path, String? override_identifier) -> void
|
30
|
-
def self.build: (Pathname path, String? override_identifier) -> MasterDataFile
|
30
|
+
def initialize: (String spec_name, String table_name, Pathname path, String? override_identifier) -> void
|
31
|
+
def self.build: (String spec_name, Pathname path, String? override_identifier) -> MasterDataFile
|
31
32
|
def basename: -> Pathname
|
32
33
|
def ==: (untyped other) -> bool
|
33
34
|
alias eql? ==
|
@@ -35,10 +36,11 @@ module MasterDataTool
|
|
35
36
|
end
|
36
37
|
|
37
38
|
class MasterDataFileCollection
|
39
|
+
@spec_name: String
|
38
40
|
@override_identifier: String?
|
39
41
|
@collection: Array[MasterDataFile]
|
40
42
|
|
41
|
-
def initialize: (override_identifier: String?) -> void
|
43
|
+
def initialize: (String spec_name, override_identifier: String?) -> void
|
42
44
|
def each: ?{ -> Array[MasterDataFile] } -> Enumerator[bot, untyped]
|
43
45
|
def to_a: -> Array[MasterDataFile]
|
44
46
|
|
@@ -48,12 +50,27 @@ module MasterDataTool
|
|
48
50
|
def overridden_master_data_csv_paths: -> Array[MasterDataFile]
|
49
51
|
end
|
50
52
|
|
53
|
+
class SpecConfig
|
54
|
+
attr_reader spec_name: String
|
55
|
+
attr_reader application_record_class: Class
|
56
|
+
attr_reader dump_ignore_tables: Array[String]
|
57
|
+
attr_reader dump_ignore_columns: Array[String]
|
58
|
+
attr_reader default_import_options: Hash[String,Class]
|
59
|
+
attr_reader logger: Logger
|
60
|
+
attr_reader preload_associations: Hash[Class, Array[Symbol]]
|
61
|
+
attr_reader eager_load_associations: Hash[Class, Array[Symbol]]
|
62
|
+
|
63
|
+
def initialize: (spec_name: String, application_record_class: Class, dump_ignore_tables: Array[String], dump_ignore_columns: Array[String],
|
64
|
+
default_import_options: Hash[String,Class], logger: Logger, preload_associations: Hash[Class, Array[Symbol]], eager_load_associations: Hash[Class, Array[Symbol]]) -> void
|
65
|
+
end
|
66
|
+
|
51
67
|
class MasterData
|
52
68
|
@loaded: bool
|
53
69
|
@affected: bool
|
54
70
|
@preload_associations: Array[untyped]
|
55
71
|
@eager_load_associations: Array[untyped]
|
56
72
|
|
73
|
+
attr_reader spec_config: SpecConfig
|
57
74
|
attr_reader master_data_file: MasterDataFile
|
58
75
|
attr_reader model_klass: untyped
|
59
76
|
attr_reader columns: Array[String]
|
@@ -69,8 +86,8 @@ module MasterDataTool
|
|
69
86
|
def before_count: -> Integer
|
70
87
|
attr_reader after_count: Integer
|
71
88
|
def after_count: -> Integer
|
72
|
-
def initialize: (MasterDataFile master_data_file, untyped model_klass) -> void
|
73
|
-
def self.build: (MasterDataFile master_data_file, ?load: bool) -> MasterData
|
89
|
+
def initialize: (SpecConfig spec_config, MasterDataFile master_data_file, untyped model_klass) -> void
|
90
|
+
def self.build: (SpecConfig spec_config, MasterDataFile master_data_file, ?load: bool) -> MasterData
|
74
91
|
def basename: -> Pathname
|
75
92
|
def load: -> true
|
76
93
|
def import_records: -> Array[untyped]
|
@@ -101,8 +118,9 @@ module MasterDataTool
|
|
101
118
|
|
102
119
|
module Report
|
103
120
|
module Printer
|
121
|
+
attr_reader spec_config: SpecConfig
|
104
122
|
attr_accessor silent: false
|
105
|
-
def initialize: (?silent: false) -> void
|
123
|
+
def initialize: (SpecConfig spec_config, ?silent: false) -> void
|
106
124
|
def print: (String message) -> nil
|
107
125
|
end
|
108
126
|
|
@@ -157,15 +175,16 @@ module MasterDataTool
|
|
157
175
|
|
158
176
|
module Dump
|
159
177
|
class Executor
|
160
|
-
DEFAULT_IGNORE_TABLES: [String
|
161
|
-
DEFAULT_IGNORE_COLUMNS: [String
|
178
|
+
DEFAULT_IGNORE_TABLES: Array[String]
|
179
|
+
DEFAULT_IGNORE_COLUMNS: Array[String]
|
180
|
+
@spec_config: SpecConfig
|
162
181
|
@ignore_empty_table: bool
|
163
182
|
@ignore_tables: Array[String]
|
164
183
|
@ignore_column_names: Array[String]
|
165
184
|
@only_tables: Array[String]
|
166
185
|
@verbose: bool
|
167
186
|
|
168
|
-
def initialize: (ignore_empty_table: bool, ignore_tables: Array[String], ignore_column_names: Array[String], only_tables: Array[String], verbose: bool) -> void
|
187
|
+
def initialize: (spec_config: SpecConfig, ignore_empty_table: bool, ignore_tables: Array[String], ignore_column_names: Array[String], only_tables: Array[String], verbose: bool) -> void
|
169
188
|
def execute: -> Array[untyped]
|
170
189
|
|
171
190
|
private
|
@@ -182,6 +201,7 @@ module MasterDataTool
|
|
182
201
|
|
183
202
|
module Import
|
184
203
|
class Executor
|
204
|
+
@spec_config: SpecConfig
|
185
205
|
@dry_run: bool
|
186
206
|
@verify: bool
|
187
207
|
@only_import_tables: Array[String]
|
@@ -195,7 +215,7 @@ module MasterDataTool
|
|
195
215
|
@report_printer: Report::DefaultPrinter
|
196
216
|
@master_data_statuses: [MasterDataStatus]
|
197
217
|
|
198
|
-
def initialize: (dry_run: bool, verify: bool, only_import_tables: Array[String], except_import_tables: Array[String], only_verify_tables: Array[String], except_verify_tables: Array[String], skip_no_change: bool, silent: bool, delete_all_ignore_foreign_key: bool, override_identifier: String?, report_printer: Report::Printer) -> void
|
218
|
+
def initialize: (spec_config: SpecConfig, dry_run: bool, verify: bool, only_import_tables: Array[String], except_import_tables: Array[String], only_verify_tables: Array[String], except_verify_tables: Array[String], skip_no_change: bool, silent: bool, delete_all_ignore_foreign_key: bool, override_identifier: String?, report_printer: Report::Printer) -> void
|
199
219
|
def execute: -> nil
|
200
220
|
|
201
221
|
private
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: master_data_tool
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.21.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takahiro Ooishi
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-11-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -221,6 +221,7 @@ files:
|
|
221
221
|
- lib/master_data_tool/report/print_affected_table_report.rb
|
222
222
|
- lib/master_data_tool/report/printer.rb
|
223
223
|
- lib/master_data_tool/report/verify_report.rb
|
224
|
+
- lib/master_data_tool/spec_config.rb
|
224
225
|
- lib/master_data_tool/version.rb
|
225
226
|
- log/test.log
|
226
227
|
- master_data_tool.gemspec
|