master_data_tool 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +12 -11
- data/exe/master_data_tool +1 -0
- data/lib/master_data_tool/import/executor.rb +18 -10
- data/lib/master_data_tool/import/master_data_file_list.rb +43 -0
- data/lib/master_data_tool/import.rb +1 -0
- data/lib/master_data_tool/master_data.rb +9 -4
- data/lib/master_data_tool/master_data_file.rb +13 -0
- data/lib/master_data_tool/master_data_status.rb +7 -6
- data/lib/master_data_tool/version.rb +1 -1
- data/lib/master_data_tool.rb +5 -2
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '09f03f9ca56ad09dd1f511a53248da3ffe917a0e8be0dba68e3bfbb8f61f74bb'
|
4
|
+
data.tar.gz: 56d9f18f92c96ca4cf2f5bac42edc369d861cbf1cb3ba3e39cb6c8b19eccbdc0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 229dc35b454ca6adafddc65125042649172d203a87554fbe161cade5ad6f17c9e645c40108cb37a937cbf1bfa496b06f0ad6406e904e706a7945b4a9ffe9bb2f
|
7
|
+
data.tar.gz: 285b4f485f23c1215b8029af39bac2808a85050506e02cb92e1662ddaf0ccf634f9a8de4b821663f297d78f6e8a74d61a0920af4e2570ebe19200a0d1ec2f121
|
data/README.md
CHANGED
@@ -35,17 +35,18 @@ Or install it yourself as:
|
|
35
35
|
|
36
36
|
### マスタデータの投入
|
37
37
|
|
38
|
-
| option | default | 内容
|
39
|
-
|----------------------| ---
|
40
|
-
| --dry-run | true | dry-runモードで実行する(データ変更は行わない)
|
41
|
-
| --verify | true | データ投入後に全テーブル・全レコードのバリデーションチェックを行う
|
42
|
-
| --only-import-tables | [] | 指定したテーブルのみデータ投入を行う
|
43
|
-
| --except-import-tables | [] | 指定したテーブルのデータ投入を行わない
|
44
|
-
| --only-verify-tables | [] | 指定したテーブルのみ投入後のバリデーションチェックを行う
|
45
|
-
| --except-verify-tables | [] | 指定したテーブルのバリデーションチェックを行わない
|
46
|
-
| --skip-no-change | true | CSVファイルに更新がないテーブルをスキップする
|
47
|
-
| --silent | false | 結果の出力をやめる
|
48
|
-
| --delete-all-ignore-foreign-key | false | 外部キー制約を無視してレコードを消すかどうか
|
38
|
+
| option | default | 内容 |
|
39
|
+
|----------------------| --- |-----------------------------------------------------------------|
|
40
|
+
| --dry-run | true | dry-runモードで実行する(データ変更は行わない) |
|
41
|
+
| --verify | true | データ投入後に全テーブル・全レコードのバリデーションチェックを行う |
|
42
|
+
| --only-import-tables | [] | 指定したテーブルのみデータ投入を行う |
|
43
|
+
| --except-import-tables | [] | 指定したテーブルのデータ投入を行わない |
|
44
|
+
| --only-verify-tables | [] | 指定したテーブルのみ投入後のバリデーションチェックを行う |
|
45
|
+
| --except-verify-tables | [] | 指定したテーブルのバリデーションチェックを行わない |
|
46
|
+
| --skip-no-change | true | CSVファイルに更新がないテーブルをスキップする |
|
47
|
+
| --silent | false | 結果の出力をやめる |
|
48
|
+
| --delete-all-ignore-foreign-key | false | 外部キー制約を無視してレコードを消すかどうか |
|
49
|
+
| --override_identifier | nil | fixtures/#{override_identifier} のディレクトリにある内容でfixturesを上書きして投入する |
|
49
50
|
|
50
51
|
```bash
|
51
52
|
bundle exec master_data_tool import
|
data/exe/master_data_tool
CHANGED
@@ -17,6 +17,7 @@ module MasterDataTool
|
|
17
17
|
option :except_verify_tables, default: nil, type: :array
|
18
18
|
option :skip_no_change, default: nil, type: :boolean
|
19
19
|
option :silent, default: nil, type: :boolean
|
20
|
+
option :override_identifier, default: nil, type: :string
|
20
21
|
option :delete_all_ignore_foreign_key, default: nil, type: :boolean
|
21
22
|
desc 'import', 'import'
|
22
23
|
def import
|
@@ -12,6 +12,7 @@ module MasterDataTool
|
|
12
12
|
skip_no_change: true,
|
13
13
|
silent: false,
|
14
14
|
delete_all_ignore_foreign_key: false,
|
15
|
+
override_identifier: nil,
|
15
16
|
report_printer: MasterDataTool::Report::DefaultPrinter.new)
|
16
17
|
|
17
18
|
@dry_run = dry_run
|
@@ -23,6 +24,7 @@ module MasterDataTool
|
|
23
24
|
@skip_no_change = skip_no_change
|
24
25
|
@silent = silent
|
25
26
|
@delete_all_ignore_foreign_key = delete_all_ignore_foreign_key
|
27
|
+
@override_identifier = override_identifier
|
26
28
|
@report_printer = report_printer
|
27
29
|
@report_printer.silent = silent
|
28
30
|
end
|
@@ -61,17 +63,16 @@ module MasterDataTool
|
|
61
63
|
|
62
64
|
def build_master_data_list
|
63
65
|
[].tap do |master_data_list|
|
64
|
-
|
65
|
-
|
66
|
-
load_skip = load_skip_table?(table_name, path)
|
66
|
+
MasterDataTool::Import::MasterDataFileList.new(override_identifier: @override_identifier).build.each do |master_data_file|
|
67
|
+
load_skip = load_skip_table?(master_data_file)
|
67
68
|
|
68
|
-
model_klass = Object.const_get(table_name.classify)
|
69
|
-
master_data = MasterData.new(
|
69
|
+
model_klass = Object.const_get(master_data_file.table_name.classify)
|
70
|
+
master_data = MasterData.new(master_data_file, model_klass)
|
70
71
|
master_data.load unless load_skip
|
71
72
|
|
72
73
|
master_data_list << master_data
|
73
74
|
end
|
74
|
-
end.sort_by { |m| m.
|
75
|
+
end.sort_by { |m| m.basename } # 外部キー制約などがある場合には先に入れておかないといけないデータなどがある。なので、プレフィックスを付けて順序を指定して貰う
|
75
76
|
end
|
76
77
|
|
77
78
|
def import_all!(master_data_list)
|
@@ -98,7 +99,7 @@ module MasterDataTool
|
|
98
99
|
master_data_list.each do |master_data|
|
99
100
|
next unless master_data.loaded?
|
100
101
|
|
101
|
-
records << MasterDataTool::MasterDataStatus.build(master_data.
|
102
|
+
records << MasterDataTool::MasterDataStatus.build(master_data.master_data_file)
|
102
103
|
end
|
103
104
|
|
104
105
|
MasterDataTool::MasterDataStatus.import_records!(records, dry_run: @dry_run)
|
@@ -114,11 +115,11 @@ module MasterDataTool
|
|
114
115
|
end
|
115
116
|
end
|
116
117
|
|
117
|
-
def load_skip_table?(
|
118
|
-
return true if import_skip_table?(table_name)
|
118
|
+
def load_skip_table?(master_data_file)
|
119
|
+
return true if import_skip_table?(master_data_file.table_name)
|
119
120
|
return false unless @skip_no_change
|
120
121
|
|
121
|
-
!MasterDataTool::MasterDataStatus.master_data_will_change?(
|
122
|
+
!MasterDataTool::MasterDataStatus.master_data_will_change?(master_data_file)
|
122
123
|
end
|
123
124
|
|
124
125
|
def import_skip_table?(table_name)
|
@@ -148,6 +149,13 @@ module MasterDataTool
|
|
148
149
|
pattern = Pathname.new(MasterDataTool.config.master_data_dir).join('*.csv').to_s
|
149
150
|
Pathname.glob(pattern).select(&:file?)
|
150
151
|
end
|
152
|
+
|
153
|
+
def overridden_master_data_csv_paths
|
154
|
+
return [] unless @override_identifier
|
155
|
+
|
156
|
+
pattern = Pathname.new(MasterDataTool.config.master_data_dir).join(@override_identifier).join('*.csv').to_s
|
157
|
+
Pathname.glob(pattern).select(&:file?)
|
158
|
+
end
|
151
159
|
end
|
152
160
|
end
|
153
161
|
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MasterDataTool
|
4
|
+
module Import
|
5
|
+
class MasterDataFileList
|
6
|
+
Result = Struct.new(:table_name, :path, :override_identifier)
|
7
|
+
|
8
|
+
def initialize(override_identifier: nil)
|
9
|
+
@override_identifier = override_identifier
|
10
|
+
end
|
11
|
+
|
12
|
+
def build
|
13
|
+
files = extract_master_data_csv_paths.presence&.index_by(&:table_name)
|
14
|
+
overridden_files = overridden_master_data_csv_paths.presence&.index_by(&:table_name) || {}
|
15
|
+
|
16
|
+
table_names = (files.keys + overridden_files.keys).uniq
|
17
|
+
table_names.map do |table_name|
|
18
|
+
overridden_files[table_name] || files[table_name]
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def extract_master_data_csv_paths
|
25
|
+
pattern = Pathname.new(MasterDataTool.config.master_data_dir).join('*.csv').to_s
|
26
|
+
Pathname.glob(pattern).select(&:file?).map do |path|
|
27
|
+
table_name = MasterDataTool.resolve_table_name(path, nil)
|
28
|
+
MasterDataTool::MasterDataFile.new(table_name, path, nil)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def overridden_master_data_csv_paths
|
33
|
+
return [] if @override_identifier.blank?
|
34
|
+
|
35
|
+
pattern = Pathname.new(MasterDataTool.config.master_data_dir).join(@override_identifier).join('*.csv').to_s
|
36
|
+
Pathname.glob(pattern).select(&:file?).map do |path|
|
37
|
+
table_name = MasterDataTool.resolve_table_name(path, @override_identifier)
|
38
|
+
MasterDataTool::MasterDataFile.new(table_name, path, @override_identifier)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -2,11 +2,12 @@
|
|
2
2
|
|
3
3
|
module MasterDataTool
|
4
4
|
class MasterData
|
5
|
-
attr_reader :
|
5
|
+
attr_reader :master_data_file, :model_klass, :columns, :new_records, :updated_records, :no_change_records, :deleted_records
|
6
6
|
attr_reader :before_count, :after_count
|
7
7
|
|
8
|
-
|
9
|
-
|
8
|
+
# @param [MasterDataTool::MasterDataFile] master_data_file
|
9
|
+
def initialize(master_data_file, model_klass)
|
10
|
+
@master_data_file = master_data_file
|
10
11
|
@model_klass = model_klass
|
11
12
|
|
12
13
|
@loaded = false
|
@@ -18,8 +19,12 @@ module MasterDataTool
|
|
18
19
|
@deleted_records = []
|
19
20
|
end
|
20
21
|
|
22
|
+
def basename
|
23
|
+
@master_data_file.basename
|
24
|
+
end
|
25
|
+
|
21
26
|
def load
|
22
|
-
csv = CSV.read(@
|
27
|
+
csv = CSV.read(@master_data_file.path, headers: true, skip_blanks: true)
|
23
28
|
old_records_by_id = @model_klass.all.index_by(&:id)
|
24
29
|
|
25
30
|
csv_records_by_id = build_records_from_csv(csv, old_records_by_id)
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module MasterDataTool
|
3
|
+
class MasterDataFile < Struct.new(:table_name, :path, :override_identifier)
|
4
|
+
def initialize(table_name, path, override_identifier)
|
5
|
+
super(table_name, path, override_identifier)
|
6
|
+
freeze
|
7
|
+
end
|
8
|
+
|
9
|
+
def basename
|
10
|
+
self.path.basename
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -15,9 +15,9 @@ module MasterDataTool
|
|
15
15
|
presence: true
|
16
16
|
|
17
17
|
class << self
|
18
|
-
def build(
|
19
|
-
version = decide_version(
|
20
|
-
new(name: MasterDataTool.resolve_table_name(
|
18
|
+
def build(master_data_file)
|
19
|
+
version = decide_version(master_data_file.path)
|
20
|
+
new(name: MasterDataTool.resolve_table_name(master_data_file.path, master_data_file.override_identifier), version: version)
|
21
21
|
end
|
22
22
|
|
23
23
|
def import_records!(records, dry_run: true)
|
@@ -28,9 +28,10 @@ module MasterDataTool
|
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
|
31
|
+
# @param [MasterDataTool::MasterDataFile] master_data_file
|
32
|
+
def master_data_will_change?(master_data_file)
|
33
|
+
new_version = decide_version(master_data_file.path)
|
34
|
+
!where(name: master_data_file.table_name, version: new_version).exists?
|
34
35
|
end
|
35
36
|
|
36
37
|
def decide_version(csv_path)
|
data/lib/master_data_tool.rb
CHANGED
@@ -4,6 +4,7 @@ require 'csv'
|
|
4
4
|
require_relative "master_data_tool/version"
|
5
5
|
require_relative "master_data_tool/config"
|
6
6
|
require_relative "master_data_tool/master_data_status"
|
7
|
+
require_relative "master_data_tool/master_data_file"
|
7
8
|
require_relative "master_data_tool/master_data"
|
8
9
|
require_relative "master_data_tool/report"
|
9
10
|
require_relative "master_data_tool/dump/executor"
|
@@ -24,9 +25,11 @@ module MasterDataTool
|
|
24
25
|
yield config
|
25
26
|
end
|
26
27
|
|
27
|
-
def resolve_table_name(csv_path)
|
28
|
+
def resolve_table_name(csv_path, override_identifier)
|
28
29
|
# 0001_table_nameのように投入順序を制御可能にする
|
29
|
-
|
30
|
+
relative_path = config.master_data_dir
|
31
|
+
relative_path = "#{relative_path}/#{override_identifier}" if override_identifier.present?
|
32
|
+
csv_path.relative_path_from(relative_path).to_s.gsub(/^\d+_/, '').delete_suffix('.csv')
|
30
33
|
end
|
31
34
|
end
|
32
35
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: master_data_tool
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.16.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takahiro Ooishi
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-06-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -194,7 +194,9 @@ files:
|
|
194
194
|
- lib/master_data_tool/dump/executor.rb
|
195
195
|
- lib/master_data_tool/import.rb
|
196
196
|
- lib/master_data_tool/import/executor.rb
|
197
|
+
- lib/master_data_tool/import/master_data_file_list.rb
|
197
198
|
- lib/master_data_tool/master_data.rb
|
199
|
+
- lib/master_data_tool/master_data_file.rb
|
198
200
|
- lib/master_data_tool/master_data_status.rb
|
199
201
|
- lib/master_data_tool/report.rb
|
200
202
|
- lib/master_data_tool/report/core.rb
|