master_data_tool 0.13.0 → 0.16.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d8d4c2c62c923b5e23871ac9e4df2f91c535c36cd8c66a6cb3d3a05ca6c7275d
4
- data.tar.gz: 39c776e2196f8a080d7caac86eb4a689701f7263b7aa2d856a4061bd8893863f
3
+ metadata.gz: '09f03f9ca56ad09dd1f511a53248da3ffe917a0e8be0dba68e3bfbb8f61f74bb'
4
+ data.tar.gz: 56d9f18f92c96ca4cf2f5bac42edc369d861cbf1cb3ba3e39cb6c8b19eccbdc0
5
5
  SHA512:
6
- metadata.gz: 624bf1b07e720e1d062c607c9dd697d5709bbc28d19f6396510a2426a0e423b56791682d57f72b6323e36a34c3b70e22f1f1e40dfa9d687cb8dc2fd071a39116
7
- data.tar.gz: 1bc885ac91429ba21b201ccf7bc275bcba05377004ac0517aec0ec0ceedb3ead0c1bb2324fe29c435a8e06a9e50101e3e7b85636c5b700ec79b4f44528585ae0
6
+ metadata.gz: 229dc35b454ca6adafddc65125042649172d203a87554fbe161cade5ad6f17c9e645c40108cb37a937cbf1bfa496b06f0ad6406e904e706a7945b4a9ffe9bb2f
7
+ data.tar.gz: 285b4f485f23c1215b8029af39bac2808a85050506e02cb92e1662ddaf0ccf634f9a8de4b821663f297d78f6e8a74d61a0920af4e2570ebe19200a0d1ec2f121
data/README.md CHANGED
@@ -35,17 +35,18 @@ Or install it yourself as:
35
35
 
36
36
  ### マスタデータの投入
37
37
 
38
- | option | default | 内容 |
39
- |----------------------| --- |-----------------------------------|
40
- | --dry-run | true | dry-runモードで実行する(データ変更は行わない) |
41
- | --verify | true | データ投入後に全テーブル・全レコードのバリデーションチェックを行う |
42
- | --only-import-tables | [] | 指定したテーブルのみデータ投入を行う |
43
- | --except-import-tables | [] | 指定したテーブルのデータ投入を行わない |
44
- | --only-verify-tables | [] | 指定したテーブルのみ投入後のバリデーションチェックを行う |
45
- | --except-verify-tables | [] | 指定したテーブルのバリデーションチェックを行わない |
46
- | --skip-no-change | true | CSVファイルに更新がないテーブルをスキップする |
47
- | --silent | false | 結果の出力をやめる |
48
- | --delete-all-ignore-foreign-key | false | 外部キー制約を無視してレコードを消すかどうか |
38
+ | option | default | 内容 |
39
+ |----------------------| --- |-----------------------------------------------------------------|
40
+ | --dry-run | true | dry-runモードで実行する(データ変更は行わない) |
41
+ | --verify | true | データ投入後に全テーブル・全レコードのバリデーションチェックを行う |
42
+ | --only-import-tables | [] | 指定したテーブルのみデータ投入を行う |
43
+ | --except-import-tables | [] | 指定したテーブルのデータ投入を行わない |
44
+ | --only-verify-tables | [] | 指定したテーブルのみ投入後のバリデーションチェックを行う |
45
+ | --except-verify-tables | [] | 指定したテーブルのバリデーションチェックを行わない |
46
+ | --skip-no-change | true | CSVファイルに更新がないテーブルをスキップする |
47
+ | --silent | false | 結果の出力をやめる |
48
+ | --delete-all-ignore-foreign-key | false | 外部キー制約を無視してレコードを消すかどうか |
49
+ | --override_identifier | nil | fixtures/#{override_identifier} のディレクトリにある内容でfixturesを上書きして投入する |
49
50
 
50
51
  ```bash
51
52
  bundle exec master_data_tool import
@@ -68,12 +69,13 @@ bundle exec thor master_data_tool import \
68
69
 
69
70
  ### ダンプ
70
71
 
71
- | option | default | 内容 |
72
- |-----------------------|---------|---------------|
73
- | --ignore-empty-table | true | 空のテーブルを無視する |
74
- | --ignore-tables | [] | 指定したテーブルを無視する |
75
- | --ignore-column-names | [] | 指定したカラムを無視する |
76
- | --verbose | false | 詳細表示 |
72
+ | option | default | 内容 |
73
+ |-----------------------|---------|-----------------|
74
+ | --ignore-empty-table | true | 空のテーブルを無視する |
75
+ | --ignore-tables | [] | 指定したテーブルを無視する |
76
+ | --ignore-column-names | [] | 指定したカラムを無視する |
77
+ | --only-tables | nil | 指定したテーブルのみダンプする |
78
+ | --verbose | false | 詳細表示 |
77
79
 
78
80
  ```bash
79
81
  bundle exec master_data_tool dump
@@ -86,6 +88,7 @@ bundle exec master_data_tool dump \
86
88
  --ignore-empty-table=true \
87
89
  --ignore-tables="" \
88
90
  --ignore-column-names="" \
91
+ --only-tables="" \
89
92
  --verbose=false
90
93
  ```
91
94
 
data/exe/master_data_tool CHANGED
@@ -17,6 +17,7 @@ module MasterDataTool
17
17
  option :except_verify_tables, default: nil, type: :array
18
18
  option :skip_no_change, default: nil, type: :boolean
19
19
  option :silent, default: nil, type: :boolean
20
+ option :override_identifier, default: nil, type: :string
20
21
  option :delete_all_ignore_foreign_key, default: nil, type: :boolean
21
22
  desc 'import', 'import'
22
23
  def import
@@ -28,18 +29,21 @@ module MasterDataTool
28
29
  option :ignore_empty_table, default: true, type: :boolean
29
30
  option :ignore_tables, default: [], type: :array
30
31
  option :ignore_column_names, default: [], type: :array
32
+ option :only_tables, default: nil, type: :array
31
33
  option :verbose, default: false, type: :boolean
32
34
  desc 'dump', 'dump'
33
35
  def dump
34
36
  ignore_empty_table = options[:ignore_empty_table]
35
37
  ignore_tables = options[:ignore_tables]
36
38
  ignore_column_names = options[:ignore_column_names]
39
+ only_tables = options[:only_tables]
37
40
  verbose = options[:verbose]
38
41
 
39
42
  executor = MasterDataTool::Dump::Executor.new(
40
43
  ignore_empty_table: ignore_empty_table,
41
44
  ignore_tables: ignore_tables,
42
45
  ignore_column_names: ignore_column_names,
46
+ only_tables: only_tables,
43
47
  verbose: verbose
44
48
  )
45
49
  errors = executor.execute
@@ -8,10 +8,11 @@ module MasterDataTool
8
8
  DEFAULT_IGNORE_TABLES = %w[ar_internal_metadata schema_migrations master_data_statuses]
9
9
  DEFAULT_IGNORE_COLUMNS = %w[created_at updated_at]
10
10
 
11
- def initialize(ignore_empty_table: true, ignore_tables: [], ignore_column_names: [], verbose: false)
11
+ def initialize(ignore_empty_table: true, ignore_tables: [], ignore_column_names: [], only_tables: [], verbose: false)
12
12
  @ignore_empty_table = ignore_empty_table
13
13
  @ignore_tables = DEFAULT_IGNORE_TABLES + Array(MasterDataTool.config.dump_ignore_tables) + ignore_tables
14
14
  @ignore_column_names = DEFAULT_IGNORE_COLUMNS + Array(MasterDataTool.config.dump_ignore_columns) + ignore_column_names
15
+ @only_tables = Array(only_tables)
15
16
  end
16
17
 
17
18
  def execute
@@ -23,6 +24,11 @@ module MasterDataTool
23
24
  next
24
25
  end
25
26
 
27
+ if @only_tables.any? && !@only_tables.include?(table)
28
+ print_message "[skip] #{table}"
29
+ next
30
+ end
31
+
26
32
  dump_to_csv(table)
27
33
  rescue => e
28
34
  errors << Error.new(table, e)
@@ -12,6 +12,7 @@ module MasterDataTool
12
12
  skip_no_change: true,
13
13
  silent: false,
14
14
  delete_all_ignore_foreign_key: false,
15
+ override_identifier: nil,
15
16
  report_printer: MasterDataTool::Report::DefaultPrinter.new)
16
17
 
17
18
  @dry_run = dry_run
@@ -23,6 +24,7 @@ module MasterDataTool
23
24
  @skip_no_change = skip_no_change
24
25
  @silent = silent
25
26
  @delete_all_ignore_foreign_key = delete_all_ignore_foreign_key
27
+ @override_identifier = override_identifier
26
28
  @report_printer = report_printer
27
29
  @report_printer.silent = silent
28
30
  end
@@ -61,17 +63,16 @@ module MasterDataTool
61
63
 
62
64
  def build_master_data_list
63
65
  [].tap do |master_data_list|
64
- extract_master_data_csv_paths.each do |path|
65
- table_name = MasterDataTool.resolve_table_name(path)
66
- load_skip = load_skip_table?(table_name, path)
66
+ MasterDataTool::Import::MasterDataFileList.new(override_identifier: @override_identifier).build.each do |master_data_file|
67
+ load_skip = load_skip_table?(master_data_file)
67
68
 
68
- model_klass = Object.const_get(table_name.classify)
69
- master_data = MasterData.new(path, model_klass)
69
+ model_klass = Object.const_get(master_data_file.table_name.classify)
70
+ master_data = MasterData.new(master_data_file, model_klass)
70
71
  master_data.load unless load_skip
71
72
 
72
73
  master_data_list << master_data
73
74
  end
74
- end.sort_by { |m| m.csv_path } # 外部キー制約などがある場合には先に入れておかないといけないデータなどがある。なので、プレフィックスを付けて順序を指定して貰う
75
+ end.sort_by { |m| m.basename } # 外部キー制約などがある場合には先に入れておかないといけないデータなどがある。なので、プレフィックスを付けて順序を指定して貰う
75
76
  end
76
77
 
77
78
  def import_all!(master_data_list)
@@ -98,7 +99,7 @@ module MasterDataTool
98
99
  master_data_list.each do |master_data|
99
100
  next unless master_data.loaded?
100
101
 
101
- records << MasterDataTool::MasterDataStatus.build(master_data.csv_path)
102
+ records << MasterDataTool::MasterDataStatus.build(master_data.master_data_file)
102
103
  end
103
104
 
104
105
  MasterDataTool::MasterDataStatus.import_records!(records, dry_run: @dry_run)
@@ -114,11 +115,11 @@ module MasterDataTool
114
115
  end
115
116
  end
116
117
 
117
- def load_skip_table?(table_name, csv_path)
118
- return true if import_skip_table?(table_name)
118
+ def load_skip_table?(master_data_file)
119
+ return true if import_skip_table?(master_data_file.table_name)
119
120
  return false unless @skip_no_change
120
121
 
121
- !MasterDataTool::MasterDataStatus.master_data_will_change?(csv_path)
122
+ !MasterDataTool::MasterDataStatus.master_data_will_change?(master_data_file)
122
123
  end
123
124
 
124
125
  def import_skip_table?(table_name)
@@ -148,6 +149,13 @@ module MasterDataTool
148
149
  pattern = Pathname.new(MasterDataTool.config.master_data_dir).join('*.csv').to_s
149
150
  Pathname.glob(pattern).select(&:file?)
150
151
  end
152
+
153
+ def overridden_master_data_csv_paths
154
+ return [] unless @override_identifier
155
+
156
+ pattern = Pathname.new(MasterDataTool.config.master_data_dir).join(@override_identifier).join('*.csv').to_s
157
+ Pathname.glob(pattern).select(&:file?)
158
+ end
151
159
  end
152
160
  end
153
161
  end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MasterDataTool
4
+ module Import
5
+ class MasterDataFileList
6
+ Result = Struct.new(:table_name, :path, :override_identifier)
7
+
8
+ def initialize(override_identifier: nil)
9
+ @override_identifier = override_identifier
10
+ end
11
+
12
+ def build
13
+ files = extract_master_data_csv_paths.presence&.index_by(&:table_name)
14
+ overridden_files = overridden_master_data_csv_paths.presence&.index_by(&:table_name) || {}
15
+
16
+ table_names = (files.keys + overridden_files.keys).uniq
17
+ table_names.map do |table_name|
18
+ overridden_files[table_name] || files[table_name]
19
+ end
20
+ end
21
+
22
+ private
23
+
24
+ def extract_master_data_csv_paths
25
+ pattern = Pathname.new(MasterDataTool.config.master_data_dir).join('*.csv').to_s
26
+ Pathname.glob(pattern).select(&:file?).map do |path|
27
+ table_name = MasterDataTool.resolve_table_name(path, nil)
28
+ MasterDataTool::MasterDataFile.new(table_name, path, nil)
29
+ end
30
+ end
31
+
32
+ def overridden_master_data_csv_paths
33
+ return [] if @override_identifier.blank?
34
+
35
+ pattern = Pathname.new(MasterDataTool.config.master_data_dir).join(@override_identifier).join('*.csv').to_s
36
+ Pathname.glob(pattern).select(&:file?).map do |path|
37
+ table_name = MasterDataTool.resolve_table_name(path, @override_identifier)
38
+ MasterDataTool::MasterDataFile.new(table_name, path, @override_identifier)
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -1,3 +1,4 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative 'import/master_data_file_list'
3
4
  require_relative "import/executor"
@@ -2,11 +2,12 @@
2
2
 
3
3
  module MasterDataTool
4
4
  class MasterData
5
- attr_reader :csv_path, :model_klass, :columns, :new_records, :updated_records, :no_change_records, :deleted_records
5
+ attr_reader :master_data_file, :model_klass, :columns, :new_records, :updated_records, :no_change_records, :deleted_records
6
6
  attr_reader :before_count, :after_count
7
7
 
8
- def initialize(csv_path, model_klass)
9
- @csv_path = csv_path
8
+ # @param [MasterDataTool::MasterDataFile] master_data_file
9
+ def initialize(master_data_file, model_klass)
10
+ @master_data_file = master_data_file
10
11
  @model_klass = model_klass
11
12
 
12
13
  @loaded = false
@@ -18,8 +19,12 @@ module MasterDataTool
18
19
  @deleted_records = []
19
20
  end
20
21
 
22
+ def basename
23
+ @master_data_file.basename
24
+ end
25
+
21
26
  def load
22
- csv = CSV.read(@csv_path, headers: true, skip_blanks: true)
27
+ csv = CSV.read(@master_data_file.path, headers: true, skip_blanks: true)
23
28
  old_records_by_id = @model_klass.all.index_by(&:id)
24
29
 
25
30
  csv_records_by_id = build_records_from_csv(csv, old_records_by_id)
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+ module MasterDataTool
3
+ class MasterDataFile < Struct.new(:table_name, :path, :override_identifier)
4
+ def initialize(table_name, path, override_identifier)
5
+ super(table_name, path, override_identifier)
6
+ freeze
7
+ end
8
+
9
+ def basename
10
+ self.path.basename
11
+ end
12
+ end
13
+ end
@@ -15,9 +15,9 @@ module MasterDataTool
15
15
  presence: true
16
16
 
17
17
  class << self
18
- def build(csv_path)
19
- version = decide_version(csv_path)
20
- new(name: MasterDataTool.resolve_table_name(csv_path), version: version)
18
+ def build(master_data_file)
19
+ version = decide_version(master_data_file.path)
20
+ new(name: MasterDataTool.resolve_table_name(master_data_file.path, master_data_file.override_identifier), version: version)
21
21
  end
22
22
 
23
23
  def import_records!(records, dry_run: true)
@@ -28,9 +28,10 @@ module MasterDataTool
28
28
  end
29
29
  end
30
30
 
31
- def master_data_will_change?(csv_path)
32
- new_version = decide_version(csv_path)
33
- !where(name: MasterDataTool.resolve_table_name(csv_path), version: new_version).exists?
31
+ # @param [MasterDataTool::MasterDataFile] master_data_file
32
+ def master_data_will_change?(master_data_file)
33
+ new_version = decide_version(master_data_file.path)
34
+ !where(name: master_data_file.table_name, version: new_version).exists?
34
35
  end
35
36
 
36
37
  def decide_version(csv_path)
@@ -6,6 +6,8 @@ module MasterDataTool
6
6
  include Printer
7
7
 
8
8
  def print(message)
9
+ return if @silent
10
+
9
11
  MasterDataTool.config.logger.info message
10
12
  puts message
11
13
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module MasterDataTool
4
- VERSION = "0.13.0"
4
+ VERSION = "0.16.0"
5
5
  end
@@ -4,6 +4,7 @@ require 'csv'
4
4
  require_relative "master_data_tool/version"
5
5
  require_relative "master_data_tool/config"
6
6
  require_relative "master_data_tool/master_data_status"
7
+ require_relative "master_data_tool/master_data_file"
7
8
  require_relative "master_data_tool/master_data"
8
9
  require_relative "master_data_tool/report"
9
10
  require_relative "master_data_tool/dump/executor"
@@ -24,9 +25,11 @@ module MasterDataTool
24
25
  yield config
25
26
  end
26
27
 
27
- def resolve_table_name(csv_path)
28
+ def resolve_table_name(csv_path, override_identifier)
28
29
  # 0001_table_nameのように投入順序を制御可能にする
29
- csv_path.relative_path_from(config.master_data_dir).to_s.gsub(/^\d+_/, '').delete_suffix('.csv')
30
+ relative_path = config.master_data_dir
31
+ relative_path = "#{relative_path}/#{override_identifier}" if override_identifier.present?
32
+ csv_path.relative_path_from(relative_path).to_s.gsub(/^\d+_/, '').delete_suffix('.csv')
30
33
  end
31
34
  end
32
35
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: master_data_tool
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.0
4
+ version: 0.16.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Takahiro Ooishi
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-03-05 00:00:00.000000000 Z
11
+ date: 2022-06-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rspec
@@ -194,7 +194,9 @@ files:
194
194
  - lib/master_data_tool/dump/executor.rb
195
195
  - lib/master_data_tool/import.rb
196
196
  - lib/master_data_tool/import/executor.rb
197
+ - lib/master_data_tool/import/master_data_file_list.rb
197
198
  - lib/master_data_tool/master_data.rb
199
+ - lib/master_data_tool/master_data_file.rb
198
200
  - lib/master_data_tool/master_data_status.rb
199
201
  - lib/master_data_tool/report.rb
200
202
  - lib/master_data_tool/report/core.rb
@@ -205,7 +207,6 @@ files:
205
207
  - lib/master_data_tool/report/verify_report.rb
206
208
  - lib/master_data_tool/version.rb
207
209
  - log/test.log
208
- - master_data_tool.gemspec
209
210
  - scripts/setup.sh
210
211
  - sig/master_data_tool.rbs
211
212
  homepage: https://github.com/taka0125/master_data_tool
@@ -228,7 +229,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
228
229
  - !ruby/object:Gem::Version
229
230
  version: '0'
230
231
  requirements: []
231
- rubygems_version: 3.2.22
232
+ rubygems_version: 3.0.3
232
233
  signing_key:
233
234
  specification_version: 4
234
235
  summary: マスタデータの管理ツール
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative "lib/master_data_tool/version"
4
-
5
- Gem::Specification.new do |spec|
6
- spec.name = "master_data_tool"
7
- spec.version = MasterDataTool::VERSION
8
- spec.authors = ["Takahiro Ooishi"]
9
- spec.email = ["taka0125@gmail.com"]
10
-
11
- spec.summary = "マスタデータの管理ツール"
12
- spec.description = "システムが稼働する上で最初から必要なデータ(マスタデータ)を管理するツールです。"
13
- spec.homepage = "https://github.com/taka0125/master_data_tool"
14
- spec.required_ruby_version = ">= 2.6.0"
15
-
16
- spec.metadata["homepage_uri"] = spec.homepage
17
- spec.metadata["source_code_uri"] = spec.homepage
18
-
19
- # Specify which files should be added to the gem when it is released.
20
- # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
21
- spec.files = Dir.chdir(File.expand_path(__dir__)) do
22
- `git ls-files -z`.split("\x0").reject do |f|
23
- (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
24
- end
25
- end
26
- spec.bindir = "exe"
27
- spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
28
- spec.require_paths = ["lib"]
29
-
30
- spec.add_development_dependency 'rspec'
31
- spec.add_development_dependency 'mysql2'
32
- spec.add_development_dependency 'psych', '~> 3.1'
33
- spec.add_development_dependency 'appraisal'
34
- spec.add_development_dependency 'ridgepole'
35
- spec.add_development_dependency 'database_cleaner-active_record'
36
- spec.add_development_dependency 'standalone_activerecord_boot_loader'
37
-
38
- spec.add_dependency 'activerecord', '>= 5.1.7'
39
- spec.add_dependency 'activesupport'
40
- spec.add_dependency 'thor'
41
- spec.add_dependency 'activerecord-import'
42
- end