master_data_tool 0.15.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9ee8634a9d307fdaa7f6cae268d9955a029309dbd020755ecac87ba0f8f6c4a
-  data.tar.gz: 31c3397ccc5d219fe933b7a0aa856878578fa061c4ab620d4e1dab39312d446c
+  metadata.gz: 553fcd688f3ae346cc356961473852142c87f0ee66145b9e1892bfac3e230927
+  data.tar.gz: 64813eea92f8afc4612cdeb19f281c11ecef5537a207a736cd38ef68d57d9adf
 SHA512:
-  metadata.gz: 8409b81d4129b18b0fa489ca0fbaf8a73f5fac79726ee4fed5745bf9312ac24958c6908735fc7a0fc453184e17609348b68e741939b98105547a4de6e1e02315
-  data.tar.gz: f1648f8c1123911468988ea553c1fbe5ad38b81893c5bd376c5f29f983d325672ce8c762020e8f91c7b83cca7d30f3da71919d69e24c11b728cdac7ce044a827
+  metadata.gz: 9b617aed72fbd9bc0a355e81baef951c4f7f3d4cdf3838cf80b7e35533c98ba4fdfc0c01137d6b28e81f4d60798f21c4b81c94f1c53badba689edfe6d9c0c117
+  data.tar.gz: 0d1daa98e982a6bb918d6dc63dde683b8ce65181337a6fb1754a124d7f93d0e6daa7b25e5cfa5f31a16261c8c4624d6544313a72105dff7746d9df69c28277d5

data/README.md

@@ -35,17 +35,18 @@ Or install it yourself as:
 
 ### マスタデータの投入
 
-| option               | default | 内容                                |
-|----------------------| --- |-----------------------------------|
-| --dry-run            | true | dry-runモードで実行する（データ変更は行わない）       |
-| --verify             | true | データ投入後に全テーブル・全レコードのバリデーションチェックを行う |
-| --only-import-tables | [] | 指定したテーブルのみデータ投入を行う                |
-| --except-import-tables | [] | 指定したテーブルのデータ投入を行わない               |
-| --only-verify-tables | [] | 指定したテーブルのみ投入後のバリデーションチェックを行う      |
-| --except-verify-tables | [] | 指定したテーブルのバリデーションチェックを行わない         |
-| --skip-no-change     | true | CSVファイルに更新がないテーブルをスキップする          |
-| --silent | false | 結果の出力をやめる                         |
-| --delete-all-ignore-foreign-key | false | 外部キー制約を無視してレコードを消すかどうか |
+| option               | default | 内容                                                              |
+|----------------------| --- |-----------------------------------------------------------------|
+| --dry-run            | true | dry-runモードで実行する（データ変更は行わない）                                     |
+| --verify             | true | データ投入後に全テーブル・全レコードのバリデーションチェックを行う                               |
+| --only-import-tables | [] | 指定したテーブルのみデータ投入を行う                                              |
+| --except-import-tables | [] | 指定したテーブルのデータ投入を行わない                                             |
+| --only-verify-tables | [] | 指定したテーブルのみ投入後のバリデーションチェックを行う                                    |
+| --except-verify-tables | [] | 指定したテーブルのバリデーションチェックを行わない                                       |
+| --skip-no-change     | true | CSVファイルに更新がないテーブルをスキップする                                        |
+| --silent | false | 結果の出力をやめる                                                       |
+| --delete-all-ignore-foreign-key | false | 外部キー制約を無視してレコードを消すかどうか                                          |
+| --override_identifier | nil | fixtures/#{override_identifier} のディレクトリにある内容でfixturesを上書きして投入する |
 
 ```bash
 bundle exec master_data_tool import

data/exe/master_data_tool

@@ -17,6 +17,7 @@ module MasterDataTool
     option :except_verify_tables, default: nil, type: :array
     option :skip_no_change, default: nil, type: :boolean
     option :silent, default: nil, type: :boolean
+    option :override_identifier, default: nil, type: :string
     option :delete_all_ignore_foreign_key, default: nil, type: :boolean
     desc 'import', 'import'
     def import

data/lib/master_data_tool/config.rb

@@ -9,12 +9,16 @@ module MasterDataTool
     config_accessor :dump_ignore_columns
     config_accessor :default_import_options
     config_accessor :logger
+    config_accessor :preload_associations
+    config_accessor :eager_load_associations
 
     def initialize
       self.master_data_dir = nil
       self.dump_ignore_tables = %w[]
       self.dump_ignore_columns = %w[]
       self.default_import_options = {}
+      self.preload_associations = {} # key: Class, value: associations
+      self.eager_load_associations = {} # key: Class, value: associations
       self.logger = Logger.new(nil)
     end
   end

data/lib/master_data_tool/import/executor.rb

@@ -12,6 +12,7 @@ module MasterDataTool
                      skip_no_change: true,
                      silent: false,
                      delete_all_ignore_foreign_key: false,
+                     override_identifier: nil,
                      report_printer: MasterDataTool::Report::DefaultPrinter.new)
 
         @dry_run = dry_run
@@ -23,6 +24,7 @@ module MasterDataTool
         @skip_no_change = skip_no_change
         @silent = silent
         @delete_all_ignore_foreign_key = delete_all_ignore_foreign_key
+        @override_identifier = override_identifier
         @report_printer = report_printer
         @report_printer.silent = silent
       end
@@ -31,17 +33,17 @@ module MasterDataTool
         ApplicationRecord.transaction do
           print_execute_options
 
-          master_data_list = build_master_data_list
+          master_data_collection = build_master_data_collection
 
-          import_all!(master_data_list)
-          verify_all!(master_data_list) if @verify
-          save_master_data_statuses!(master_data_list)
+          import_all!(master_data_collection)
+          verify_all!(master_data_collection) if @verify
+          save_master_data_statuses!(master_data_collection)
 
-          print_affected_tables(master_data_list)
+          print_affected_tables(master_data_collection)
 
           raise DryRunError if @dry_run
 
-          master_data_list
+          master_data_collection
         end
       rescue DryRunError
         puts "[DryRun] end"
@@ -59,23 +61,18 @@ module MasterDataTool
         puts "================="
       end
 
-      def build_master_data_list
-        [].tap do |master_data_list|
-          extract_master_data_csv_paths.each do |path|
-            table_name = MasterDataTool.resolve_table_name(path)
-            load_skip = load_skip_table?(table_name, path)
-
-            model_klass = Object.const_get(table_name.classify)
-            master_data = MasterData.new(path, model_klass)
-            master_data.load unless load_skip
-
-            master_data_list << master_data
+      def build_master_data_collection
+        MasterDataCollection.new.tap do |collection|
+          MasterDataTool::MasterDataFileCollection.new(override_identifier: @override_identifier).each do |master_data_file|
+            load_skip = load_skip_table?(master_data_file)
+            master_data = MasterData.build(master_data_file, load: !load_skip)
+            collection.append(master_data)
           end
-        end.sort_by { |m| m.csv_path } # 外部キー制約などがある場合には先に入れておかないといけないデータなどがある。なので、プレフィックスを付けて順序を指定して貰う
+        end
       end
 
-      def import_all!(master_data_list)
-        master_data_list.each do |master_data|
+      def import_all!(master_data_collection)
+        master_data_collection.each do |master_data|
           next unless master_data.loaded?
           next if import_skip_table?(master_data.table_name)
 
@@ -84,8 +81,8 @@ module MasterDataTool
         end
       end
 
-      def verify_all!(master_data_list)
-        master_data_list.each do |master_data|
+      def verify_all!(master_data_collection)
+        master_data_collection.each do |master_data|
           next if verify_skip_table?(master_data.table_name)
 
           report = master_data.verify!(ignore_fail: @dry_run)
@@ -93,19 +90,19 @@ module MasterDataTool
         end
       end
 
-      def save_master_data_statuses!(master_data_list)
+      def save_master_data_statuses!(master_data_collection)
         records = []
-        master_data_list.each do |master_data|
+        master_data_collection.each do |master_data|
           next unless master_data.loaded?
 
-          records << MasterDataTool::MasterDataStatus.build(master_data.csv_path)
+          records << MasterDataTool::MasterDataStatus.build(master_data.master_data_file)
         end
 
         MasterDataTool::MasterDataStatus.import_records!(records, dry_run: @dry_run)
       end
 
-      def print_affected_tables(master_data_list)
-        master_data_list.each do |master_data|
+      def print_affected_tables(master_data_collection)
+        master_data_collection.each do |master_data|
           next unless master_data.loaded?
           next unless master_data.affected?
 
@@ -114,11 +111,11 @@ module MasterDataTool
         end
       end
 
-      def load_skip_table?(table_name, csv_path)
-        return true if import_skip_table?(table_name)
+      def load_skip_table?(master_data_file)
+        return true if import_skip_table?(master_data_file.table_name)
         return false unless @skip_no_change
 
-        !MasterDataTool::MasterDataStatus.master_data_will_change?(csv_path)
+        !MasterDataTool::MasterDataStatus.master_data_will_change?(master_data_file)
       end
 
       def import_skip_table?(table_name)
@@ -148,6 +145,13 @@ module MasterDataTool
         pattern = Pathname.new(MasterDataTool.config.master_data_dir).join('*.csv').to_s
         Pathname.glob(pattern).select(&:file?)
       end
+
+      def overridden_master_data_csv_paths
+        return [] unless @override_identifier
+
+        pattern = Pathname.new(MasterDataTool.config.master_data_dir).join(@override_identifier).join('*.csv').to_s
+        Pathname.glob(pattern).select(&:file?)
+      end
     end
   end
 end

data/lib/master_data_tool/master_data.rb

@@ -2,11 +2,12 @@
 
 module MasterDataTool
   class MasterData
-    attr_reader :csv_path, :model_klass, :columns, :new_records, :updated_records, :no_change_records, :deleted_records
+    attr_reader :master_data_file, :model_klass, :columns, :new_records, :updated_records, :no_change_records, :deleted_records
     attr_reader :before_count, :after_count
 
-    def initialize(csv_path, model_klass)
-      @csv_path = csv_path
+    # @param [MasterDataTool::MasterDataFile] master_data_file
+    def initialize(master_data_file, model_klass)
+      @master_data_file = master_data_file
       @model_klass = model_klass
 
       @loaded = false
@@ -18,8 +19,21 @@ module MasterDataTool
       @deleted_records = []
     end
 
+    class << self
+      def build(master_data_file, load: false)
+        model_klass = Object.const_get(master_data_file.table_name.classify)
+        new(master_data_file, model_klass).tap do |record|
+          record.load if load
+        end
+      end
+    end
+
+    def basename
+      @master_data_file.basename
+    end
+
     def load
-      csv = CSV.read(@csv_path, headers: true, skip_blanks: true)
+      csv = CSV.read(@master_data_file.path, headers: true, skip_blanks: true)
       old_records_by_id = @model_klass.all.index_by(&:id)
 
       csv_records_by_id = build_records_from_csv(csv, old_records_by_id)
@@ -121,7 +135,11 @@ module MasterDataTool
 
     def verify!(ignore_fail: false)
       MasterDataTool::Report::VerifyReport.new(self).tap do |report|
-        @model_klass.all.find_each do |record|
+        scoped = @model_klass.all
+        scoped = scoped.preload(preload_associations) if preload_associations
+        scoped = scoped.eager_load(eager_load_associations) if eager_load_associations
+
+        scoped.find_each do |record|
           valid = record.valid?
           report.append(MasterDataTool::Report::VerifyReport.build_verify_record_report(self, record, valid))
           next if ignore_fail
@@ -140,6 +158,14 @@ module MasterDataTool
 
     private
 
+    def preload_associations
+      @preload_associations ||= MasterDataTool.config.preload_associations.dig(@model_klass.to_s.to_sym)
+    end
+
+    def eager_load_associations
+      @eager_load_associations ||= MasterDataTool.config.eager_load_associations.dig(@model_klass.to_s.to_sym)
+    end
+
     def build_records_from_csv(csv, old_records_by_id)
       {}.tap do |records|
         csv.each do |row|

data/lib/master_data_tool/master_data_collection.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module MasterDataTool
+  class MasterDataCollection
+    def initialize
+      @collection = []
+    end
+
+    def append(master_data)
+      @collection << master_data
+    end
+
+    def each
+      return enum_for(:each) unless block_given?
+
+      @collection.sort_by(&:basename).each do |master_data|
+        yield master_data
+      end
+    end
+
+    def to_a
+      each.to_a
+    end
+  end
+end

data/lib/master_data_tool/master_data_file.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+module MasterDataTool
+  class MasterDataFile
+    attr_reader :table_name, :path, :override_identifier
+
+    def initialize(table_name, path, override_identifier)
+      @table_name = table_name
+      @path = path
+      @override_identifier = override_identifier
+      freeze
+    end
+
+    class << self
+      def build(path, override_identifier)
+        table_name = MasterDataTool.resolve_table_name(path, override_identifier)
+        new(table_name, path, override_identifier)
+      end
+    end
+
+    def basename
+      @path.basename
+    end
+
+    def ==(other)
+      other.class === self &&
+        other.hash == hash
+    end
+
+    alias eql? ==
+
+    def hash
+      [@table_name, @path, @override_identifier].join.hash
+    end
+  end
+end

data/lib/master_data_tool/master_data_file_collection.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module MasterDataTool
+  class MasterDataFileCollection
+    def initialize(override_identifier: nil)
+      @override_identifier = override_identifier
+      @collection = build
+      freeze
+    end
+
+    def each
+      return enum_for(:each) unless block_given?
+
+      @collection.each do |file|
+        yield file
+      end
+    end
+
+    def to_a
+      each.to_a
+    end
+
+    private
+
+    def build
+      files = extract_master_data_csv_paths.presence&.index_by(&:table_name)
+      overridden_files = overridden_master_data_csv_paths.presence&.index_by(&:table_name) || {}
+
+      table_names = (files.keys + overridden_files.keys).uniq
+      table_names.map do |table_name|
+        overridden_files[table_name] || files[table_name]
+      end
+    end
+
+    def extract_master_data_csv_paths
+      pattern = Pathname.new(MasterDataTool.config.master_data_dir).join('*.csv').to_s
+      Pathname.glob(pattern).select(&:file?).map do |path|
+        MasterDataFile.build(path, nil)
+      end
+    end
+
+    def overridden_master_data_csv_paths
+      return [] if @override_identifier.blank?
+
+      pattern = Pathname.new(MasterDataTool.config.master_data_dir).join(@override_identifier).join('*.csv').to_s
+      Pathname.glob(pattern).select(&:file?).map do |path|
+        MasterDataFile.build(path, @override_identifier)
+      end
+    end
+  end
+end

data/lib/master_data_tool/master_data_status.rb

@@ -15,9 +15,9 @@ module MasterDataTool
               presence: true
 
     class << self
-      def build(csv_path)
-        version = decide_version(csv_path)
-        new(name: MasterDataTool.resolve_table_name(csv_path), version: version)
+      def build(master_data_file)
+        version = decide_version(master_data_file.path)
+        new(name: MasterDataTool.resolve_table_name(master_data_file.path, master_data_file.override_identifier), version: version)
       end
 
       def import_records!(records, dry_run: true)
@@ -28,9 +28,10 @@ module MasterDataTool
         end
       end
 
-      def master_data_will_change?(csv_path)
-        new_version = decide_version(csv_path)
-        !where(name: MasterDataTool.resolve_table_name(csv_path), version: new_version).exists?
+      # @param [MasterDataTool::MasterDataFile] master_data_file
+      def master_data_will_change?(master_data_file)
+        new_version = decide_version(master_data_file.path)
+        !where(name: master_data_file.table_name, version: new_version).exists?
       end
 
       def decide_version(csv_path)

data/lib/master_data_tool/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MasterDataTool
-  VERSION = "0.15.0"
+  VERSION = "0.18.0"
 end

data/lib/master_data_tool.rb

@@ -4,7 +4,10 @@ require 'csv'
 require_relative "master_data_tool/version"
 require_relative "master_data_tool/config"
 require_relative "master_data_tool/master_data_status"
+require_relative "master_data_tool/master_data_file"
+require_relative "master_data_tool/master_data_file_collection"
 require_relative "master_data_tool/master_data"
+require_relative "master_data_tool/master_data_collection"
 require_relative "master_data_tool/report"
 require_relative "master_data_tool/dump/executor"
 require_relative "master_data_tool/import"
@@ -24,9 +27,11 @@ module MasterDataTool
       yield config
     end
 
-    def resolve_table_name(csv_path)
+    def resolve_table_name(csv_path, override_identifier)
       # 0001_table_nameのように投入順序を制御可能にする
-      csv_path.relative_path_from(config.master_data_dir).to_s.gsub(/^\d+_/, '').delete_suffix('.csv')
+      relative_path = config.master_data_dir
+      relative_path = "#{relative_path}/#{override_identifier}" if override_identifier.present?
+      csv_path.relative_path_from(relative_path).to_s.gsub(/^\d+_/, '').delete_suffix('.csv')
     end
   end
 end

data/master_data_tool.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "lib/master_data_tool/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "master_data_tool"
+  spec.version = MasterDataTool::VERSION
+  spec.authors = ["Takahiro Ooishi"]
+  spec.email = ["taka0125@gmail.com"]
+
+  spec.summary = "マスタデータの管理ツール"
+  spec.description = "システムが稼働する上で最初から必要なデータ（マスタデータ）を管理するツールです。"
+  spec.homepage = "https://github.com/taka0125/master_data_tool"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'mysql2'
+  spec.add_development_dependency 'psych', '~> 3.1'
+  spec.add_development_dependency 'appraisal'
+  spec.add_development_dependency 'ridgepole'
+  spec.add_development_dependency 'database_cleaner-active_record'
+  spec.add_development_dependency 'standalone_activerecord_boot_loader'
+
+  spec.add_dependency 'activerecord', '>= 5.1.7'
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'thor'
+  spec.add_dependency 'activerecord-import'
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: master_data_tool
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.18.0
 platform: ruby
 authors:
 - Takahiro Ooishi
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-06 00:00:00.000000000 Z
+date: 2022-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -195,6 +195,9 @@ files:
 - lib/master_data_tool/import.rb
 - lib/master_data_tool/import/executor.rb
 - lib/master_data_tool/master_data.rb
+- lib/master_data_tool/master_data_collection.rb
+- lib/master_data_tool/master_data_file.rb
+- lib/master_data_tool/master_data_file_collection.rb
 - lib/master_data_tool/master_data_status.rb
 - lib/master_data_tool/report.rb
 - lib/master_data_tool/report/core.rb
@@ -205,6 +208,7 @@ files:
 - lib/master_data_tool/report/verify_report.rb
 - lib/master_data_tool/version.rb
 - log/test.log
+- master_data_tool.gemspec
 - scripts/setup.sh
 - sig/master_data_tool.rbs
 homepage: https://github.com/taka0125/master_data_tool
@@ -227,7 +231,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: マスタデータの管理ツール