gamma 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: eddbdd518cc437aa37849d0d87ec4e9f6fdc726d
-  data.tar.gz: 8d6f3c05fffb45b7cc622f113e51e3f53a451024
+SHA256:
+  metadata.gz: 7d404140fc674781a746207fd1aac842168089729524c03862731dc4ac1c3be6
+  data.tar.gz: 98314431134675ac63c19e594d2a8175a7c69b1adf3f16fbb2da5f4666a00aa7
 SHA512:
-  metadata.gz: ca6074f5e1871af0db1bf377a7fe435f1c26f51de1524961cfe7a9a7fef2f83b98348afb281c96a368c1e850cfd9dc2bb68620ce74b895ccee1635ec419d93f3
-  data.tar.gz: bf5ed88d704f86f870a97a5e70bbfc38b85a1fe5a8cc9a6c4d0a8d7603421301cb9d32018386dbbe27523eb673ca686b92f17e98732b86f52ae342a64ac95764
+  metadata.gz: 244af3cecb957aed2f8aa8c6ad5a1dcb3a5d0a320cb17805e327622476092bc94334e91d2518dc4008a6100bd0aa40f299b2983766bf9f8ae3314144533ef5fc
+  data.tar.gz: daa2b883c86cb2a452e1331bb911f9963f8d9c28096ec71d4b8e7b8c9103b007e2bb0836773a27f8ab821fdfa99a8866e1bb093de53bb8d0a9e2006f89f64cfe

data/Gemfile.lock

@@ -1,46 +1,46 @@
 PATH
   remote: .
   specs:
-    gamma (0.1.7)
+    gamma (0.2.0)
       activesupport
       colorize (~> 0.8.1)
       mysql2
+      parallel
       thor (~> 0.20)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.5)
+    activesupport (7.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-    coderay (1.1.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    coderay (1.1.3)
     colorize (0.8.1)
-    concurrent-ruby (1.0.5)
-    i18n (0.9.5)
+    concurrent-ruby (1.1.10)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    method_source (0.9.0)
-    minitest (5.11.3)
-    mysql2 (0.4.10)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    method_source (1.0.0)
+    minitest (5.16.2)
+    mysql2 (0.5.4)
+    parallel (1.22.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     rake (10.5.0)
-    thor (0.20.0)
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
+    thor (0.20.3)
+    tzinfo (2.0.5)
+      concurrent-ruby (~> 1.0)
 
 PLATFORMS
-  ruby
+  arm64-darwin-21
 
 DEPENDENCIES
-  bundler (~> 1.16)
   gamma!
   minitest (~> 5.0)
   pry
   rake (~> 10.0)
 
 BUNDLED WITH
-   1.16.1
+   2.2.32

data/gamma.gemspec

@@ -1,35 +1,34 @@
-
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "gamma/version"
+require 'gamma/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "gamma"
+  spec.name          = 'gamma'
   spec.version       = Gamma::VERSION
-  spec.authors       = ["Shinsuke Nishio"]
-  spec.email         = ["nishio@densan-labs.net"]
+  spec.authors       = ['Shinsuke Nishio']
+  spec.email         = ['nishio@densan-labs.net']
 
-  spec.summary       = %q{DBSync}
-  spec.description   = %q{DBSync}
-  spec.homepage      = "https://github.com/nishio-dens/gamma"
-  spec.license       = "MIT"
+  spec.summary       = 'DBSync'
+  spec.description   = 'DBSync'
+  spec.homepage      = 'https://github.com/nishio-dens/gamma'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_dependency "mysql2"
-  spec.add_dependency "activesupport"
-  spec.add_dependency "thor", "~> 0.20"
-  spec.add_dependency "colorize", "~> 0.8.1"
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'colorize', '~> 0.8.1'
+  spec.add_dependency 'mysql2'
+  spec.add_dependency 'parallel'
+  spec.add_dependency 'thor', '~> 0.20'
 
-  spec.required_ruby_version = ">= 2.3.0"
+  spec.required_ruby_version = '>= 2.3.0'
 
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest", "~> 5.0"
-  spec.add_development_dependency "pry"
+  spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rake', '~> 10.0'
 end

data/lib/gamma/command/apply.rb

@@ -1,11 +1,13 @@
+require 'parallel'
+
 class Gamma::Command::Apply < Gamma::Command
   def initialize(opts)
     @database_settings = Gamma::DatabaseSettings.new(opts[:settings])
     # TODO: support postgres adapter
     @in_client = Gamma::DatabaseConnector::MysqlConnector.new(@database_settings.in_database)
     @out_client = Gamma::DatabaseConnector::MysqlConnector.new(@database_settings.out_database)
-    @hook_root_dir = opts[:hook_dir] || "."
-    @syncdb = Gamma::SyncDatabase.new(opts[:sync_history] || "./history.json")
+    @hook_root_dir = opts[:hook_dir] || '.'
+    @syncdb = Gamma::SyncDatabase.new(opts[:sync_history] || './history.json')
     @data_parser = Gamma::Parser::DataParser.new(opts[:data], @hook_root_dir, @in_client, @out_client, apply: true)
   end
 
@@ -13,15 +15,23 @@ class Gamma::Command::Apply < Gamma::Command
     tables = @data_parser.gamma_tables
     output_setting_warning(tables)
 
-    tables.each do |t|
+    sync = lambda do |t|
       logger.info("[#{t.sync_mode}] Sync Start #{t.table_name}".green)
 
       case t.sync_mode
-      when "replace"
+      when 'replace'
         Gamma::Importer::Replace.new(@in_client, @out_client, t, apply: true).execute
+      when 'force_replace'
+        Gamma::Importer::Replace.new(@in_client, @out_client, t, apply: true, ignore_error: true).execute
       else
         logger.info("[#{t.sync_mode}] Sync Failed #{t.table_name}. Unknown Sync mode".red)
       end
     end
+
+    if ENV['PARALLEL_COUNT']
+      Parallel.each(tables, in_processes: ENV['PARALLEL_COUNT'].to_i) { |t| sync.call(t) }
+    else
+      tables.each { |t| sync.call(t) }
+    end
   end
 end

data/lib/gamma/command/dryrun.rb

@@ -19,6 +19,8 @@ class Gamma::Command::Dryrun < Gamma::Command
       case t.sync_mode
       when "replace"
         Gamma::Importer::Replace.new(@in_client, @out_client, t).execute
+      when "force_replace"
+        Gamma::Importer::Replace.new(@in_client, @out_client, t, ignore_error: true).execute
       else
         logger.info("[#{t.sync_mode}] Sync Failed #{t.table_name}. Unknown Sync mode".red)
       end

data/lib/gamma/importer/replace.rb

@@ -1,11 +1,12 @@
 class Gamma::Importer::Replace < Gamma::Importer
   BATCH_SIZE = 1000
 
-  def initialize(in_client, out_client, table, apply: false)
+  def initialize(in_client, out_client, table, apply: false, ignore_error: false)
     @in_client = in_client
     @out_client = out_client
     @table = table
     @apply = apply
+    @ignore_error = ignore_error
   end
 
   def execute
@@ -19,22 +20,26 @@ class Gamma::Importer::Replace < Gamma::Importer
 
   def batch_sync
     columns = @table.in_exist_columns & @table.out_exist_columns
-    primary_key = "id" # TODO: Fixme
+    primary_key = 'id' # TODO: Fixme
 
     current_in_pid = 0
-    while true do
-      select_columns = columns.map { |c| "`#{c}`" }.join(",")
+    while true
+      select_columns = columns.map { |c| "`#{c}`" }.join(',')
       break unless select_columns.present?
 
       in_query = "SELECT #{select_columns} FROM #{@table.table_name} WHERE #{primary_key} > #{current_in_pid} ORDER BY #{primary_key} ASC LIMIT #{BATCH_SIZE}"
-      logger.info(in_query) if ENV["DEBUG"]
+      logger.info(in_query) if ENV['DEBUG']
       in_records = @in_client.client.query(in_query).to_a
 
       break unless in_records.present?
 
       out_records = exist_records(select_columns, primary_key, in_records.map { |v| v[primary_key] })
 
-      in_records.map { |ir| [ir, out_records.find { |v| ir[primary_key] == v[primary_key] }] }.each do |in_record, out_record|
+      in_records.map do |ir|
+        [ir, out_records.find do |v|
+               ir[primary_key] == v[primary_key]
+             end]
+      end.each do |in_record, out_record|
         if out_record.present?
           update_out_record(in_record, out_record, primary_key)
         else
@@ -44,14 +49,14 @@ class Gamma::Importer::Replace < Gamma::Importer
 
       current_in_pid = in_records.last[primary_key]
     end
-  rescue => e
+  rescue StandardError => e
     logger.error("Sync Error #{@table.table_name} \n #{e}\n #{e.backtrace.join("\n")}".red)
   end
 
   def exist_records(select_columns, primary_key, in_pids)
     return [] unless in_pids.present?
 
-    query = "SELECT #{select_columns} FROM #{@table.table_name} WHERE #{primary_key} in (#{in_pids.join(",")})"
+    query = "SELECT #{select_columns} FROM #{@table.table_name} WHERE #{primary_key} in (#{in_pids.join(',')})"
     @out_client.client.query(query).to_a
   end
 
@@ -67,10 +72,10 @@ class Gamma::Importer::Replace < Gamma::Importer
         UPDATE `#{@table.table_name}` SET #{values} WHERE #{primary_key} = #{record[primary_key]}
       EOS
       query = query.strip_heredoc
-      logger.info(query) if ENV["DEBUG"]
+      logger.info(query) if ENV['DEBUG']
 
       if @apply
-        @out_client.client.query(query)
+        exec_query(query)
       else
         logger.info("DRYRUN: #{query}")
       end
@@ -80,17 +85,17 @@ class Gamma::Importer::Replace < Gamma::Importer
   def insert_out_record(in_record)
     record = @table.record_value(in_record)
     columns = (@table.in_exist_columns & @table.out_exist_columns).reject { |c| record[c].nil? }
-    select_columns = columns.map { |c| "`#{c}`" }.join(",")
+    select_columns = columns.map { |c| "`#{c}`" }.join(',')
     values = insert_record_values(record, columns)
 
     query = <<-EOS
       INSERT INTO #{@table.table_name}(#{select_columns}) VALUES (#{values})
     EOS
     query = query.strip_heredoc
-    logger.info(query) if ENV["DEBUG"]
+    logger.info(query) if ENV['DEBUG']
 
     if @apply
-      @out_client.client.query(query)
+      exec_query(query)
     else
       logger.info("DRYRUN: #{query}")
     end
@@ -100,23 +105,34 @@ class Gamma::Importer::Replace < Gamma::Importer
     r = record
     columns.map do |v|
       c = if r[v].is_a?(Time)
-            r[v].strftime("%Y-%m-%d %H:%M:%S")
+            r[v].strftime('%Y-%m-%d %H:%M:%S')
           else
             r[v]
           end
-      "\"#{c.to_s.gsub('"', '\"').gsub("'", "\'")}\""
-    end.join(",")
+      "\"#{@out_client.client.escape(c.to_s)}\""
+    end.join(',')
   end
 
   def update_record_values(record, columns)
     r = record
     columns.map do |v|
       c = if r[v].is_a?(Time)
-            r[v].strftime("%Y-%m-%d %H:%M:%S")
+            r[v].strftime('%Y-%m-%d %H:%M:%S')
           else
             r[v]
           end
-      "`#{v}` = \"#{c.to_s.gsub('"', '\"').gsub("'", "\'")}\""
-    end.join(",")
+      "`#{v}` = \"#{@out_client.client.escape(c.to_s)}\""
+    end.join(',')
+  end
+
+  def exec_query(query)
+    @out_client.client.query(query)
+  rescue StandardError => e
+    if @ignore_error
+      logger.error('An ERROR has occurred and ignore it.')
+      logger.error(e)
+    else
+      throw e
+    end
   end
 end

data/lib/gamma/version.rb

@@ -1,3 +1,3 @@
 module Gamma
-  VERSION = "0.1.7"
+  VERSION = '0.2.0'
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: gamma
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - Shinsuke Nishio
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-03-09 00:00:00.000000000 Z
+date: 2022-07-22 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: mysql2
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -39,61 +25,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: 0.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
-  name: colorize
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
+        version: '0.20'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0.20'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: DBSync
 email:
 - nishio@densan-labs.net
@@ -164,7 +164,7 @@ homepage: https://github.com/nishio-dens/gamma
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -179,9 +179,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.8
-signing_key: 
+rubygems_version: 3.2.32
+signing_key:
 specification_version: 4
 summary: DBSync
 test_files: []