gamma 0.1.9 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 734f8c0e5eb5b42cd1c65cd7b21e496be67c3ab7
4
- data.tar.gz: 86291665b498c940064b29e3efab8b1cd5e87f6a
2
+ SHA256:
3
+ metadata.gz: 7d404140fc674781a746207fd1aac842168089729524c03862731dc4ac1c3be6
4
+ data.tar.gz: 98314431134675ac63c19e594d2a8175a7c69b1adf3f16fbb2da5f4666a00aa7
5
5
  SHA512:
6
- metadata.gz: 1a6b61e68d399a374d15bcb079f2ce06700442b3d00b70ee308bd166f2733dff45c9fe15912d0e82c17ebfaaed38979da301b6e75c220480c20c9373a1c78796
7
- data.tar.gz: ac1f8a1d79f795cb08fa185f81c0066449a2c283a2a54ff0442199bd4f1af34f202619f81641ac299768d611e63a338b450c439e6ff6fc1c3eca78297cd05b3e
6
+ metadata.gz: 244af3cecb957aed2f8aa8c6ad5a1dcb3a5d0a320cb17805e327622476092bc94334e91d2518dc4008a6100bd0aa40f299b2983766bf9f8ae3314144533ef5fc
7
+ data.tar.gz: daa2b883c86cb2a452e1331bb911f9963f8d9c28096ec71d4b8e7b8c9103b007e2bb0836773a27f8ab821fdfa99a8866e1bb093de53bb8d0a9e2006f89f64cfe
data/Gemfile.lock CHANGED
@@ -1,46 +1,46 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gamma (0.1.9)
4
+ gamma (0.2.0)
5
5
  activesupport
6
6
  colorize (~> 0.8.1)
7
7
  mysql2
8
+ parallel
8
9
  thor (~> 0.20)
9
10
 
10
11
  GEM
11
12
  remote: https://rubygems.org/
12
13
  specs:
13
- activesupport (5.2.1)
14
+ activesupport (7.0.3.1)
14
15
  concurrent-ruby (~> 1.0, >= 1.0.2)
15
- i18n (>= 0.7, < 2)
16
- minitest (~> 5.1)
17
- tzinfo (~> 1.1)
18
- coderay (1.1.2)
16
+ i18n (>= 1.6, < 2)
17
+ minitest (>= 5.1)
18
+ tzinfo (~> 2.0)
19
+ coderay (1.1.3)
19
20
  colorize (0.8.1)
20
- concurrent-ruby (1.0.5)
21
- i18n (1.1.1)
21
+ concurrent-ruby (1.1.10)
22
+ i18n (1.12.0)
22
23
  concurrent-ruby (~> 1.0)
23
- method_source (0.9.0)
24
- minitest (5.11.3)
25
- mysql2 (0.5.2)
26
- pry (0.11.3)
27
- coderay (~> 1.1.0)
28
- method_source (~> 0.9.0)
24
+ method_source (1.0.0)
25
+ minitest (5.16.2)
26
+ mysql2 (0.5.4)
27
+ parallel (1.22.1)
28
+ pry (0.14.1)
29
+ coderay (~> 1.1)
30
+ method_source (~> 1.0)
29
31
  rake (10.5.0)
30
- thor (0.20.0)
31
- thread_safe (0.3.6)
32
- tzinfo (1.2.5)
33
- thread_safe (~> 0.1)
32
+ thor (0.20.3)
33
+ tzinfo (2.0.5)
34
+ concurrent-ruby (~> 1.0)
34
35
 
35
36
  PLATFORMS
36
- ruby
37
+ arm64-darwin-21
37
38
 
38
39
  DEPENDENCIES
39
- bundler (~> 1.16)
40
40
  gamma!
41
41
  minitest (~> 5.0)
42
42
  pry
43
43
  rake (~> 10.0)
44
44
 
45
45
  BUNDLED WITH
46
- 1.16.1
46
+ 2.2.32
data/gamma.gemspec CHANGED
@@ -1,35 +1,34 @@
1
-
2
- lib = File.expand_path("../lib", __FILE__)
1
+ lib = File.expand_path('lib', __dir__)
3
2
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
- require "gamma/version"
3
+ require 'gamma/version'
5
4
 
6
5
  Gem::Specification.new do |spec|
7
- spec.name = "gamma"
6
+ spec.name = 'gamma'
8
7
  spec.version = Gamma::VERSION
9
- spec.authors = ["Shinsuke Nishio"]
10
- spec.email = ["nishio@densan-labs.net"]
8
+ spec.authors = ['Shinsuke Nishio']
9
+ spec.email = ['nishio@densan-labs.net']
11
10
 
12
- spec.summary = %q{DBSync}
13
- spec.description = %q{DBSync}
14
- spec.homepage = "https://github.com/nishio-dens/gamma"
15
- spec.license = "MIT"
11
+ spec.summary = 'DBSync'
12
+ spec.description = 'DBSync'
13
+ spec.homepage = 'https://github.com/nishio-dens/gamma'
14
+ spec.license = 'MIT'
16
15
 
17
16
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
17
  f.match(%r{^(test|spec|features)/})
19
18
  end
20
- spec.bindir = "exe"
19
+ spec.bindir = 'exe'
21
20
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
22
- spec.require_paths = ["lib"]
21
+ spec.require_paths = ['lib']
23
22
 
24
- spec.add_dependency "mysql2"
25
- spec.add_dependency "activesupport"
26
- spec.add_dependency "thor", "~> 0.20"
27
- spec.add_dependency "colorize", "~> 0.8.1"
23
+ spec.add_dependency 'activesupport'
24
+ spec.add_dependency 'colorize', '~> 0.8.1'
25
+ spec.add_dependency 'mysql2'
26
+ spec.add_dependency 'parallel'
27
+ spec.add_dependency 'thor', '~> 0.20'
28
28
 
29
- spec.required_ruby_version = ">= 2.3.0"
29
+ spec.required_ruby_version = '>= 2.3.0'
30
30
 
31
- spec.add_development_dependency "bundler", "~> 1.16"
32
- spec.add_development_dependency "rake", "~> 10.0"
33
- spec.add_development_dependency "minitest", "~> 5.0"
34
- spec.add_development_dependency "pry"
31
+ spec.add_development_dependency 'minitest', '~> 5.0'
32
+ spec.add_development_dependency 'pry'
33
+ spec.add_development_dependency 'rake', '~> 10.0'
35
34
  end
@@ -1,11 +1,13 @@
1
+ require 'parallel'
2
+
1
3
  class Gamma::Command::Apply < Gamma::Command
2
4
  def initialize(opts)
3
5
  @database_settings = Gamma::DatabaseSettings.new(opts[:settings])
4
6
  # TODO: support postgres adapter
5
7
  @in_client = Gamma::DatabaseConnector::MysqlConnector.new(@database_settings.in_database)
6
8
  @out_client = Gamma::DatabaseConnector::MysqlConnector.new(@database_settings.out_database)
7
- @hook_root_dir = opts[:hook_dir] || "."
8
- @syncdb = Gamma::SyncDatabase.new(opts[:sync_history] || "./history.json")
9
+ @hook_root_dir = opts[:hook_dir] || '.'
10
+ @syncdb = Gamma::SyncDatabase.new(opts[:sync_history] || './history.json')
9
11
  @data_parser = Gamma::Parser::DataParser.new(opts[:data], @hook_root_dir, @in_client, @out_client, apply: true)
10
12
  end
11
13
 
@@ -13,17 +15,23 @@ class Gamma::Command::Apply < Gamma::Command
13
15
  tables = @data_parser.gamma_tables
14
16
  output_setting_warning(tables)
15
17
 
16
- tables.each do |t|
18
+ sync = lambda do |t|
17
19
  logger.info("[#{t.sync_mode}] Sync Start #{t.table_name}".green)
18
20
 
19
21
  case t.sync_mode
20
- when "replace"
22
+ when 'replace'
21
23
  Gamma::Importer::Replace.new(@in_client, @out_client, t, apply: true).execute
22
- when "force_replace"
24
+ when 'force_replace'
23
25
  Gamma::Importer::Replace.new(@in_client, @out_client, t, apply: true, ignore_error: true).execute
24
26
  else
25
27
  logger.info("[#{t.sync_mode}] Sync Failed #{t.table_name}. Unknown Sync mode".red)
26
28
  end
27
29
  end
30
+
31
+ if ENV['PARALLEL_COUNT']
32
+ Parallel.each(tables, in_processes: ENV['PARALLEL_COUNT'].to_i) { |t| sync.call(t) }
33
+ else
34
+ tables.each { |t| sync.call(t) }
35
+ end
28
36
  end
29
37
  end
@@ -19,6 +19,8 @@ class Gamma::Command::Dryrun < Gamma::Command
19
19
  case t.sync_mode
20
20
  when "replace"
21
21
  Gamma::Importer::Replace.new(@in_client, @out_client, t).execute
22
+ when "force_replace"
23
+ Gamma::Importer::Replace.new(@in_client, @out_client, t, ignore_error: true).execute
22
24
  else
23
25
  logger.info("[#{t.sync_mode}] Sync Failed #{t.table_name}. Unknown Sync mode".red)
24
26
  end
@@ -20,22 +20,26 @@ class Gamma::Importer::Replace < Gamma::Importer
20
20
 
21
21
  def batch_sync
22
22
  columns = @table.in_exist_columns & @table.out_exist_columns
23
- primary_key = "id" # TODO: Fixme
23
+ primary_key = 'id' # TODO: Fixme
24
24
 
25
25
  current_in_pid = 0
26
- while true do
27
- select_columns = columns.map { |c| "`#{c}`" }.join(",")
26
+ while true
27
+ select_columns = columns.map { |c| "`#{c}`" }.join(',')
28
28
  break unless select_columns.present?
29
29
 
30
30
  in_query = "SELECT #{select_columns} FROM #{@table.table_name} WHERE #{primary_key} > #{current_in_pid} ORDER BY #{primary_key} ASC LIMIT #{BATCH_SIZE}"
31
- logger.info(in_query) if ENV["DEBUG"]
31
+ logger.info(in_query) if ENV['DEBUG']
32
32
  in_records = @in_client.client.query(in_query).to_a
33
33
 
34
34
  break unless in_records.present?
35
35
 
36
36
  out_records = exist_records(select_columns, primary_key, in_records.map { |v| v[primary_key] })
37
37
 
38
- in_records.map { |ir| [ir, out_records.find { |v| ir[primary_key] == v[primary_key] }] }.each do |in_record, out_record|
38
+ in_records.map do |ir|
39
+ [ir, out_records.find do |v|
40
+ ir[primary_key] == v[primary_key]
41
+ end]
42
+ end.each do |in_record, out_record|
39
43
  if out_record.present?
40
44
  update_out_record(in_record, out_record, primary_key)
41
45
  else
@@ -45,14 +49,14 @@ class Gamma::Importer::Replace < Gamma::Importer
45
49
 
46
50
  current_in_pid = in_records.last[primary_key]
47
51
  end
48
- rescue => e
52
+ rescue StandardError => e
49
53
  logger.error("Sync Error #{@table.table_name} \n #{e}\n #{e.backtrace.join("\n")}".red)
50
54
  end
51
55
 
52
56
  def exist_records(select_columns, primary_key, in_pids)
53
57
  return [] unless in_pids.present?
54
58
 
55
- query = "SELECT #{select_columns} FROM #{@table.table_name} WHERE #{primary_key} in (#{in_pids.join(",")})"
59
+ query = "SELECT #{select_columns} FROM #{@table.table_name} WHERE #{primary_key} in (#{in_pids.join(',')})"
56
60
  @out_client.client.query(query).to_a
57
61
  end
58
62
 
@@ -68,7 +72,7 @@ class Gamma::Importer::Replace < Gamma::Importer
68
72
  UPDATE `#{@table.table_name}` SET #{values} WHERE #{primary_key} = #{record[primary_key]}
69
73
  EOS
70
74
  query = query.strip_heredoc
71
- logger.info(query) if ENV["DEBUG"]
75
+ logger.info(query) if ENV['DEBUG']
72
76
 
73
77
  if @apply
74
78
  exec_query(query)
@@ -81,14 +85,14 @@ class Gamma::Importer::Replace < Gamma::Importer
81
85
  def insert_out_record(in_record)
82
86
  record = @table.record_value(in_record)
83
87
  columns = (@table.in_exist_columns & @table.out_exist_columns).reject { |c| record[c].nil? }
84
- select_columns = columns.map { |c| "`#{c}`" }.join(",")
88
+ select_columns = columns.map { |c| "`#{c}`" }.join(',')
85
89
  values = insert_record_values(record, columns)
86
90
 
87
91
  query = <<-EOS
88
92
  INSERT INTO #{@table.table_name}(#{select_columns}) VALUES (#{values})
89
93
  EOS
90
94
  query = query.strip_heredoc
91
- logger.info(query) if ENV["DEBUG"]
95
+ logger.info(query) if ENV['DEBUG']
92
96
 
93
97
  if @apply
94
98
  exec_query(query)
@@ -101,31 +105,31 @@ class Gamma::Importer::Replace < Gamma::Importer
101
105
  r = record
102
106
  columns.map do |v|
103
107
  c = if r[v].is_a?(Time)
104
- r[v].strftime("%Y-%m-%d %H:%M:%S")
108
+ r[v].strftime('%Y-%m-%d %H:%M:%S')
105
109
  else
106
110
  r[v]
107
111
  end
108
- "\"#{@out_client.client.escape(c.to_s)}\""
109
- end.join(",")
112
+ "\"#{@out_client.client.escape(c.to_s)}\""
113
+ end.join(',')
110
114
  end
111
115
 
112
116
  def update_record_values(record, columns)
113
117
  r = record
114
118
  columns.map do |v|
115
119
  c = if r[v].is_a?(Time)
116
- r[v].strftime("%Y-%m-%d %H:%M:%S")
120
+ r[v].strftime('%Y-%m-%d %H:%M:%S')
117
121
  else
118
122
  r[v]
119
123
  end
120
124
  "`#{v}` = \"#{@out_client.client.escape(c.to_s)}\""
121
- end.join(",")
125
+ end.join(',')
122
126
  end
123
127
 
124
128
  def exec_query(query)
125
129
  @out_client.client.query(query)
126
- rescue => e
130
+ rescue StandardError => e
127
131
  if @ignore_error
128
- logger.error("An ERROR has occurred and ignore it.")
132
+ logger.error('An ERROR has occurred and ignore it.')
129
133
  logger.error(e)
130
134
  else
131
135
  throw e
data/lib/gamma/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Gamma
2
- VERSION = "0.1.9"
2
+ VERSION = '0.2.0'
3
3
  end
metadata CHANGED
@@ -1,29 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gamma
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.9
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shinsuke Nishio
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-11-02 00:00:00.000000000 Z
11
+ date: 2022-07-22 00:00:00.000000000 Z
12
12
  dependencies:
13
- - !ruby/object:Gem::Dependency
14
- name: mysql2
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ">="
18
- - !ruby/object:Gem::Version
19
- version: '0'
20
- type: :runtime
21
- prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - ">="
25
- - !ruby/object:Gem::Version
26
- version: '0'
27
13
  - !ruby/object:Gem::Dependency
28
14
  name: activesupport
29
15
  requirement: !ruby/object:Gem::Requirement
@@ -39,61 +25,61 @@ dependencies:
39
25
  - !ruby/object:Gem::Version
40
26
  version: '0'
41
27
  - !ruby/object:Gem::Dependency
42
- name: thor
28
+ name: colorize
43
29
  requirement: !ruby/object:Gem::Requirement
44
30
  requirements:
45
31
  - - "~>"
46
32
  - !ruby/object:Gem::Version
47
- version: '0.20'
33
+ version: 0.8.1
48
34
  type: :runtime
49
35
  prerelease: false
50
36
  version_requirements: !ruby/object:Gem::Requirement
51
37
  requirements:
52
38
  - - "~>"
53
39
  - !ruby/object:Gem::Version
54
- version: '0.20'
40
+ version: 0.8.1
55
41
  - !ruby/object:Gem::Dependency
56
- name: colorize
42
+ name: mysql2
57
43
  requirement: !ruby/object:Gem::Requirement
58
44
  requirements:
59
- - - "~>"
45
+ - - ">="
60
46
  - !ruby/object:Gem::Version
61
- version: 0.8.1
47
+ version: '0'
62
48
  type: :runtime
63
49
  prerelease: false
64
50
  version_requirements: !ruby/object:Gem::Requirement
65
51
  requirements:
66
- - - "~>"
52
+ - - ">="
67
53
  - !ruby/object:Gem::Version
68
- version: 0.8.1
54
+ version: '0'
69
55
  - !ruby/object:Gem::Dependency
70
- name: bundler
56
+ name: parallel
71
57
  requirement: !ruby/object:Gem::Requirement
72
58
  requirements:
73
- - - "~>"
59
+ - - ">="
74
60
  - !ruby/object:Gem::Version
75
- version: '1.16'
76
- type: :development
61
+ version: '0'
62
+ type: :runtime
77
63
  prerelease: false
78
64
  version_requirements: !ruby/object:Gem::Requirement
79
65
  requirements:
80
- - - "~>"
66
+ - - ">="
81
67
  - !ruby/object:Gem::Version
82
- version: '1.16'
68
+ version: '0'
83
69
  - !ruby/object:Gem::Dependency
84
- name: rake
70
+ name: thor
85
71
  requirement: !ruby/object:Gem::Requirement
86
72
  requirements:
87
73
  - - "~>"
88
74
  - !ruby/object:Gem::Version
89
- version: '10.0'
90
- type: :development
75
+ version: '0.20'
76
+ type: :runtime
91
77
  prerelease: false
92
78
  version_requirements: !ruby/object:Gem::Requirement
93
79
  requirements:
94
80
  - - "~>"
95
81
  - !ruby/object:Gem::Version
96
- version: '10.0'
82
+ version: '0.20'
97
83
  - !ruby/object:Gem::Dependency
98
84
  name: minitest
99
85
  requirement: !ruby/object:Gem::Requirement
@@ -122,6 +108,20 @@ dependencies:
122
108
  - - ">="
123
109
  - !ruby/object:Gem::Version
124
110
  version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rake
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '10.0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '10.0'
125
125
  description: DBSync
126
126
  email:
127
127
  - nishio@densan-labs.net
@@ -164,7 +164,7 @@ homepage: https://github.com/nishio-dens/gamma
164
164
  licenses:
165
165
  - MIT
166
166
  metadata: {}
167
- post_install_message:
167
+ post_install_message:
168
168
  rdoc_options: []
169
169
  require_paths:
170
170
  - lib
@@ -179,9 +179,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
179
179
  - !ruby/object:Gem::Version
180
180
  version: '0'
181
181
  requirements: []
182
- rubyforge_project:
183
- rubygems_version: 2.6.8
184
- signing_key:
182
+ rubygems_version: 3.2.32
183
+ signing_key:
185
184
  specification_version: 4
186
185
  summary: DBSync
187
186
  test_files: []