naginegi 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/naginegi.rb +9 -9
- data/lib/naginegi/bigquery.rb +3 -1
- data/lib/naginegi/embulk.rb +5 -2
- data/lib/naginegi/embulk_config.rb +25 -1
- data/lib/naginegi/mysql.rb +0 -24
- data/lib/naginegi/postgresql.rb +0 -24
- data/lib/naginegi/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2a23dc48ebec8810113d0b5a3a085c106048bb8aa24cfffc25d7e9c420ff5504
|
4
|
+
data.tar.gz: 3cdfdf331f116492d7457072fc16f93bb6ab88fb7b8c8efdfb286f745f8dd11d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4af85f8bdad42f90f06478954ccda166f35028a4660b11b08dbfe7a3ef94a9e8c8652212673549d1d2bac485cebd4dd05102cf021328a63fb67ccf97e6e4dfef
|
7
|
+
data.tar.gz: 803c5fe510fccc0e98dd9fec3294a576f58c2f25c3df531b4f53e1bad42e90fd68754bca8b5076dd6894c919112577ff3c5101de16cb6c68592ce59de3e42daa
|
data/README.md
CHANGED
data/lib/naginegi.rb
CHANGED
@@ -7,13 +7,17 @@ require 'logger'
|
|
7
7
|
|
8
8
|
module Naginegi
|
9
9
|
class EmbulkRunner
|
10
|
-
def initialize
|
10
|
+
def initialize(db_configs: nil, log_level: 'warn', embulk_run_option: '')
|
11
11
|
@logger = Logger.new(STDOUT)
|
12
12
|
@logger.datetime_format = '%Y-%m-%d %H:%M:%S'
|
13
|
+
|
14
|
+
@db_configs = db_configs || YAML.load_file('database.yml')
|
15
|
+
@log_level = log_level
|
16
|
+
@embulk_run_option = embulk_run_option
|
13
17
|
end
|
14
18
|
|
15
19
|
def generate_config(bq_config)
|
16
|
-
Naginegi::EmbulkConfig.new.generate_config(db_configs, bq_config)
|
20
|
+
Naginegi::EmbulkConfig.new.generate_config(@db_configs, bq_config)
|
17
21
|
end
|
18
22
|
|
19
23
|
def run(bq_config, target_table_names = [], retry_max = 0)
|
@@ -31,8 +35,8 @@ module Naginegi
|
|
31
35
|
private
|
32
36
|
|
33
37
|
def run_and_retry(bq_config, target_table_names, retry_max, retry_count)
|
34
|
-
error_tables = Naginegi::Embulk.new.run(
|
35
|
-
db_configs,
|
38
|
+
error_tables = Naginegi::Embulk.new(@log_level, @embulk_run_option).run(
|
39
|
+
@db_configs,
|
36
40
|
table_configs,
|
37
41
|
bq_config,
|
38
42
|
target_table_names
|
@@ -46,12 +50,8 @@ module Naginegi
|
|
46
50
|
error_tables
|
47
51
|
end
|
48
52
|
|
49
|
-
def db_configs
|
50
|
-
@db_configs ||= YAML.load_file('database.yml')
|
51
|
-
end
|
52
|
-
|
53
53
|
def table_configs
|
54
|
-
@table_configs ||= Naginegi::
|
54
|
+
@table_configs ||= Naginegi::TableConfig.generate_table_configs
|
55
55
|
end
|
56
56
|
end
|
57
57
|
end
|
data/lib/naginegi/bigquery.rb
CHANGED
@@ -9,10 +9,11 @@ module Naginegi
|
|
9
9
|
CONTENTS = <<-EOS.unindent
|
10
10
|
in:
|
11
11
|
type: <%= db_type %>
|
12
|
+
host: <%= host %>
|
12
13
|
user: <%= user %>
|
13
14
|
password: <%= password %>
|
14
15
|
database: <%= database %>
|
15
|
-
|
16
|
+
ssl: <%= ssl %>
|
16
17
|
query: |
|
17
18
|
<%= query %>
|
18
19
|
<%= options %>
|
@@ -61,6 +62,7 @@ module Naginegi
|
|
61
62
|
user = db_config['username']
|
62
63
|
password = db_config['password']
|
63
64
|
database = db_config['database']
|
65
|
+
ssl = db_config['embulk_ssl_enable'] || false
|
64
66
|
options = if db_type == 'mysql'
|
65
67
|
"options: {useLegacyDatetimeCode: false, serverTimezone: #{db_config['timezone']}}"
|
66
68
|
else
|
data/lib/naginegi/embulk.rb
CHANGED
@@ -2,9 +2,12 @@ require 'logger'
|
|
2
2
|
|
3
3
|
module Naginegi
|
4
4
|
class Embulk
|
5
|
-
def initialize
|
5
|
+
def initialize(log_level, embulk_run_option)
|
6
6
|
@logger = Logger.new(STDOUT)
|
7
7
|
@logger.datetime_format = '%Y-%m-%d %H:%M:%S'
|
8
|
+
|
9
|
+
@log_level = log_level
|
10
|
+
@embulk_run_option = embulk_run_option
|
8
11
|
end
|
9
12
|
|
10
13
|
def run(db_configs, all_table_configs, bq_config, target_table_names = [])
|
@@ -45,7 +48,7 @@ module Naginegi
|
|
45
48
|
@logger.warn(e.message)
|
46
49
|
end
|
47
50
|
|
48
|
-
cmd = "embulk run #{bq_config['config_dir']}/#{db_name}/#{table_config.name}.yml"
|
51
|
+
cmd = "embulk run #{@embulk_run_option} #{bq_config['config_dir']}/#{db_name}/#{table_config.name}.yml --log-level #{@log_level}"
|
49
52
|
@logger.info("cmd: #{cmd}")
|
50
53
|
|
51
54
|
if system(cmd)
|
@@ -43,7 +43,31 @@ module Naginegi
|
|
43
43
|
end
|
44
44
|
|
45
45
|
def all_table_configs
|
46
|
-
@all_table_configs ||=
|
46
|
+
@all_table_configs ||= Naginegi::TableConfig.generate_table_configs
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
class TableConfig
|
51
|
+
attr_reader :name, :daily_snapshot, :condition
|
52
|
+
|
53
|
+
def initialize(config)
|
54
|
+
@name = config['name']
|
55
|
+
@daily_snapshot = config['daily_snapshot'] || false
|
56
|
+
@condition = config['condition']
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.generate_table_configs(file_path = 'table.yml')
|
60
|
+
configs = YAML.load_file(file_path)
|
61
|
+
configs.each_with_object({}) do |(db, database_config), table_configs|
|
62
|
+
table_configs[db] = database_config['tables'].map { |config| TableConfig.new(config) }
|
63
|
+
table_configs
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def ==(other)
|
68
|
+
instance_variables.all? do |v|
|
69
|
+
instance_variable_get(v) == other.instance_variable_get(v)
|
70
|
+
end
|
47
71
|
end
|
48
72
|
end
|
49
73
|
end
|
data/lib/naginegi/mysql.rb
CHANGED
@@ -39,30 +39,6 @@ module Naginegi
|
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
-
class TableConfig
|
43
|
-
attr_reader :name, :daily_snapshot, :condition
|
44
|
-
|
45
|
-
def initialize(config)
|
46
|
-
@name = config['name']
|
47
|
-
@daily_snapshot = config['daily_snapshot'] || false
|
48
|
-
@condition = config['condition']
|
49
|
-
end
|
50
|
-
|
51
|
-
def self.generate_table_configs(file_path = 'table.yml')
|
52
|
-
configs = YAML.load_file(file_path)
|
53
|
-
configs.each_with_object({}) do |(db, database_config), table_configs|
|
54
|
-
table_configs[db] = database_config['tables'].map { |config| TableConfig.new(config) }
|
55
|
-
table_configs
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
def ==(other)
|
60
|
-
instance_variables.all? do |v|
|
61
|
-
instance_variable_get(v) == other.instance_variable_get(v)
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
42
|
class Column
|
67
43
|
attr_reader :column_name, :data_type
|
68
44
|
|
data/lib/naginegi/postgresql.rb
CHANGED
@@ -38,30 +38,6 @@ module Naginegi
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
|
41
|
-
class TableConfig
|
42
|
-
attr_reader :name, :daily_snapshot, :condition
|
43
|
-
|
44
|
-
def initialize(config)
|
45
|
-
@name = config['name']
|
46
|
-
@daily_snapshot = config['daily_snapshot'] || false
|
47
|
-
@condition = config['condition']
|
48
|
-
end
|
49
|
-
|
50
|
-
def self.generate_table_configs(file_path = 'table.yml')
|
51
|
-
configs = YAML.load_file(file_path)
|
52
|
-
configs.each_with_object({}) do |(db, db_config), table_configs|
|
53
|
-
table_configs[db] = db_config['tables'].map { |config| TableConfig.new(config) }
|
54
|
-
table_configs
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def ==(other)
|
59
|
-
instance_variables.all? do |v|
|
60
|
-
instance_variable_get(v) == other.instance_variable_get(v)
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
41
|
class Column
|
66
42
|
attr_reader :column_name, :data_type
|
67
43
|
|
data/lib/naginegi/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: naginegi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- cobot00
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-01-
|
11
|
+
date: 2020-01-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|