flydata 0.5.5 → 0.5.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/mysql/command_generator.rb +2 -0
- data/flydata-core/lib/flydata-core/mysql/config.rb +32 -0
- data/flydata-core/spec/mysql/command_generator_spec.rb +7 -0
- data/flydata-core/spec/mysql/config_spec.rb +92 -0
- data/flydata.gemspec +6 -4
- data/lib/flydata/command/sync.rb +7 -1
- data/lib/flydata/compatibility_check.rb +2 -2
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +1 -1
- data/lib/flydata/fluent-plugins/mysql/table_meta.rb +2 -3
- data/lib/flydata/fluent-plugins/preference.rb +1 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +3 -2
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c8603d9b83a0d1f764610c72f23c822b8442322b
|
4
|
+
data.tar.gz: 653eb4641b6602f760156e8b8bd83168642e40a8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0d3835c0914fcb12a951e58584a435528fc42f3aa7905420b02905ac9eca26354b122d2b82d45b8b83a8ef10a6b9b13ce3684b6336508aee8a207783fe2e693b
|
7
|
+
data.tar.gz: 6c0819ccc0c95efb01e6c6728c21e25d034ca7612ea02a2ed3671028c7c147065e1fd47f15f902c57cb5ce5efd39a47cadc3c5db30e07a736d94cd37e252bbbe
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.6
|
@@ -32,9 +32,11 @@ module FlydataCore
|
|
32
32
|
database = option[:database]
|
33
33
|
tables = option[:tables] ? option[:tables].join(' ') : nil
|
34
34
|
ssl_ca = option[:ssl_ca] ? option[:ssl_ca] : nil
|
35
|
+
ssl_cipher = option[:ssl_cipher] ? option[:ssl_cipher] : nil
|
35
36
|
|
36
37
|
default_option = option[:no_default_option] ? "" : DEFAULT_MYSQL_CMD_OPTION
|
37
38
|
default_option += " --ssl-ca=#{ssl_ca}" if ssl_ca
|
39
|
+
default_option += " --ssl-cipher=#{ssl_cipher}" unless ssl_cipher.to_s.empty?
|
38
40
|
default_option = nil if default_option == ''
|
39
41
|
|
40
42
|
custom_option = option[:custom_option]
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module FlydataCore
|
2
|
+
module Mysql
|
3
|
+
class Config
|
4
|
+
def self.build_mysql_db_opts(db_conf)
|
5
|
+
db_opts = [:host,
|
6
|
+
:port,
|
7
|
+
:username,
|
8
|
+
:password,
|
9
|
+
:database,
|
10
|
+
:ssl_ca,
|
11
|
+
:ssl_cipher].inject({}) { |h, sym|
|
12
|
+
h[sym] = if db_conf.has_key?(sym)
|
13
|
+
db_conf[sym]
|
14
|
+
else
|
15
|
+
db_conf[sym.to_s]
|
16
|
+
end
|
17
|
+
h
|
18
|
+
}
|
19
|
+
|
20
|
+
# for mysql2 gem
|
21
|
+
unless db_opts[:ssl_ca].to_s.empty?
|
22
|
+
db_opts[:sslca] = db_opts[:ssl_ca]
|
23
|
+
end
|
24
|
+
unless db_opts[:ssl_cipher].to_s.empty?
|
25
|
+
db_opts[:sslcipher] = db_opts[:ssl_cipher]
|
26
|
+
end
|
27
|
+
|
28
|
+
db_opts
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -91,6 +91,13 @@ module FlydataCore
|
|
91
91
|
) }
|
92
92
|
end
|
93
93
|
|
94
|
+
context 'when ssl_cipher is specified' do
|
95
|
+
before { option[:ssl_cipher] = 'AES256-GCM-SHA384:AES256-SHA' }
|
96
|
+
it { is_expected.to eq(
|
97
|
+
'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp --ssl-cipher=AES256-GCM-SHA384:AES256-SHA testdb'
|
98
|
+
) }
|
99
|
+
end
|
100
|
+
|
94
101
|
context 'when no_default_option is turned on' do
|
95
102
|
before { option[:no_default_option] = true }
|
96
103
|
it { is_expected.to eq(
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata-core/mysql/config'
|
3
|
+
|
4
|
+
module FlydataCore
|
5
|
+
module Mysql
|
6
|
+
describe Config do
|
7
|
+
describe '.build_mysql_db_opts' do
|
8
|
+
let(:base_conf) do
|
9
|
+
{
|
10
|
+
'host' => 'localhost',
|
11
|
+
'port' => 1234,
|
12
|
+
'username' => 'testuser',
|
13
|
+
'password' => 'password',
|
14
|
+
'database' => 'testdb',
|
15
|
+
}
|
16
|
+
end
|
17
|
+
let(:conf) { base_conf }
|
18
|
+
subject { described_class.build_mysql_db_opts(conf) }
|
19
|
+
|
20
|
+
context 'with basic conf' do
|
21
|
+
it { is_expected.to eq(
|
22
|
+
host: 'localhost',
|
23
|
+
port: 1234,
|
24
|
+
username: 'testuser',
|
25
|
+
password: 'password',
|
26
|
+
database: 'testdb',
|
27
|
+
ssl_ca: nil,
|
28
|
+
ssl_cipher: nil,
|
29
|
+
) }
|
30
|
+
end
|
31
|
+
|
32
|
+
context 'with conf having ssl_ca' do
|
33
|
+
context 'when ssl_ca is not empty' do
|
34
|
+
let(:conf) { base_conf.merge('ssl_ca' => 'test_path') }
|
35
|
+
it { is_expected.to eq(
|
36
|
+
host: 'localhost',
|
37
|
+
port: 1234,
|
38
|
+
username: 'testuser',
|
39
|
+
password: 'password',
|
40
|
+
database: 'testdb',
|
41
|
+
ssl_ca: 'test_path',
|
42
|
+
sslca: 'test_path',
|
43
|
+
ssl_cipher: nil,
|
44
|
+
) }
|
45
|
+
end
|
46
|
+
|
47
|
+
context 'when ssl_ca is empty' do
|
48
|
+
let(:conf) { base_conf.merge('ssl_ca' => '') }
|
49
|
+
it { is_expected.to eq(
|
50
|
+
host: 'localhost',
|
51
|
+
port: 1234,
|
52
|
+
username: 'testuser',
|
53
|
+
password: 'password',
|
54
|
+
database: 'testdb',
|
55
|
+
ssl_ca: '',
|
56
|
+
ssl_cipher: nil,
|
57
|
+
) }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'with conf having ssl_cipher' do
|
62
|
+
context 'when ssl_cipher is not empty' do
|
63
|
+
let(:conf) { base_conf.merge('ssl_cipher' => 'AAAA::BBBB') }
|
64
|
+
it { is_expected.to eq(
|
65
|
+
host: 'localhost',
|
66
|
+
port: 1234,
|
67
|
+
username: 'testuser',
|
68
|
+
password: 'password',
|
69
|
+
database: 'testdb',
|
70
|
+
ssl_ca: nil,
|
71
|
+
ssl_cipher: 'AAAA::BBBB',
|
72
|
+
sslcipher: 'AAAA::BBBB',
|
73
|
+
) }
|
74
|
+
end
|
75
|
+
|
76
|
+
context 'when ssl_cipher is empty' do
|
77
|
+
let(:conf) { base_conf.merge('ssl_cipher' => '') }
|
78
|
+
it { is_expected.to eq(
|
79
|
+
host: 'localhost',
|
80
|
+
port: 1234,
|
81
|
+
username: 'testuser',
|
82
|
+
password: 'password',
|
83
|
+
database: 'testdb',
|
84
|
+
ssl_ca: nil,
|
85
|
+
ssl_cipher: '',
|
86
|
+
) }
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
data/flydata.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: flydata 0.5.
|
5
|
+
# stub: flydata 0.5.6 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "flydata"
|
9
|
-
s.version = "0.5.
|
9
|
+
s.version = "0.5.6"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
14
|
-
s.date = "2015-09-
|
14
|
+
s.date = "2015-09-22"
|
15
15
|
s.description = "FlyData Agent"
|
16
16
|
s.email = "sysadmin@flydata.com"
|
17
17
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
@@ -49,6 +49,7 @@ Gem::Specification.new do |s|
|
|
49
49
|
"flydata-core/lib/flydata-core/logger.rb",
|
50
50
|
"flydata-core/lib/flydata-core/mysql/command_generator.rb",
|
51
51
|
"flydata-core/lib/flydata-core/mysql/compatibility_checker.rb",
|
52
|
+
"flydata-core/lib/flydata-core/mysql/config.rb",
|
52
53
|
"flydata-core/lib/flydata-core/option_validator.rb",
|
53
54
|
"flydata-core/lib/flydata-core/query_job.rb",
|
54
55
|
"flydata-core/lib/flydata-core/record/record.rb",
|
@@ -65,6 +66,7 @@ Gem::Specification.new do |s|
|
|
65
66
|
"flydata-core/spec/logger_spec.rb",
|
66
67
|
"flydata-core/spec/mysql/command_generator_spec.rb",
|
67
68
|
"flydata-core/spec/mysql/compatibility_checker_spec.rb",
|
69
|
+
"flydata-core/spec/mysql/config_spec.rb",
|
68
70
|
"flydata-core/spec/option_validator_spec.rb",
|
69
71
|
"flydata-core/spec/redshift/string_spec.rb",
|
70
72
|
"flydata-core/spec/spec_helper.rb",
|
@@ -226,7 +228,7 @@ Gem::Specification.new do |s|
|
|
226
228
|
]
|
227
229
|
s.homepage = "http://flydata.com/"
|
228
230
|
s.licenses = ["All right reserved."]
|
229
|
-
s.rubygems_version = "2.4.
|
231
|
+
s.rubygems_version = "2.4.6"
|
230
232
|
s.summary = "FlyData Agent"
|
231
233
|
|
232
234
|
if s.respond_to? :specification_version then
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -950,8 +950,14 @@ Thank you for using FlyData!
|
|
950
950
|
sync_resumed = options[:resume] && !!sync_info
|
951
951
|
|
952
952
|
#full_tables will either include all tables including invalid tables or all valid tables that aren't new tables
|
953
|
+
|
954
|
+
# The 'new_tables' list may include tables which has already been
|
955
|
+
# synced for backward compatibility reason.
|
956
|
+
# Filter out such tables so that we get a list of 'new' tables with no
|
957
|
+
# position file.
|
958
|
+
real_new_tables = sync_fm.get_new_table_list(de['mysql_data_entry_preference']['new_tables'], "pos")
|
953
959
|
@full_tables = options[:include_all_tables] ? de['mysql_data_entry_preference']['tables'] + de['mysql_data_entry_preference']['invalid_tables'] :
|
954
|
-
de['mysql_data_entry_preference']['tables'] -
|
960
|
+
de['mysql_data_entry_preference']['tables'] - real_new_tables
|
955
961
|
|
956
962
|
@unsynced_tables = sync_fm.get_new_table_list(@full_tables, "pos") # Get list of tables that do not have a .pos file
|
957
963
|
@ddl_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl")
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'mysql2'
|
2
2
|
require 'flydata/command_loggable'
|
3
|
+
require 'flydata-core/mysql/config'
|
3
4
|
require 'flydata-core/mysql/command_generator'
|
4
5
|
require 'flydata-core/mysql/compatibility_checker'
|
5
6
|
require 'flydata-core/errors'
|
@@ -72,8 +73,7 @@ module Flydata
|
|
72
73
|
|
73
74
|
def initialize(dp_hash, de_hash, options={})
|
74
75
|
super
|
75
|
-
@db_opts =
|
76
|
-
@db_opts[:sslca] = @db_opts[:ssl_ca] # for mysql2 gem
|
76
|
+
@db_opts = FlydataCore::Mysql::Config.build_mysql_db_opts(de_hash)
|
77
77
|
@dump_dir = options[:dump_dir] || nil
|
78
78
|
@backup_dir = options[:backup_dir] || nil
|
79
79
|
@tables = de_hash['tables']
|
@@ -94,7 +94,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
94
94
|
end
|
95
95
|
|
96
96
|
# Db access opts
|
97
|
-
@db_opts = { host: @host, port: @port, username: @username, password: @password, database: @database, ssl_ca: @ssl_ca_path }
|
97
|
+
@db_opts = { host: @host, port: @port, username: @username, password: @password, database: @database, ssl_ca: @ssl_ca_path, ssl_cipher: @ssl_cipher }
|
98
98
|
|
99
99
|
$log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
|
100
100
|
$log.info "mysql client version: #{`mysql -V`}"
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'mysql2'
|
2
|
+
require 'flydata-core/mysql/config'
|
2
3
|
require 'flydata-core/table_def/mysql_table_def'
|
3
4
|
|
4
5
|
module Flydata
|
@@ -29,9 +30,7 @@ EOT
|
|
29
30
|
missing_opts = MANDATORY_OPTS - opts.keys
|
30
31
|
raise "Mandatory option(s) are missing: #{missing_opts.join(', ')}" unless (missing_opts.empty?)
|
31
32
|
|
32
|
-
@db_opts =
|
33
|
-
@db_opts[:sslca] = @db_opts[:ssl_ca] # for mysql2 gem
|
34
|
-
|
33
|
+
@db_opts = FlydataCore::Mysql::Config.build_mysql_db_opts(opts)
|
35
34
|
@database = opts[:database]
|
36
35
|
@tables = opts[:tables]
|
37
36
|
@table_meta = Hash.new{|h, k| h[k] = {}}
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'fiber'
|
2
2
|
require 'io/wait'
|
3
|
+
require 'flydata-core/mysql/config'
|
3
4
|
require 'flydata-core/mysql/command_generator'
|
4
5
|
|
5
6
|
module Flydata
|
@@ -8,7 +9,7 @@ module Flydata
|
|
8
9
|
|
9
10
|
module MysqlAccessible
|
10
11
|
def mysql_conf(conf)
|
11
|
-
@mysql_conf =
|
12
|
+
@mysql_conf = FlydataCore::Mysql::Config.build_mysql_db_opts(conf)
|
12
13
|
end
|
13
14
|
|
14
15
|
def mysql_cli(conf = nil)
|
@@ -43,7 +44,7 @@ module Flydata
|
|
43
44
|
class MysqlDumpGenerator
|
44
45
|
def initialize(conf)
|
45
46
|
@conf = conf
|
46
|
-
@db_opts =
|
47
|
+
@db_opts = FlydataCore::Mysql::Config.build_mysql_db_opts(conf)
|
47
48
|
end
|
48
49
|
|
49
50
|
def dump(file_path)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2015-09-
|
15
|
+
date: 2015-09-22 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -486,6 +486,7 @@ files:
|
|
486
486
|
- flydata-core/lib/flydata-core/logger.rb
|
487
487
|
- flydata-core/lib/flydata-core/mysql/command_generator.rb
|
488
488
|
- flydata-core/lib/flydata-core/mysql/compatibility_checker.rb
|
489
|
+
- flydata-core/lib/flydata-core/mysql/config.rb
|
489
490
|
- flydata-core/lib/flydata-core/option_validator.rb
|
490
491
|
- flydata-core/lib/flydata-core/query_job.rb
|
491
492
|
- flydata-core/lib/flydata-core/record/record.rb
|
@@ -502,6 +503,7 @@ files:
|
|
502
503
|
- flydata-core/spec/logger_spec.rb
|
503
504
|
- flydata-core/spec/mysql/command_generator_spec.rb
|
504
505
|
- flydata-core/spec/mysql/compatibility_checker_spec.rb
|
506
|
+
- flydata-core/spec/mysql/config_spec.rb
|
505
507
|
- flydata-core/spec/option_validator_spec.rb
|
506
508
|
- flydata-core/spec/redshift/string_spec.rb
|
507
509
|
- flydata-core/spec/spec_helper.rb
|
@@ -680,7 +682,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
680
682
|
version: '0'
|
681
683
|
requirements: []
|
682
684
|
rubyforge_project:
|
683
|
-
rubygems_version: 2.4.
|
685
|
+
rubygems_version: 2.4.6
|
684
686
|
signing_key:
|
685
687
|
specification_version: 4
|
686
688
|
summary: FlyData Agent
|