flydata 0.5.5 → 0.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0882affb434c077a4d37f2127b7ca91aab356603
-  data.tar.gz: 83d402b1b492178d70a4bb6f71a2ada8c515e43e
+  metadata.gz: c8603d9b83a0d1f764610c72f23c822b8442322b
+  data.tar.gz: 653eb4641b6602f760156e8b8bd83168642e40a8
 SHA512:
-  metadata.gz: add50b1e166a42d338de4c386272e207bc4089b4ba746cdf52f961f8a344ecd0a676b6411bd0552239b6960d208d51bb9ac13f01448f85704a55a8805ec87104
-  data.tar.gz: 3028bb9b76675e98bf31eac4c6226ceb1abd001f019fa5702403632d29310b79b7e1a7b3f21a5e5bf71146ef3a778dfa62efe9db47c6f792f2552d505245bf44
+  metadata.gz: 0d3835c0914fcb12a951e58584a435528fc42f3aa7905420b02905ac9eca26354b122d2b82d45b8b83a8ef10a6b9b13ce3684b6336508aee8a207783fe2e693b
+  data.tar.gz: 6c0819ccc0c95efb01e6c6728c21e25d034ca7612ea02a2ed3671028c7c147065e1fd47f15f902c57cb5ce5efd39a47cadc3c5db30e07a736d94cd37e252bbbe

data/VERSION

@@ -1 +1 @@
-0.5.5
+0.5.6

data/flydata-core/lib/flydata-core/mysql/command_generator.rb

@@ -32,9 +32,11 @@ module FlydataCore
         database = option[:database]
         tables = option[:tables] ? option[:tables].join(' ') : nil
         ssl_ca = option[:ssl_ca] ? option[:ssl_ca] : nil
+        ssl_cipher = option[:ssl_cipher] ? option[:ssl_cipher] : nil
 
         default_option = option[:no_default_option] ? "" : DEFAULT_MYSQL_CMD_OPTION
         default_option += " --ssl-ca=#{ssl_ca}" if ssl_ca
+        default_option += " --ssl-cipher=#{ssl_cipher}" unless ssl_cipher.to_s.empty?
         default_option = nil if default_option == ''
 
         custom_option = option[:custom_option]

data/flydata-core/lib/flydata-core/mysql/config.rb

@@ -0,0 +1,32 @@
+module FlydataCore
+  module Mysql
+    class Config
+      def self.build_mysql_db_opts(db_conf)
+        db_opts = [:host,
+                   :port,
+                   :username,
+                   :password,
+                   :database,
+                   :ssl_ca,
+                   :ssl_cipher].inject({}) { |h, sym|
+                     h[sym] = if db_conf.has_key?(sym)
+                                db_conf[sym]
+                              else
+                                db_conf[sym.to_s]
+                              end
+                     h
+                   }
+
+        # for mysql2 gem
+        unless db_opts[:ssl_ca].to_s.empty?
+          db_opts[:sslca] = db_opts[:ssl_ca]
+        end
+        unless db_opts[:ssl_cipher].to_s.empty?
+          db_opts[:sslcipher] = db_opts[:ssl_cipher]
+        end
+
+        db_opts
+      end
+    end
+  end
+end

data/flydata-core/spec/mysql/command_generator_spec.rb

@@ -91,6 +91,13 @@ module FlydataCore
           ) }
         end
 
+        context 'when ssl_cipher is specified' do
+          before { option[:ssl_cipher] = 'AES256-GCM-SHA384:AES256-SHA' }
+          it { is_expected.to eq(
+            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp --ssl-cipher=AES256-GCM-SHA384:AES256-SHA testdb'
+          ) }
+        end
+
         context 'when no_default_option is turned on' do
           before { option[:no_default_option] = true }
           it { is_expected.to eq(

data/flydata-core/spec/mysql/config_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+require 'flydata-core/mysql/config'
+
+module FlydataCore
+  module Mysql
+    describe Config do
+      describe '.build_mysql_db_opts' do
+        let(:base_conf) do
+          {
+            'host' => 'localhost',
+            'port' => 1234,
+            'username' => 'testuser',
+            'password' => 'password',
+            'database' => 'testdb',
+          }
+        end
+        let(:conf) { base_conf }
+        subject { described_class.build_mysql_db_opts(conf) }
+
+        context 'with basic conf' do
+          it { is_expected.to eq(
+            host: 'localhost',
+            port: 1234,
+            username: 'testuser',
+            password: 'password',
+            database: 'testdb',
+            ssl_ca: nil,
+            ssl_cipher: nil,
+          ) }
+        end
+
+        context 'with conf having ssl_ca' do
+          context 'when ssl_ca is not empty' do
+            let(:conf) { base_conf.merge('ssl_ca' => 'test_path') }
+            it { is_expected.to eq(
+              host: 'localhost',
+              port: 1234,
+              username: 'testuser',
+              password: 'password',
+              database: 'testdb',
+              ssl_ca: 'test_path',
+              sslca: 'test_path',
+              ssl_cipher: nil,
+            ) }
+          end
+
+          context 'when ssl_ca is empty' do
+            let(:conf) { base_conf.merge('ssl_ca' => '') }
+            it { is_expected.to eq(
+              host: 'localhost',
+              port: 1234,
+              username: 'testuser',
+              password: 'password',
+              database: 'testdb',
+              ssl_ca: '',
+              ssl_cipher: nil,
+            ) }
+          end
+        end
+
+        context 'with conf having ssl_cipher' do
+          context 'when ssl_cipher is not empty' do
+            let(:conf) { base_conf.merge('ssl_cipher' => 'AAAA::BBBB') }
+            it { is_expected.to eq(
+              host: 'localhost',
+              port: 1234,
+              username: 'testuser',
+              password: 'password',
+              database: 'testdb',
+              ssl_ca: nil,
+              ssl_cipher: 'AAAA::BBBB',
+              sslcipher: 'AAAA::BBBB',
+            ) }
+          end
+
+          context 'when ssl_cipher is empty' do
+            let(:conf) { base_conf.merge('ssl_cipher' => '') }
+            it { is_expected.to eq(
+              host: 'localhost',
+              port: 1234,
+              username: 'testuser',
+              password: 'password',
+              database: 'testdb',
+              ssl_ca: nil,
+              ssl_cipher: '',
+            ) }
+          end
+        end
+      end
+    end
+  end
+end

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.5.5 ruby lib
+# stub: flydata 0.5.6 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.5.5"
+  s.version = "0.5.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-09-11"
+  s.date = "2015-09-22"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -49,6 +49,7 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/logger.rb",
     "flydata-core/lib/flydata-core/mysql/command_generator.rb",
     "flydata-core/lib/flydata-core/mysql/compatibility_checker.rb",
+    "flydata-core/lib/flydata-core/mysql/config.rb",
     "flydata-core/lib/flydata-core/option_validator.rb",
     "flydata-core/lib/flydata-core/query_job.rb",
     "flydata-core/lib/flydata-core/record/record.rb",
@@ -65,6 +66,7 @@ Gem::Specification.new do |s|
     "flydata-core/spec/logger_spec.rb",
     "flydata-core/spec/mysql/command_generator_spec.rb",
     "flydata-core/spec/mysql/compatibility_checker_spec.rb",
+    "flydata-core/spec/mysql/config_spec.rb",
     "flydata-core/spec/option_validator_spec.rb",
     "flydata-core/spec/redshift/string_spec.rb",
     "flydata-core/spec/spec_helper.rb",
@@ -226,7 +228,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.3"
+  s.rubygems_version = "2.4.6"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/sync.rb

@@ -950,8 +950,14 @@ Thank you for using FlyData!
         sync_resumed = options[:resume] && !!sync_info
 
         #full_tables will either include all tables including invalid tables or all valid tables that aren't new tables
+
+        # The 'new_tables' list may include tables which has already been
+        # synced for backward compatibility reason.
+        # Filter out such tables so that we get a list of 'new' tables with no
+        # position file.
+        real_new_tables = sync_fm.get_new_table_list(de['mysql_data_entry_preference']['new_tables'], "pos")
         @full_tables = options[:include_all_tables] ? de['mysql_data_entry_preference']['tables'] + de['mysql_data_entry_preference']['invalid_tables'] :
-                                                      de['mysql_data_entry_preference']['tables'] - de['mysql_data_entry_preference']['new_tables']
+                                                      de['mysql_data_entry_preference']['tables'] - real_new_tables
 
         @unsynced_tables = sync_fm.get_new_table_list(@full_tables, "pos") # Get list of tables that do not have a .pos file
         @ddl_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl")

data/lib/flydata/compatibility_check.rb

@@ -1,5 +1,6 @@
 require 'mysql2'
 require 'flydata/command_loggable'
+require 'flydata-core/mysql/config'
 require 'flydata-core/mysql/command_generator'
 require 'flydata-core/mysql/compatibility_checker'
 require 'flydata-core/errors'
@@ -72,8 +73,7 @@ module Flydata
 
     def initialize(dp_hash, de_hash, options={})
       super
-      @db_opts = [:host, :port, :username, :password, :database, :ssl_ca].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
-      @db_opts[:sslca] = @db_opts[:ssl_ca] # for mysql2 gem
+      @db_opts = FlydataCore::Mysql::Config.build_mysql_db_opts(de_hash)
       @dump_dir = options[:dump_dir] || nil
       @backup_dir = options[:backup_dir] || nil
       @tables = de_hash['tables']

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -94,7 +94,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     end
 
     # Db access opts
-    @db_opts = { host: @host, port: @port, username: @username, password: @password, database: @database, ssl_ca: @ssl_ca_path }
+    @db_opts = { host: @host, port: @port, username: @username, password: @password, database: @database, ssl_ca: @ssl_ca_path, ssl_cipher: @ssl_cipher }
 
     $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
     $log.info "mysql client version: #{`mysql -V`}"

data/lib/flydata/fluent-plugins/mysql/table_meta.rb

@@ -1,4 +1,5 @@
 require 'mysql2'
+require 'flydata-core/mysql/config'
 require 'flydata-core/table_def/mysql_table_def'
 
 module Flydata
@@ -29,9 +30,7 @@ EOT
       missing_opts = MANDATORY_OPTS - opts.keys
       raise "Mandatory option(s) are missing: #{missing_opts.join(', ')}" unless (missing_opts.empty?)
 
-      @db_opts = [:host, :port, :username, :password, :database, :ssl_ca].inject({}) {|h, sym| h[sym] = opts[sym]; h}
-      @db_opts[:sslca] = @db_opts[:ssl_ca] # for mysql2 gem
-
+      @db_opts = FlydataCore::Mysql::Config.build_mysql_db_opts(opts)
       @database = opts[:database]
       @tables = opts[:tables]
       @table_meta = Hash.new{|h, k| h[k] = {}}

data/lib/flydata/fluent-plugins/preference.rb

@@ -77,6 +77,7 @@ module Fluent
         username: {},
         password: {encrypted: true},
         ssl_ca_content: {},
+        ssl_cipher: {},
       },
     }
 

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -1,5 +1,6 @@
 require 'fiber'
 require 'io/wait'
+require 'flydata-core/mysql/config'
 require 'flydata-core/mysql/command_generator'
 
 module Flydata
@@ -8,7 +9,7 @@ module Flydata
 
       module MysqlAccessible
         def mysql_conf(conf)
-          @mysql_conf = [:host, :port, :username, :password, :database, :ssl_ca, :sslca].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
+          @mysql_conf = FlydataCore::Mysql::Config.build_mysql_db_opts(conf)
         end
 
         def mysql_cli(conf = nil)
@@ -43,7 +44,7 @@ module Flydata
       class MysqlDumpGenerator
         def initialize(conf)
           @conf = conf
-          @db_opts = [:host, :port, :username, :password, :database, :ssl_ca, :sslca].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
+          @db_opts = FlydataCore::Mysql::Config.build_mysql_db_opts(conf)
         end
 
         def dump(file_path)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.5.6
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-11 00:00:00.000000000 Z
+date: 2015-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -486,6 +486,7 @@ files:
 - flydata-core/lib/flydata-core/logger.rb
 - flydata-core/lib/flydata-core/mysql/command_generator.rb
 - flydata-core/lib/flydata-core/mysql/compatibility_checker.rb
+- flydata-core/lib/flydata-core/mysql/config.rb
 - flydata-core/lib/flydata-core/option_validator.rb
 - flydata-core/lib/flydata-core/query_job.rb
 - flydata-core/lib/flydata-core/record/record.rb
@@ -502,6 +503,7 @@ files:
 - flydata-core/spec/logger_spec.rb
 - flydata-core/spec/mysql/command_generator_spec.rb
 - flydata-core/spec/mysql/compatibility_checker_spec.rb
+- flydata-core/spec/mysql/config_spec.rb
 - flydata-core/spec/option_validator_spec.rb
 - flydata-core/spec/redshift/string_spec.rb
 - flydata-core/spec/spec_helper.rb
@@ -680,7 +682,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: FlyData Agent