flydata 0.4.0 → 0.4.1

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -9,7 +9,7 @@ lib = File.expand_path(File.join(File.dirname(__FILE__), '../../'))
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'flydata'
 require 'flydata/sync_file_manager'
-require 'flydata/mysql/mysql_util'
+require 'flydata-core/mysql/command_generator'
 require 'flydata/fluent-plugins/preference'
 require 'flydata/fluent-plugins/mysql/binlog_position_file'
 require 'flydata/fluent-plugins/mysql/binlog_record_dispatcher'
@@ -53,8 +53,8 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
   config_param :tables, :string
   config_param :tables_append_only, :string
   config_param :initial_idle_interval, :integer, :default => 30     # 30 sec
-  config_param :continuous_idle_interval, :integer, :default => 600 # 600 sec
-  config_param :idle_timeout, :time, :default => 3600               # 3600 sec
+  config_param :continuous_idle_interval, :integer, :default => 600 # 10 min
+  config_param :idle_timeout, :time, :default => 1800               # 30 min
   config_param :check_interval, :integer, :default => 5             # 5 sec
   config_param :ssl_ca_content, :string, :default => ''
 
@@ -81,7 +81,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
     $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
     $log.info "mysql client version: #{`mysql -V`}"
-    server_msg = `echo 'select version();' | #{Flydata::Mysql::MysqlUtil.generate_mysql_cmd(@db_opts)} 2>&1`
+    server_msg = `echo 'select version();' | #{FlydataCore::Mysql::CommandGenerator.generate_mysql_cmd(@db_opts)} 2>&1`
     server_msg = server_msg.each_line.select{|l| l.start_with?('ERROR ')}.join("\n")
     if ($?.exitstatus == 0)
       $log.info "mysql server version: #{server_msg.strip}"

data/lib/flydata/mysql/table_ddl.rb

@@ -1,5 +1,5 @@
 require 'flydata/mysql/binlog_position'
-require 'flydata/mysql/mysql_util'
+require 'flydata-core/mysql/command_generator'
 
 module Flydata
 module Mysql
@@ -50,7 +50,7 @@ EOS
       next if version >= V2_TARGET_VERSION
 
       if mysql_tabledefs.nil?
-        MysqlUtil.each_mysql_tabledef(tables, mysql_opts) do |mysql_tabledef, error|
+        FlydataCore::Mysql::CommandGenerator.each_mysql_tabledef(tables, mysql_opts) do |mysql_tabledef, error|
           raise error if error
 
           mysql_tabledefs ||= {}

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -1,6 +1,6 @@
 require 'fiber'
 require 'io/wait'
-require 'flydata/mysql/mysql_util'
+require 'flydata-core/mysql/command_generator'
 
 module Flydata
   module Parser
@@ -141,7 +141,7 @@ module Flydata
         end
 
         def generate_dump_cmd(conf, file_path = nil)
-          Flydata::Mysql::MysqlUtil.generate_mysqldump_without_master_data_cmd(conf.merge(result_file: file_path))
+          FlydataCore::Mysql::CommandGenerator.generate_mysqldump_without_master_data_cmd(conf.merge(result_file: file_path))
         end
 
         private

data/spec/flydata/command/sync_spec.rb

@@ -33,7 +33,8 @@ module Flydata
         "data_port_key"=>"a458c641",
         "mysql_data_entry_preference" =>
           { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
-            "password"=>"welcome", "database"=>"sync_test", "tables"=>["table1", " table2"],
+            "password"=>"welcome", "database"=>"sync_test", "tables"=>["table1", "table2"],
+            "invalid_tables"=>["table3"],
             "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
             "data_servers"=>"localhost:9905" }
          }
@@ -136,7 +137,8 @@ module Flydata
       describe '#do_generate_table_ddl' do
         before do
           allow(subject).to receive(:data_entry).and_return(default_data_entry)
-          subject.send(:set_current_tables)
+          allow_any_instance_of(Flydata::Api::DataEntry).to receive(:update_table_validity).and_return(true)
+          subject.send(:set_current_tables, nil, include_invalid_tables: true)
         end
         shared_examples 'throws an error' do
           it "throws an error" do
@@ -148,7 +150,7 @@ module Flydata
         context 'with full options' do
           it 'issues mysqldump command with expected parameters' do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 3306 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
+             'mysqldump -h localhost -P 3306 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table3')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -170,7 +172,7 @@ module Flydata
           end
           it "uses the default port" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
+             'mysqldump -h localhost -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table3')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -180,7 +182,7 @@ module Flydata
           end
           it "uses the specified port" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 1234 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
+             'mysqldump -h localhost -P 1234 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table3')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -202,7 +204,7 @@ module Flydata
           end
           it "call mysqldump without MYSQL_PW set" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 3306 -umasashi --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
+             'mysqldump -h localhost -P 3306 -umasashi --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table3')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -213,7 +215,7 @@ module Flydata
           end
           it "call mysqldump with MYSQL_PW set with correct symbols" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 3306 -umasashi -p"welcome&!@^@#^" --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
+             'mysqldump -h localhost -P 3306 -umasashi -p"welcome&!@^@#^" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table3')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -229,17 +231,17 @@ module Flydata
           end
           include_examples 'throws an error'
         end
-        context 'without_tables' do
-          before do
-            default_data_entry['mysql_data_entry_preference'].delete('tables')
-          end
-          include_examples 'throws an error'
-        end
         context 'with empty tables' do
+          let(:sync_cmd) { described_class.new }
           before do
-            default_data_entry['mysql_data_entry_preference']['tables'] = ""
+            default_data_entry['mysql_data_entry_preference']['tables'] = []
+            default_data_entry['mysql_data_entry_preference']['invalid_tables'] = []
+            allow(sync_cmd).to receive(:data_entry).and_return(default_data_entry)
+            sync_cmd.send(:set_current_tables, nil, include_invalid_tables: true)
+          end
+          it 'should raise error' do
+            expect{sync_cmd.send(:do_generate_table_ddl, default_data_entry)}.to raise_error
           end
-          include_examples 'throws an error'
         end
       end
       describe '#convert_to_flydata_values' do

data/spec/flydata/compatibility_check_spec.rb

@@ -27,7 +27,7 @@ module Flydata
             allow(TCPSocket).to receive(:new).and_raise(Errno::ETIMEDOUT)
           end
           it do
-            expect{subject.check_outgoing_ports}.to raise_error(Flydata::AgentCompatibilityCheck::AgentCompatibilityError, /ports/)
+            expect{subject.check_outgoing_ports}.to raise_error(FlydataCore::AgentCompatibilityError, /ports/)
           end
         end
       end
@@ -73,7 +73,7 @@ module Flydata
                                                        {"Grants for test"=>"GRANT SELECT, RELOAD, REPLICATION CLIENT ON `test_db`.* TO 'test'@'host"}])
         end
         it do
-          expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /test_db': LOCK TABLES, REPLICATION SLAVE/)
+          expect{subject.check_mysql_user_compat}.to raise_error(FlydataCore::MysqlCompatibilityError, /test_db': LOCK TABLES, REPLICATION SLAVE/)
         end
       end
       context "with all required privileges in between all and specific databases" do
@@ -93,7 +93,7 @@ module Flydata
                                                        {"Grants for test"=>"GRANT SELECT, LOCK TABLES ON `mysql`.* TO 'test'@'host"}])
         end
         it do
-          expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /mysql': RELOAD\n.*test_db': RELOAD/)
+          expect{subject.check_mysql_user_compat}.to raise_error(FlydataCore::MysqlCompatibilityError, /mysql': RELOAD\n.*test_db': RELOAD/)
         end
       end
       context "with all required privileges in all databases" do
@@ -109,7 +109,7 @@ module Flydata
           allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT RELOAD, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'test'@'host"}])
         end
         it do
-          expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /mysql': SELECT, LOCK TABLES\n.*test_db': SELECT, LOCK TABLES/)
+          expect{subject.check_mysql_user_compat}.to raise_error(FlydataCore::MysqlCompatibilityError, /mysql': SELECT, LOCK TABLES\n.*test_db': SELECT, LOCK TABLES/)
         end
       end
       context "with privileges for other databases only" do
@@ -119,7 +119,7 @@ module Flydata
                                                        {"Grants for test"=>"GRANT SELECT ON `text_db_03`.* TO 'test'@'host"}])
         end
         it do
-          expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /mysql': SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT\n.*test_db': SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT/)
+          expect{subject.check_mysql_user_compat}.to raise_error(FlydataCore::MysqlCompatibilityError, /mysql': SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT\n.*test_db': SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT/)
         end
       end
 
@@ -132,19 +132,19 @@ module Flydata
           let(:client) { double('client') }
           before do
             allow(Mysql2::Client).to receive(:new).and_return(client)
-            allow(client).to receive(:query).and_return([{"@@expire_logs_days"=>1}])
+            allow(client).to receive(:query).and_return([{"Variable_name" => "expire_logs_days", "Value" => 1}])
             allow(client).to receive(:close)
             allow(subject).to receive(:is_rds?).and_return(false)
           end
           it do
-            expect{subject.check_mysql_binlog_retention}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /expire_logs_days/)
+            expect{subject.check_mysql_binlog_retention}.to raise_error(FlydataCore::MysqlCompatibilityError, /expire_logs_days/)
           end
         end
         context "where retention is 0" do
           let(:client) { double('client') }
           before do
             allow(Mysql2::Client).to receive(:new).and_return(client)
-            allow(client).to receive(:query).and_return([{"@@expire_logs_days"=>0}])
+            allow(client).to receive(:query).and_return([{"Variable_name" => "expire_logs_days", "Value" => 0}])
             allow(client).to receive(:close)
             allow(subject).to receive(:is_rds?).and_return(false)
           end
@@ -156,7 +156,8 @@ module Flydata
           let(:client) { double('client') }
           before do
             allow(Mysql2::Client).to receive(:new).and_return(client)
-            allow(client).to receive(:query).and_return([{"@@expire_logs_days"=>11}])
+            allow(client).to receive(:query).and_return([{"expire_logs_days"=>11}])
+            allow(client).to receive(:query).and_return([{"Variable_name" => "expire_logs_days", "Value" => 11}])
             allow(client).to receive(:close)
             allow(subject).to receive(:is_rds?).and_return(false)
           end
@@ -177,7 +178,7 @@ module Flydata
             allow(subject).to receive(:is_rds?).and_return(true)
           end
           it do
-            expect{subject.check_mysql_binlog_retention}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /rds_set_config/)
+            expect{subject.check_mysql_binlog_retention}.to raise_error(FlydataCore::MysqlCompatibilityError, /rds_set_config/)
           end
         end
         context "where retention period is too low" do
@@ -190,7 +191,7 @@ module Flydata
             allow(subject).to receive(:is_rds?).and_return(true)
           end
           it do
-            expect{subject.check_mysql_binlog_retention}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /rds_set_config/)
+            expect{subject.check_mysql_binlog_retention}.to raise_error(FlydataCore::MysqlCompatibilityError, /rds_set_config/)
           end
         end
         context "where retention period is over recommended limit" do
@@ -236,7 +237,7 @@ module Flydata
       let(:view) { {"table_name"=>"view_table", "table_type"=>"VIEW", "engine"=>nil} }
       let(:client) { double('client') }
       let(:subject_object) { MysqlCompatibilityCheck.new(default_data_port,test_data_entry, {}) }
-      let(:error) { Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError } 
+      let(:error) { FlydataCore::MysqlCompatibilityError }
       let(:base_error_msg) { "FlyData does not support VIEW and MEMORY ENGINE table. Remove following tables from data entry: %s" }
       subject { subject_object.check_mysql_table_types }
       before do

data/spec/flydata/mysql/table_ddl_spec.rb

@@ -80,7 +80,7 @@ describe TableDdl do
       it do
         matcher = receive(:each_mysql_tabledef).with(table_names, mysql_opts)
         matcher = table_names.inject(matcher) {|m, tn| m.and_yield(mysql_tabledef[tn], nil) }
-        expect(MysqlUtil).to matcher
+        expect(FlydataCore::Mysql::CommandGenerator).to matcher
         expect(File).to receive(:open).with(position_file).and_return(master_binlog_position)
         expect(context).to receive(:current_binlog_file).and_return(original_current_binlog_file)
         expect(context).to receive(:current_binlog_file=).with(binlog_file)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-17 00:00:00.000000000 Z
+date: 2015-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -248,7 +248,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: 0.1.6
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.2
@@ -258,7 +258,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: 0.1.6
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.2
@@ -484,6 +484,10 @@ files:
 - flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb
 - flydata-core/lib/flydata-core/fluent/config_helper.rb
 - flydata-core/lib/flydata-core/logger.rb
+- flydata-core/lib/flydata-core/mysql/command_generator.rb
+- flydata-core/lib/flydata-core/mysql/compatibility_checker.rb
+- flydata-core/lib/flydata-core/query_job.rb
+- flydata-core/lib/flydata-core/query_job/redshift.rb
 - flydata-core/lib/flydata-core/record/record.rb
 - flydata-core/lib/flydata-core/redshift/string.rb
 - flydata-core/lib/flydata-core/table_def.rb
@@ -495,6 +499,9 @@ files:
 - flydata-core/spec/config/user_maintenance_spec.rb
 - flydata-core/spec/fluent/config_helper_spec.rb
 - flydata-core/spec/logger_spec.rb
+- flydata-core/spec/mysql/command_generator_spec.rb
+- flydata-core/spec/mysql/compatibility_checker.rb
+- flydata-core/spec/query_job/redshift_spec.rb
 - flydata-core/spec/redshift/string_spec.rb
 - flydata-core/spec/spec_helper.rb
 - flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb
@@ -582,7 +589,6 @@ files:
 - lib/flydata/heroku/instance_methods.rb
 - lib/flydata/log_monitor.rb
 - lib/flydata/mysql/binlog_position.rb
-- lib/flydata/mysql/mysql_util.rb
 - lib/flydata/mysql/table_ddl.rb
 - lib/flydata/output/forwarder.rb
 - lib/flydata/parser/mysql/dump_parser.rb
@@ -638,7 +644,6 @@ files:
 - spec/flydata/helper/worker_spec.rb
 - spec/flydata/heroku_spec.rb
 - spec/flydata/mysql/binlog_position_spec.rb
-- spec/flydata/mysql/mysql_util_spec.rb
 - spec/flydata/mysql/table_ddl_spec.rb
 - spec/flydata/output/forwarder_spec.rb
 - spec/flydata/parser/mysql/alter_table_parser_spec.rb
@@ -670,7 +675,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: FlyData Agent