flydata 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0bbdc5f5871df8b3fec4c8c92abd1fdb6b6b24f6
-  data.tar.gz: 0be09c4771db44cac9c3229251598f6f5c3c23fd
+  metadata.gz: 3363c696e28b167255482e1d6b7352d5e3989786
+  data.tar.gz: 57d1039674d07396d58b2cf32d25c5c38db703ca
 SHA512:
-  metadata.gz: 3bdacc04e29100c441e7e1eb1660eb9763e2566f0ac05846a7cc85ac2067e4f0171c41d95e4a12ae5fa7b9c3720b5583040191a68b669b392e360fa58be79abb
-  data.tar.gz: 932a0de391228f576f389c731f312108931be9473875fe8265cc982d58771a6ea20671a762eeaf7b30a5aefbdfa0d60bb4572f409635f757e2bd900fad65e966
+  metadata.gz: 24e8879e214ec3b0cfa21d7b725e05bca66b4c668f59d3bfc5869285e70d1a92f97971002566f9d7e65332d91897561a20ffab26916ee7ce35c163e25066546a
+  data.tar.gz: 6303e50004f8d927867892a94e309d8e578cee1ed633f2768bae4470aca808cd9945ff2694ac220a07d4e20bea35df2df194963475ee971417910c22bdc6914b

data/Gemfile

@@ -13,7 +13,7 @@ gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
 gem "io-console", '~> 0.4.2', '>= 0.4.2'
-gem "kodama", '~> 0.1.2', '>= 0.1.6'
+gem "kodama", '~> 0.1.2', '>= 0.1.7'
 gem "serverengine", '~> 1.5'
 
 group :development do

data/Gemfile.lock

@@ -61,8 +61,8 @@ GEM
       rdoc
     json (1.8.1)
     jwt (1.0.0)
-    kodama (0.1.6)
-      ruby-binlog (~> 1.0, >= 1.0.4)
+    kodama (0.1.7)
+      ruby-binlog (~> 1.0, >= 1.0.6)
     method_source (0.8.2)
     mime-types (2.3)
     minitest (4.7.5)
@@ -105,7 +105,7 @@ GEM
     rspec-mocks (3.0.3)
       rspec-support (~> 3.0.0)
     rspec-support (3.0.3)
-    ruby-binlog (1.0.4)
+    ruby-binlog (1.0.6)
     ruby-prof (0.15.1)
     serverengine (1.5.10)
       sigdump (~> 0.2.2)
@@ -135,7 +135,7 @@ DEPENDENCIES
   io-console (~> 0.4.2, >= 0.4.2)
   jeweler (~> 1.8, >= 1.8.8)
   json (~> 1.8, >= 1.8.0)
-  kodama (~> 0.1.2, >= 0.1.6)
+  kodama (~> 0.1.2, >= 0.1.7)
   mysql2 (~> 0.3, >= 0.3.17)
   protected_attributes (~> 1.0, >= 1.0.8)
   pry

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.5.1

data/flydata-core/.gitignore

@@ -14,3 +14,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+/vendor/

data/flydata-core/Gemfile

@@ -1,11 +1,14 @@
 source 'https://rubygems.org'
 
+
 # for dependencies
+gem 'aws-sdk', "1.18.0"
+
 group :development, :test do
    gem 'timecop'
    gem 'ruby-prof'
    gem 'rubocop'
+   gem 'mysql2'
    gem 'rspec', '~> 3.1'
    gem 'simplecov', :require => false
 end
-

data/flydata-core/Gemfile.lock

@@ -4,9 +4,16 @@ GEM
     ast (2.0.0)
     astrolabe (1.3.0)
       parser (>= 2.2.0.pre.3, < 3.0)
+    aws-sdk (1.18.0)
+      json (~> 1.4)
+      nokogiri (< 1.6.0)
+      uuidtools (~> 2.1)
     diff-lcs (1.2.5)
     docile (1.1.5)
+    json (1.8.2)
     multi_json (1.10.1)
+    mysql2 (0.3.18)
+    nokogiri (1.5.11)
     parser (2.2.0.pre.8)
       ast (>= 1.1, < 3.0)
       slop (~> 3.4, >= 3.4.5)
@@ -39,11 +46,14 @@ GEM
     simplecov-html (0.8.0)
     slop (3.6.0)
     timecop (0.7.1)
+    uuidtools (2.1.5)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  aws-sdk (= 1.18.0)
+  mysql2
   rspec (~> 3.1)
   rubocop
   ruby-prof

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -118,34 +118,57 @@ module FlydataCore
         'binlog_format'=>'ROW',
         'binlog_checksum'=>'NONE',
         'log_bin_use_v1_row_events'=>'ON',
-        'log_slave_updates'=>'ON'
+        'log_slave_updates'=>'ON',
+        # parameter => {"comparison sign"=>minimum value}
+        # TODO: Handled the error message of following items as warning
+        #'net_read_timeout'=>{'>'=>600},
+        #'net_write_timeout'=>{'>'=>600},
+        #'wait_timeout'=>{'>'=>60}
       }
+      OPTIONAL_SYS_VAR = %w(binlog_checksum log_bin_use_v1_row_events)
 
       def check_result(result, option = @option)
         errors = {}
         param_hash = convert_result_to_hash(result)
 
-        SYS_VAR_TO_CHECK.each_key do |sys_var|
-          if param_hash.has_key?(sys_var)
-            actual_val = param_hash[sys_var]
-            expected_val = SYS_VAR_TO_CHECK[sys_var]
-            unless actual_val == expected_val
-              errors[sys_var] = actual_val
-            end
-          elsif not %w(binlog_checksum log_bin_use_v1_row_events).include?(sys_var)
-            # Mark variables that do not exist as error
-            errors[sys_var] = false
-          end
-        end
+        compare_sys_var_values(param_hash, SYS_VAR_TO_CHECK, errors, OPTIONAL_SYS_VAR)
 
         unless errors.empty?
           error_explanation = ""
           errors.each_key do |err_key|
-            error_explanation << "\n    * #{err_key} is #{errors[err_key]} but should be #{SYS_VAR_TO_CHECK[err_key]}"
+            if SYS_VAR_TO_CHECK[err_key].is_a?(Hash)
+              error_explanation << "\n    * #{err_key} is #{errors[err_key]} but should be at least #{SYS_VAR_TO_CHECK[err_key].values.first}"
+            else
+              error_explanation << "\n    * #{err_key} is #{errors[err_key]} but should be #{SYS_VAR_TO_CHECK[err_key]}"
+            end
           end
           raise FlydataCore::MysqlCompatibilityError,
             "These system variable(s) are not the correct value: #{error_explanation}\n" +
-            "   Please change these system variables for FlyData Sync to run correctly"
+            "   Please change these system variables for FlyData Sync to run"
+        end
+      end
+
+      private
+      def compare_sys_var_values(input_value_hash, expected_value_hash, errors, exception_arr)
+        expected_value_hash.each_key do |exp_val|
+          if input_value_hash.has_key?(exp_val)
+            actual_val = input_value_hash[exp_val]
+            compare_sym = expected_value_hash[exp_val].is_a?(Hash) ? expected_value_hash[exp_val].keys.first : "="
+            case compare_sym
+            when "="
+              expected_val = expected_value_hash[exp_val]
+              unless actual_val == expected_val
+                errors[exp_val] = actual_val
+              end
+            when ">"
+              expected_val = expected_value_hash[exp_val].values.first
+              unless actual_val.to_i >= expected_val.to_i
+                errors[exp_val] = actual_val
+              end
+            end
+          elsif not exception_arr.include?(exp_val)
+            errors[exp_val] = false
+          end
         end
       end
     end
@@ -181,6 +204,34 @@ module FlydataCore
       end
     end
 
+    class PrimaryKeyChecker < MysqlCompatibilityChecker
+      PK_CHECK_QUERY_TMPLT = <<EOT
+SELECT k.table_name as table_name
+FROM information_schema.table_constraints t
+JOIN information_schema.key_column_usage k
+USING(constraint_name,table_schema,table_name)
+WHERE t.constraint_type='PRIMARY KEY'
+  AND t.table_schema='%{database}'
+  AND t.table_name in (%{table_names})
+GROUP BY k.table_name;
+EOT
+      def create_query(option = @option)
+        PK_CHECK_QUERY_TMPLT % {database: option[:database], table_names: option[:tables].collect{|tn| "'#{tn}'"}.join(',')}
+      end
+
+      def check_result(result, option = @option)
+        pk_tables = []
+        result.each {|r| pk_tables << r['table_name']}
+        missing_pk_tables = option[:tables] - pk_tables
+
+        unless missing_pk_tables.empty?
+          raise FlydataCore::MysqlCompatibilityError,
+            "Primary key is required for tables to sync. " +
+            "Add primary key or remove following tables from data entry: #{missing_pk_tables.join(", ")}"
+        end
+      end
+    end
+
     class NonRdsRetentionChecker < MysqlCompatibilityChecker
       include MysqlVariablesHandling
 

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -87,6 +87,19 @@ EOS
     schema_name.to_s.empty? ? "\"#{table_name}\"" : "\"#{schema_name}\".\"#{table_name}\""
   end
 
+  def self.fq_table_name(table_name, schema_name)
+    idx = table_name.rindex('.')
+    if (idx)
+      # table name has schame qualifier.  Remove it.
+      table_name = table_name[idx+1..-1]
+    end
+
+    result =
+      schema_name.to_s.empty? ? table_name : "#{schema_name}.#{table_name}"
+
+    convert_to_valid_table_name(result)
+  end
+
   def self.flydata_ctl_table_for_ddl(schema_name, ctl_table_type = :columns)
     table_name = case ctl_table_type
                  when :columns; FLYDATA_CTL_COLUMNS_TABLE

data/flydata-core/lib/flydata-core/wrapper_forwardable.rb

@@ -0,0 +1,28 @@
+module FlydataCore
+
+module WrapperSingleForwardable
+  # Delegates a method call `delegator` to the given `accessor` like the
+  # original `def_single_delegator` does.  In addition, it wraps the returned
+  # object (or each object in an array) with the caller's own instance.
+  def def_wrapper_single_delegator(accessor, delegator)
+    self.define_singleton_method(delegator) do |*args|
+      result = accessor.send(delegator, *args)
+      case result
+      when Array
+        result.collect{|res| self.new(res)}
+      when NilClass
+        nil
+      else
+        self.new(result)
+      end
+    end
+  end
+
+  def def_wrapper_single_delegators(accessor, *delegators)
+    delegators.each do |delegator|
+      def_wrapper_single_delegator(accessor, delegator)
+    end
+  end
+end
+
+end

data/flydata-core/spec/mysql/compatibility_checker.rb

@@ -1,9 +1,186 @@
 require 'spec_helper'
+require 'flydata-core/mysql/compatibility_checker'
 
 module FlydataCore
   module Mysql
-    describe SyncPermissionChecker do
+    describe CompatibilityChecker do
+      let(:database) { 'test_db' }
+      let(:tables) { %w(table_1 table_2 table_3) }
+      let(:option) do
+        { database: database, tables: tables }
+      end
 
+      describe PrimaryKeyChecker do
+        let(:subject_object) { described_class.new(option) }
+
+        describe '#create_query' do
+          subject { subject_object.create_query }
+          it { is_expected.to eq <<EOT
+SELECT k.table_name as table_name
+FROM information_schema.table_constraints t
+JOIN information_schema.key_column_usage k
+USING(constraint_name,table_schema,table_name)
+WHERE t.constraint_type='PRIMARY KEY'
+  AND t.table_schema='test_db'
+  AND t.table_name in ('table_1','table_2','table_3')
+GROUP BY k.table_name;
+EOT
+          }
+        end
+
+        describe '#check_reesult' do
+          let(:result) { [] }
+          subject { subject_object.check_result(result) }
+
+          context 'when all tables have pk' do
+            let(:result) do
+              [ {"table_name" => "table_1"},
+                {"table_name" => "table_2"},
+                {"table_name" => "table_3"}, ]
+            end
+            it { expect{subject}.not_to raise_error }
+          end
+
+          context 'when some tables does not have pk' do
+            let(:result) { [{"table_name" => "table_2"} ] }
+            it do
+              expect{subject}.to raise_error {|e|
+                expect(e).to be_kind_of(FlydataCore::MysqlCompatibilityError)
+                expect(e.to_s.include?("table_1")).to be_truthy
+                expect(e.to_s.include?("table_2")).to be_falsey
+                expect(e.to_s.include?("table_3")).to be_truthy
+              }
+            end
+          end
+        end
+      end
+    end
+
+    describe BinlogParameterChecker do
+      let(:subject_object) { BinlogParameterChecker.new({}) }
+
+      describe "#check_result" do
+        let(:sys_var) do
+          [
+            {'Variable_name'=>'binlog_format','Value'=>'ROW'},
+            {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
+            {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
+            {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
+            {'Variable_name'=>'net_read_timeout','Value'=>600},
+            {'Variable_name'=>'net_write_timeout','Value'=>600},
+            {'Variable_name'=>'wait_timeout','Value'=>60}
+          ]
+        end
+        subject { subject_object.check_result(sys_var, {}) }
+
+        context "where parameters are valid" do
+          it do
+            expect{subject}.to_not raise_error
+          end
+        end
+        context "where parameters are invalid" do
+          let(:sys_var) do
+            [
+              {'Variable_name'=>'binlog_format','Value'=>'MIXED'},
+              {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
+              {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
+              {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
+              {'Variable_name'=>'net_read_timeout','Value'=>600},
+              {'Variable_name'=>'net_write_timeout','Value'=>600},
+              {'Variable_name'=>'wait_timeout','Value'=>30}
+            ]
+          end
+          it do
+            expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /binlog_format.*\n.*wait_timeout/)
+          end
+        end
+      end
+      describe "#compare_sys_var_values" do
+        let(:expected_val) do
+          {
+            :value_1 => 'foo',
+            :value_2 => 50,
+            :value_3 => {">"=>100}
+          }
+        end
+        let(:errors) { {} }
+        let(:exception_arr) { [] }
+        subject { subject_object.send(:compare_sys_var_values, input_val, expected_val, errors, exception_arr) }
+
+        shared_examples "expect correct errors from values" do
+          it do
+            subject
+            expect(errors).to eq( expecting_errors )
+          end
+        end
+
+        context "when getting valid parameters" do
+          let(:input_val) do
+            {
+              :value_1 => 'foo',
+              :value_2 => 50,
+              :value_3 => 200
+            }
+          end
+          let(:expecting_errors) { {} }
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting one invalid exact parameters for int" do
+          let(:input_val) do
+            {
+              :value_1 => 'foo',
+              :value_2 => 20,
+              :value_3 => 100
+            }
+          end
+          let(:expecting_errors) { {value_2:20} }
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting one invalid exact parameter for string" do
+          let(:input_val) do
+            {
+              :value_1 => 'fo',
+              :value_2 => 50,
+              :value_3 => 100
+            }
+          end
+          let(:expecting_errors) { {value_1:'fo'} }
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting one invalid comparitive parameter that's lower than required" do
+          let(:input_val) do
+            {
+              :value_1 => 'foo',
+              :value_2 => 50,
+              :value_3 => 1
+            }
+          end
+          let(:expecting_errors) { {value_3:1} }
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting a valid comparitive parameter that's higher than required" do
+          let(:input_val) do
+            {
+              :value_1 => 'foo',
+              :value_2 => 50,
+              :value_3 => 101
+            }
+          end
+          let(:expecting_errors) { {} }
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting muliple invalid parameters" do
+          let(:input_val) do
+            {
+              :value_1 => 'fo',
+              :value_2 => 50,
+              :value_3 => 15
+            }
+          end
+          let(:expecting_errors) { {value_1:'fo', value_3:15} }
+          it_behaves_like "expect correct errors from values"
+        end
+      end
     end
   end
 end

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -896,6 +896,43 @@ EOS
 
     it_behaves_like "expecting no underscore in front of numbers"
   end
+
+  describe ".fq_table_name" do
+    subject { subject_object.fq_table_name(table_name, schema_name) }
+    context "with no schema name" do
+      let(:schema_name) { nil }
+      context "with a table name with no schema prefix" do
+        let(:table_name) { "test_table" }
+        it { is_expected.to eq "test_table" }
+      end
+      context "with a table name with a schema prefix" do
+        let(:table_name) { "some_schema.test_table" }
+        it { is_expected.to eq "test_table" }
+      end
+      context "with a mixed-case table name" do
+        let(:table_name) { "Test_Table" }
+        it { is_expected.to eq "test_table" }
+      end
+    end
+    context "with a schema name" do
+      let(:schema_name) { "test_schema" }
+      context "with a table name with no schema prefix" do
+        let(:table_name) { "test_table" }
+        it { is_expected.to eq "test_schema.test_table" }
+      end
+      context "with a table name with a schema prefix" do
+        let(:table_name) { "some_schema.test_table" }
+        it { is_expected.to eq "test_schema.test_table" }
+      end
+    end
+    context "with a mixed-case schema name" do
+      let(:schema_name) { "Test_Schema" }
+      context "with a table name with no schema prefix" do
+        let(:table_name) { "test_table" }
+        it { is_expected.to eq "test_schema.test_table" }
+      end
+    end
+  end
 end
 
 end

data/flydata-core/spec/wrapper_forwardable_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+require 'flydata-core/wrapper_forwardable'
+
+module FlydataCore
+
+describe WrapperSingleForwardable do
+  class Target
+    def self.foo(a, b)
+      target_foo_result
+    end
+  end
+
+  class Test
+    extend WrapperSingleForwardable
+
+    def_wrapper_single_delegator Target, :foo
+
+    def initialize(item)
+      @item = item
+    end
+
+    def ==(other)
+      @item == other.item
+    end
+
+    attr_reader :item
+  end
+  let(:subject_object) { Test }
+  let(:args) { [ 'arg1', 'arg2' ] }
+
+  describe '.def_wrapper_single_delegator' do
+    it 'defines a class method of the given name to the subject class' do
+      expect(subject_object.respond_to?(:foo)).to eq true
+    end
+
+    describe 'the method defined by it' do
+      subject { subject_object.foo(*args) }
+      before do
+        expect(Target).to receive(:foo).with(*args).
+                            and_return(target_foo_result)
+      end
+
+      context 'when the target method returns an object' do
+        let(:target_foo_result) { double('single_object') }
+
+        it 'returns an object of the subject class containing the target result' do
+          is_expected.to eq subject_object.new(target_foo_result)
+        end
+      end
+      context 'when the target method returns nil' do
+        let(:target_foo_result) { nil }
+
+        it 'returns nil' do
+          is_expected.to eq nil
+        end
+      end
+      context 'when the target method returns an array' do
+        let(:target_foo_result) { [ double('object1'), double('object2') ] }
+        it 'returns an array of instances of the subject class.  Each instance contains an object returned from the target' do
+          is_expected.to eq target_foo_result.collect{|item| subject_object.new(item) }
+        end
+      end
+    end
+  end
+end
+
+end

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.5.0 ruby lib
+# stub: flydata 0.5.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.5.0"
+  s.version = "0.5.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-07-24"
+  s.date = "2015-08-04"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -51,8 +51,6 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/mysql/compatibility_checker.rb",
     "flydata-core/lib/flydata-core/option_validator.rb",
     "flydata-core/lib/flydata-core/query_job.rb",
-    "flydata-core/lib/flydata-core/query_job/redshift.rb",
-    "flydata-core/lib/flydata-core/query_job/table_status.rb",
     "flydata-core/lib/flydata-core/record/record.rb",
     "flydata-core/lib/flydata-core/redshift/string.rb",
     "flydata-core/lib/flydata-core/table_def.rb",
@@ -61,13 +59,13 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
     "flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb",
     "flydata-core/lib/flydata-core/thread_context.rb",
+    "flydata-core/lib/flydata-core/wrapper_forwardable.rb",
     "flydata-core/spec/config/user_maintenance_spec.rb",
     "flydata-core/spec/fluent/config_helper_spec.rb",
     "flydata-core/spec/logger_spec.rb",
     "flydata-core/spec/mysql/command_generator_spec.rb",
     "flydata-core/spec/mysql/compatibility_checker.rb",
     "flydata-core/spec/option_validator_spec.rb",
-    "flydata-core/spec/query_job/redshift_spec.rb",
     "flydata-core/spec/redshift/string_spec.rb",
     "flydata-core/spec/spec_helper.rb",
     "flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb",
@@ -88,6 +86,7 @@ Gem::Specification.new do |s|
     "flydata-core/spec/table_def/mysqldump_test_unsigned.dump",
     "flydata-core/spec/table_def/redshift_table_def_spec.rb",
     "flydata-core/spec/table_def/sync_redshift_table_def_spec.rb",
+    "flydata-core/spec/wrapper_forwardable_spec.rb",
     "flydata.gemspec",
     "lib/fly_data_model.rb",
     "lib/flydata.rb",
@@ -243,7 +242,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_runtime_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_runtime_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-      s.add_runtime_dependency(%q<kodama>, [">= 0.1.6", "~> 0.1.2"])
+      s.add_runtime_dependency(%q<kodama>, [">= 0.1.7", "~> 0.1.2"])
       s.add_runtime_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_development_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_development_dependency(%q<rspec>, ["~> 3.0"])
@@ -267,7 +266,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-      s.add_dependency(%q<kodama>, [">= 0.1.6", "~> 0.1.2"])
+      s.add_dependency(%q<kodama>, [">= 0.1.7", "~> 0.1.2"])
       s.add_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_dependency(%q<rspec>, ["~> 3.0"])
@@ -292,7 +291,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
     s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
     s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-    s.add_dependency(%q<kodama>, [">= 0.1.6", "~> 0.1.2"])
+    s.add_dependency(%q<kodama>, [">= 0.1.7", "~> 0.1.2"])
     s.add_dependency(%q<serverengine>, ["~> 1.5"])
     s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
     s.add_dependency(%q<rspec>, ["~> 3.0"])

data/lib/flydata/command/sender.rb

@@ -37,7 +37,7 @@ module Flydata
           # Start sender(fluentd) process
           log_info_stdout("Starting FlyData Agent sender process.") unless options[:quiet]
           raw_start(options)
-  
+
           wait_until_client_ready(options)
           #wait_until_logs_uploaded
           fluentd_started = true
@@ -117,7 +117,7 @@ EOF
         `[ -f #{pid_file} ] && pgrep -P \`cat #{pid_file}\``.to_i > 0
       end
       def kill_all(options = {})
-        if Kernel.system("ps ax | grep 'flydata' | grep -v grep | awk '{print \"kill  \" $1}' | sh")
+        if Kernel.system("ps ax | grep 'flydata' | grep -v grep | grep fluentd | awk '{print \"kill  \" $1}' | sh")
           log_info_stdout("Done.") unless options[:quiet]
           return true
         else
@@ -172,7 +172,7 @@ EOF
         raise "Somthing has gone wrong... Please try setup command again."
       end
       def wait_until_client_stop(options = {})
-        retry_count = 5
+        retry_count = 10
         1.upto(retry_count) do |i|
           return true unless process_exist?
           log_info_stdout("Waiting for the client to stop... (#{i}/#{retry_count})") unless options[:quiet]

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -57,6 +57,23 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
   config_param :idle_timeout, :time, :default => 1800               # 30 min
   config_param :check_interval, :integer, :default => 5             # 5 sec
   config_param :ssl_ca_content, :string, :default => ''
+  config_param :ssl_cipher, :string, :default => nil
+
+  # Secondary ssl cipher is for a workaround to handle 'dh key too small' error
+  # If mysql-server is v5.6.25 or older, and client openssl is
+  # 1.0.1f-1ubuntu-2.15(for ubuntu) or later, client will get the above error because.
+  # the key length returned from mysql-server is only 512 bits.
+  #
+  # In case of dh key error, we retry with non-DH cipher.
+  #
+  # * SECURITY IMPROVEMENT: reject dh keys smaller than 768 bits
+  # https://launchpad.net/ubuntu/+source/openssl/1.0.1f-1ubuntu2.15
+  #
+  # Supported ssl cipher list for mysql(openssl)
+  # https://dev.mysql.com/doc/refman/5.6/en/ssl-options.html
+
+  NON_DH_SSL_CIPHER = "AES256-GCM-SHA384:AES256-SHA:AES256-SHA256:CAMELLIA256-SHA:DES-CBC3-SHA:PSK-3DES-EDE-CBC-SHA:PSK-AES256-CBC-SHA:SRP-DSS-3DES-EDE-CBC-SHA:SRP-DSS-AES-128-CBC-SHA:SRP-DSS-AES-256-CBC-SHA:SRP-RSA-3DES-EDE-CBC-SHA:SRP-RSA-AES-128-CBC-S:SRP-RSA-AES-256-CBC-SHA"
+  config_param :secondary_ssl_cipher, :string, :default => NON_DH_SSL_CIPHER
 
   def configure(conf)
     super
@@ -110,7 +127,6 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       h
     end
 
-
     # Set context
     @context = Mysql::Context.new(
       database: @database, tables: @tables,
@@ -162,29 +178,48 @@ EOS
                                             @context) do |event|
       @record_dispatcher.dispatch(event)
     end
-    start_kodama(mysql_url) do |c|
-      c.binlog_position_file = @position_file
-      if @ssl_ca_path.to_s != '' && c.respond_to?(:ssl_ca=)
-        $log.info "SSL is enabled. (ssl_ca: #{@ssl_ca_path})"
-        c.ssl_ca = @ssl_ca_path
-      end
 
-      if c.respond_to?(:sent_binlog_position_file=)
-        $log.info "Sent position feature is enabled. sent_position_file:#{@sent_position_file_path}"
-        c.sent_binlog_position_file = @sent_position_file_path
-      end
+    current_ssl_cipher = @ssl_cipher
+    retried = false
+    begin
+      start_kodama(mysql_url) do |c|
+        c.binlog_position_file = @position_file
+        if @ssl_ca_path.to_s != '' && c.respond_to?(:ssl_ca=)
+          $log.info "SSL is enabled. (ssl_ca: #{@ssl_ca_path})"
+          c.ssl_ca = @ssl_ca_path
+          unless current_ssl_cipher.to_s.empty?
+            $log.info "SSL cipher is set. (ssl_cipher: #{current_ssl_cipher})"
+            c.ssl_cipher = current_ssl_cipher
+          end
+        end
 
-      $log.info("Binlog position - resume_pos:'#{IO.read(@position_file) rescue IOError}' " +
-                                  "sent_pos:'#{IO.read(@sent_position_file_path) rescue IOError}'")
+        if c.respond_to?(:sent_binlog_position_file=)
+          $log.info "Sent position feature is enabled. sent_position_file:#{@sent_position_file_path}"
+          c.sent_binlog_position_file = @sent_position_file_path
+        end
 
-      c.connection_retry_limit = @retry_limit
-      c.connection_retry_wait = @retry_wait
-      c.log_level = @log_level.to_sym
-      @listen_events.each do |event_type|
-        $log.trace { "registered binlog event listener '#{event_type}'" }
-        c.send("on_#{event_type}", &method(:event_listener))
+        $log.info("Binlog position - resume_pos:'#{IO.read(@position_file) rescue IOError}' " +
+                                    "sent_pos:'#{IO.read(@sent_position_file_path) rescue IOError}'")
+
+        c.connection_retry_limit = @retry_limit
+        c.connection_retry_wait = @retry_wait
+        c.log_level = @log_level.to_sym
+        @listen_events.each do |event_type|
+          $log.trace { "registered binlog event listener '#{event_type}'" }
+          c.send("on_#{event_type}", &method(:event_listener))
+        end
+      end
+    rescue Binlog::Error => e
+      if /dh key too small/.match(e.to_s) && !retried && !@secondary_ssl_cipher.to_s.empty?
+        retried = true
+        current_ssl_cipher = @secondary_ssl_cipher
+        $log.warn("Retry with secondary ssl cipher list due to '#{e}' - secondary_ssl_cipher: '#{@secondary_ssl_cipher}'")
+        retry
+      else
+        raise e
       end
     end
+
   rescue => e
     # HACK: mysql-replication-listener has a network connection leak bug which doesn't release a connection
     # to MySQL.  Rather than fixing the bug, restarting the fluentd process for now.

data/lib/flydata/parser/mysql/mysql_alter_table.treetop

@@ -894,7 +894,7 @@ grammar MysqlAlterTable
     }
   end
   rule character_set_opt
-    'character set'i sp value {
+    charset sp value {
       def character_set_opt_option
         { cs: FlydataCore::TableDef::MysqlTableDef.flydata_charset(value.raw_value) }
       end

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -185,19 +185,50 @@ describe 'MysqlAlterTableParser' do
       end
 
       context 'with after option' do
-        let(:query) { "alter table test_table add column value varchar(26) after id" }
-        it do
-          expect(subject).to eq({
-            type: :alter_table,
-            table_name: "test_table",
-            actions: [{
-              action: :add_column,
-              column: "value",
-              type: "varchar(78)",
-              after: 'id',
-              query: "add column value varchar(26) after id"
-            }]
-          })
+        context 'without charset' do
+          let(:query) { "alter table test_table add column value varchar(26) after id" }
+          it do
+            expect(subject).to eq({
+              type: :alter_table,
+              table_name: "test_table",
+              actions: [{
+                action: :add_column,
+                column: "value",
+                type: "varchar(78)",
+                after: 'id',
+                query: "add column value varchar(26) after id"
+              }]
+            })
+          end
+        end
+        context 'with charset and collate' do
+          shared_examples "parse character set correctly" do
+            let(:query) { "alter table `kiikplan`.`plan` add column `email_header` varchar(1024) #{charset} utf8 COLLATE utf8_general_ci NULL after `color`" }
+            it do
+              expect(subject).to eq({
+                type: :alter_table,
+                table_name: "plan",
+                actions: [{
+                  action: :add_column,
+                  column: "email_header",
+                  type: "varchar(3072)",
+                  cs: "UTF_8",
+                  after: 'color',
+                  query: "add column `email_header` varchar(1024) #{charset} utf8 COLLATE utf8_general_ci NULL after `color`"
+                }],
+                schema_name: "kiikplan"
+              })
+            end
+          end
+          context "with character set" do
+            let(:charset) { "character set" }
+            include_examples "parse character set correctly"
+          end
+
+          context "with charset" do
+            let(:charset) { "CHARSET" }
+            include_examples "parse character set correctly"
+          end
         end
       end
       shared_examples "generating a flydata record for the given query" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-24 00:00:00.000000000 Z
+date: 2015-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -248,7 +248,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.6
+        version: 0.1.7
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.2
@@ -258,7 +258,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.6
+        version: 0.1.7
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.2
@@ -488,8 +488,6 @@ files:
 - flydata-core/lib/flydata-core/mysql/compatibility_checker.rb
 - flydata-core/lib/flydata-core/option_validator.rb
 - flydata-core/lib/flydata-core/query_job.rb
-- flydata-core/lib/flydata-core/query_job/redshift.rb
-- flydata-core/lib/flydata-core/query_job/table_status.rb
 - flydata-core/lib/flydata-core/record/record.rb
 - flydata-core/lib/flydata-core/redshift/string.rb
 - flydata-core/lib/flydata-core/table_def.rb
@@ -498,13 +496,13 @@ files:
 - flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
 - flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb
 - flydata-core/lib/flydata-core/thread_context.rb
+- flydata-core/lib/flydata-core/wrapper_forwardable.rb
 - flydata-core/spec/config/user_maintenance_spec.rb
 - flydata-core/spec/fluent/config_helper_spec.rb
 - flydata-core/spec/logger_spec.rb
 - flydata-core/spec/mysql/command_generator_spec.rb
 - flydata-core/spec/mysql/compatibility_checker.rb
 - flydata-core/spec/option_validator_spec.rb
-- flydata-core/spec/query_job/redshift_spec.rb
 - flydata-core/spec/redshift/string_spec.rb
 - flydata-core/spec/spec_helper.rb
 - flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb
@@ -525,6 +523,7 @@ files:
 - flydata-core/spec/table_def/mysqldump_test_unsigned.dump
 - flydata-core/spec/table_def/redshift_table_def_spec.rb
 - flydata-core/spec/table_def/sync_redshift_table_def_spec.rb
+- flydata-core/spec/wrapper_forwardable_spec.rb
 - flydata.gemspec
 - lib/fly_data_model.rb
 - lib/flydata.rb

data/flydata-core/lib/flydata-core/query_job/redshift.rb

@@ -1,27 +0,0 @@
-require 'flydata-core/table_def/redshift_table_def'
-
-module FlydataCore
-module QueryJob
-
-class Redshift
-  # returns a list of table names used in the Query Job system.
-  def self.target_table_names(flydata_table_names)
-    flydata_table_names = [ flydata_table_names ] unless flydata_table_names.kind_of?(Array)
-
-    flydata_table_names.collect {|flydata_table_name|
-      redshift_table_name = TableDef::RedshiftTableDef.convert_to_valid_table_name(flydata_table_name)
-      table_names = [ redshift_table_name ]
-      if redshift_table_name != flydata_table_name
-        # for backward compatibility
-        # Old implementation used FlyData table names.  To support it, the
-        # method adds the flydata_table_name as well.
-        table_names << flydata_table_name
-      end
-
-      table_names
-    }.flatten
-  end
-end
-
-end
-end

data/flydata-core/lib/flydata-core/query_job/table_status.rb

@@ -1,44 +0,0 @@
-require 'flydata-core/query_job/redshift'
-require 'flydata-core/table_def/redshift_table_def'
-
-module FlydataCore
-module QueryJob
-
-class TableStatus
-  @@domain = 'flydata_redshift_table_status'
-
-  def self.domain=(domain)
-    @@domain = domain
-  end
-
-  def self.sdb=(sdb)
-    @@sdb = sdb
-  end
-
-  def self.where(*args)
-    table_status_items = {}
-    @@sdb.domains[@@domain].items.where(*args).each(select: :all) do |item|
-      class << item
-        # returns the corect (Redshift) table name
-        def table_name
-          FlydataCore::TableDef::RedshiftTableDef.convert_to_valid_table_name(attributes['table_name'].first)
-        end
-
-        def target_table_names
-          FlydataCore::QueryJob::Redshift.target_table_names(attributes['table_name'].first)
-        end
-      end
-      tn = item.attributes['table_name'].first
-      redshift_table_name = FlydataCore::TableDef::RedshiftTableDef.convert_to_valid_table_name(tn)
-      if redshift_table_name == tn
-        table_status_items[redshift_table_name] ||= item
-      else
-        table_status_items[redshift_table_name] = item
-      end
-    end
-    table_status_items.values
-  end
-end
-
-end
-end

data/flydata-core/spec/query_job/redshift_spec.rb

@@ -1,34 +0,0 @@
-require 'spec_helper'
-require 'flydata-core/query_job/redshift'
-
-module FlydataCore::QueryJob
-
-describe Redshift do
-  describe '.target_table_names' do
-    subject { described_class.target_table_names(flydata_table_names) }
-
-    context 'with a single table name' do
-      let(:flydata_table_names) { "my_table" }
-
-      it { is_expected.to eq [ flydata_table_names ] }
-    end
-
-    context 'with multiples table names' do
-      let(:flydata_table_names) { %w(my_table1 my_table2) }
-
-      it { is_expected.to eq flydata_table_names }
-    end
-    context 'with a mixed case table name' do
-      let(:flydata_table_names) { "MyTable" }
-
-      it { is_expected.to eq [ flydata_table_names.downcase, flydata_table_names ] }
-    end
-    context 'with table names including mixed case one' do
-      let(:flydata_table_names) { %w(MyTable my_table2) }
-
-      it { is_expected.to eq %w(mytable MyTable my_table2) }
-    end
-  end
-end
-
-end