flydata 0.3.7 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df77223c799282a371eb699e729724caa3d5430a
-  data.tar.gz: 4ef3a2a774c4af5562def918f9a47d544e7cac87
+  metadata.gz: 652af727f40585a7cb70d5b1f71fcdd9beda4a7e
+  data.tar.gz: f8e94b1dd631c553c787175910695065bcedee65
 SHA512:
-  metadata.gz: 6b0dc6d7a68cdf9e3f66d242d87ebad2174ae19815b6e1c460fd26cf89722ce50cc8db478fc2816de096168e185745b98050d4c59b8e41a4fddf3e7ad9a6cccf
-  data.tar.gz: 410125ce7864d924fc38774324819542184f75f60a5abf1713873e889488156967a54e70eb888a7a0f19b4fe5f26e858ff4368c257074314f455f8ad8efc74a1
+  metadata.gz: 2e815f1b604d0ba454e82bbaae84ac13b6173507bc4066a138af3221fdaa3327aed5f636e9ebfc2c061b38eda6e0410e3ea6b57dcfcdca592b07f0cc64a6fc01
+  data.tar.gz: bdb92aee00edac39d02d4a374ca4dbf555c4b80672bcaade05801db0a0d3612715b0d19036c65162f8911401d9c7d31a5c9cc7c30730855cf0778eb53e808816

data/VERSION

@@ -1 +1 @@
-0.3.7
+0.3.8

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.7 ruby lib
+# stub: flydata 0.3.8 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.7"
+  s.version = "0.3.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-03-05"
+  s.date = "2015-03-09"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -81,7 +81,6 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
     table_meta = Mysql::TableMeta.new(
       mysql_url: mysql_url, database: @database, tables: @tables)
-    table_meta.update
 
     table_revs = tables.inject({}) do |h, table_name|
       h[table_name] = @sync_fm.table_rev(table_name)
@@ -129,6 +128,7 @@ EOS
   end
 
   def run
+    @context.table_meta.update
     start_kodama(mysql_url) do |c|
       c.binlog_position_file = @position_file
       c.connection_retry_limit = @retry_limit
@@ -139,9 +139,12 @@ EOS
         c.send("on_#{event_type}", &method(:event_listener))
       end
     end
-  rescue
-    $log.error "unexpected error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
-    raise
+  rescue => e
+    # HACK: mysql-replication-listener has a network connection leak bug which doesn't release a connection
+    # to MySQL.  Rather than fixing the bug, restarting the fluentd process for now.
+    $log.warn "kodama died with an error.  Restart the process after #{@retry_wait} seconds.  error: #{e.class.to_s} '#{e.to_s}'"
+    sleep @retry_wait
+    Process.kill(:HUP, Process.ppid) # Send SIGHUP to the supervisor to restart fluentd
   rescue SignalException
     $log.debug "signal exception. exception: #{$!.class.to_s}, error: #{$!.to_s}"
     raise

data/lib/flydata/parser/mysql/mysql_alter_table.treetop

@@ -218,9 +218,9 @@ grammar MysqlAlterTable
         { action: :convert_charset }
       end
     }
-    / create_table_options_space_separated {
+    / ( create_table_options_space_separated ) 1..1 {
       def action
-        { action: :default_charset }
+        elements[0].action
       end
     }
     / force_sym {
@@ -990,8 +990,13 @@ grammar MysqlAlterTable
     'tablespace'i ![A-Za-z0-9_]
   end
 
+  rule engine_sym
+    'engine'i ![A-Za-z0-9_]
+  end
+
   rule ident_or_text
     ident
+    / value # This matches TEXT_STRING_sys
     # TEXT_STRING # TODO - To be implemented when required
     # LEX_HOSTNAME # TODO
   end
@@ -1025,13 +1030,38 @@ grammar MysqlAlterTable
   end
 
   rule create_table_options_space_separated
-    create_table_option sp create_table_options_space_separated
-    / create_table_option
+    create_table_option sp create_table_options_space_separated {
+      def action
+        ret = [create_table_option.action]
+        other_actions = create_table_options_space_separated.action
+        ret + (other_actions.kind_of?(Array) ? other_actions : [other_actions])
+      end
+    }
+    / ( create_table_option ) 1..1 {
+      def action
+        elements[0].action
+      end
+    }
   end
 
   rule create_table_option
-    default_charset
-    / default_collation
+    default_charset {
+      def action
+        { action: :default_charset }
+      end
+    }
+    / default_collation {
+      def action
+        { action: :default_collate }
+      end
+    }
+    / engine_sym opt_equal nsp storage_engines {
+      def action
+        { action: :storage_engine,
+          support_level: :nonbreaking
+        }
+      end
+    }
     #TODO - There are other rules which need to be implemented when required
   end
 
@@ -1047,6 +1077,10 @@ grammar MysqlAlterTable
     opt_default nsp collate_sym opt_equal nsp collation_name_or_default
   end
 
+  rule storage_engines
+    ident_or_text
+  end
+
   ######## keys
 
   rule alter_key 'alter'i end

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -553,7 +553,7 @@ describe 'MysqlAlterTableParser' do
           type: :alter_table,
           schema_name: "test_schema",
           table_name: "test_table",
-          actions: [action_hash]
+          actions: action_hash
         })
       end
     end
@@ -564,20 +564,13 @@ describe 'MysqlAlterTableParser' do
     end
     shared_examples "a parser parsing a nonbreaking query" do
       let(:action_hash) {
-          {
-            action: action,
-            support_level: :nonbreaking,
-            query: alter_table_action,
-          }
+        Array(action).inject([]) { |ret,a| ret << { action: a, support_level: :nonbreaking, query: alter_table_action } }
       }
       it_behaves_like "a parser parsing a single action alter table query"
     end
     shared_examples "a parser parsing a breaking query" do
       let(:action_hash) {
-        {
-          action: action,
-          query: alter_table_action,
-        }
+        Array(action).inject([]) { |ret,a| ret << { action: a, query: alter_table_action } }
       }
       it_behaves_like "a parser parsing a single action alter table query"
     end
@@ -1076,6 +1069,15 @@ describe 'MysqlAlterTableParser' do
         end
       end
     end
+    shared_examples "test storage engines" do |*examples|
+      ["innodb", "MyISAM", "MEMORY", '"InnoDB"', "'myisam'"].each do |storage_engine|
+        context "storage engine is #{storage_engine}" do
+          it_behaves_like *examples do
+            let(:storage_engine) {storage_engine}
+          end
+        end
+      end
+    end
     context 'add index' do
       let(:action) { :add_index }
       context "with a simple add index" do
@@ -1231,11 +1233,11 @@ describe 'MysqlAlterTableParser' do
       let(:alter_table_action) { "ADD#{constraint_keyword} PRIMARY KEY#{key_name}#{using_option} (#{key_list})#{normal_key_options}" }
       let(:action) { :add_primary_key_constraint }
       let(:action_hash) {
-        {
+        [{
           action: action,
           keys: expected_key_list,
           query: alter_table_action,
-        }
+        }]
       }
 
       it_behaves_like "test constraint keyword",
@@ -1336,23 +1338,32 @@ describe 'MysqlAlterTableParser' do
         it_behaves_like "test charset", "test charsetname", "a parser parsing a breaking query"
       end
     end
-    context "default charset" do
-      let(:action) { :default_charset }
-      context "with collate" do
-        let(:alter_table_action) { "#{default}#{charset}=#{charsetname} #{collate}" }
-        it_behaves_like "test optional default", "test charset",  "test charsetname", "test collate", "a parser parsing a breaking query"
-        it_behaves_like "test optional default",  "test charset", "test charsetname", "test collate with equals", "a parser parsing a breaking query"
-      end
-      context "without collate" do
-        let(:alter_table_action) { "#{default}#{charset} #{charsetname}" }
-        it_behaves_like "test optional default", "test charset",  "test charsetname", "a parser parsing a breaking query"
+    context "table options" do
+      context "default charset" do
+        context "with collate" do
+          let(:action) { [:default_charset, :default_collate] }
+          let(:alter_table_action) { "#{default}#{charset}=#{charsetname} #{collate}" }
+          it_behaves_like "test optional default", "test charset",  "test charsetname", "test collate", "a parser parsing a breaking query"
+          it_behaves_like "test optional default",  "test charset", "test charsetname", "test collate with equals", "a parser parsing a breaking query"
+        end
+        context "without collate" do
+          let(:action) { :default_charset }
+          let(:alter_table_action) { "#{default}#{charset} #{charsetname}" }
+          it_behaves_like "test optional default", "test charset",  "test charsetname", "a parser parsing a breaking query"
+        end
+        context "without charset" do
+        #The doc at http://dev.mysql.com/doc/refman/5.6/en/alter-table.html does not indicate that
+        #charset is optional. But this works (tested on mysql 5.5.38)
+          let(:alter_table_action) { "#{default}#{collate}" }
+          let(:action) { :default_collate }
+          it_behaves_like  "test optional default","test collate", "a parser parsing a breaking query"
+          it_behaves_like  "test optional default","test collate with equals", "a parser parsing a breaking query"
+        end
       end
-      context "without charset" do
-      #The doc at http://dev.mysql.com/doc/refman/5.6/en/alter-table.html does not indicate that
-      #charset is optional. But this works (tested on mysql 5.5.38)
-        let(:alter_table_action) { "#{default}#{collate}" }
-        it_behaves_like  "test optional default","test collate", "a parser parsing a breaking query"
-        it_behaves_like  "test optional default","test collate with equals", "a parser parsing a breaking query"
+      context "storage engine" do
+        let(:action) { :storage_engine }
+        let(:alter_table_action) { "engine#{equal} #{storage_engine}"}
+        it_behaves_like "test optional equal", "test storage engines", "a parser parsing a nonbreaking query"
       end
     end
     context "force" do
@@ -1698,10 +1709,10 @@ describe 'MysqlAlterTableParser' do
     end
     shared_examples "test parser parsing a change column" do
       let(:action_hash) {
-        change_action_hash.merge!({
+        [change_action_hash.merge!({
           action: :change_column,
           query: alter_table_action
-        })
+        })]
       }
       it_behaves_like "a parser parsing a single action alter table query"
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.8
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-05 00:00:00.000000000 Z
+date: 2015-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client