flydata 0.2.7 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bfd589bd9943f41f8e817ec796aa74aef758d1f9
-  data.tar.gz: dc62b1dde189197fb3552abde072b40697e92dd5
+  metadata.gz: c5fda2fb95c38692e1a0995b3ee7430ea706ec09
+  data.tar.gz: 5028033ad26d7399e7bcb091257540f1fb4ab46a
 SHA512:
-  metadata.gz: 34006fd5e1f6052f901b92df572ae71c64e3deea4449d3c92aded6cf7013ecf5cd142e2d875c800c5778a5962af5d8add5627e315c3a46806df5980e8bbabceb
-  data.tar.gz: 9a7b312e2c03d2048878f211a6c0233d5d9d31bf7cce0cf655a75bc14287e226094779a45d0dd67df81d1b9f6d19aca1f74a4c04870da4fe6a969bcb4fa34ac5
+  metadata.gz: dbcb9055ead8575095ef96d378d2817743f0c03cb629d6d7ef5d46bc99f04fd22e420f2c8011c28a97a3c5041ec683ed57045d59bd86c31fdb6ec680dbb81769
+  data.tar.gz: 1e77c0720dd22cad760cd00f11eef55514bb768937acb5b6970027f352d5cf97d9f2177b6d904aeb74c8716ab6f9f374757f3ce09780e3c166f79ad3b6a7360d

data/VERSION

@@ -1 +1 @@
-0.2.7
+0.2.8

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.2.7 ruby lib
+# stub: flydata 0.2.8 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.7"
+  s.version = "0.2.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2014-10-07"
+  s.date = "2014-10-15"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]

data/lib/flydata/command/base.rb

@@ -35,8 +35,14 @@ module Flydata
       end
       def ask_yes_no(message, default_yes=true)
         suffix = default_yes ? "(Y/n)" : "(y/n)"
+        prompt = "#{message} #{suffix}:  "
+        if opts && opts.yes? # Yes option
+          puts "#{prompt}Yes"
+          return true
+        end
+
         loop do
-          ans = ask("#{message} #{suffix}:  ")
+          ans = ask(prompt)
           return true if default_yes and ans == ''
           if ans.size > 0
             case ans[0].downcase

data/lib/flydata/command/sync.rb

@@ -21,6 +21,7 @@ module Flydata
       def self.slop
         Slop.new do
           on 'c', 'skip-cleanup', 'Skip server cleanup'
+          on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
         end
       end
 
@@ -42,6 +43,12 @@ module Flydata
         sync_mysql_to_redshift(de)
       end
 
+      def self.slop_flush
+        Slop.new do
+          on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
+        end
+      end
+
       def flush
         flush_buffer_and_stop
         puts "Buffers have been flushed and the sender process has been stopped."
@@ -50,6 +57,7 @@ module Flydata
       def self.slop_reset
         Slop.new do
           on 'c', 'client', 'Resets client only.'
+          on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
         end
       end
 
@@ -134,6 +142,7 @@ module Flydata
       def self.slop_generate_table_ddl
         Slop.new do
           on 'c', 'ctl-only', 'Only generate FlyData Control definitions'
+          on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
         end
       end
 

data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb

@@ -14,14 +14,16 @@ module Mysql
     end
 
     def process(record, normalized_query)
-      emit_record(:alter_table, record, increment_table_rev: true) do
+      emit_record(:alter_table, record) do |opt|
         ret = nil
         begin
           result = ParserProvider.parser(:mysql, :mysql_alter_table).new.parse(record['query'])
           if result.nil?
-            $log.warn("Received unsupported alter table query. query:'#{record['query']}'")
+            $log.error("Received unsupported alter table query. query:'#{record['query']}'")
           else
             ret = result.tree
+            breaking_query = ret[:actions].any?{|action| !action.has_key?(:support_level) || action[:support_level] != :nonbreaking}
+            opt[:increment_table_rev] = true if breaking_query
           end
         rescue => e
           msg = <<EOS

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -54,13 +54,14 @@ module Mysql
       @context.omit_events[table].nil? || !@context.omit_events[table].include?(type)
     end
 
-    def emit_record(type, record, opt = {})
+    def emit_record(type, record)
       return unless acceptable_db?(record)
       return unless record["table_name"].nil? or acceptable_table?(record, record["table_name"])
 
       check_empty_binlog
 
-      records = yield
+      opt = {}
+      records = yield(opt) # The block may set options as necessary
       return if records.nil? # skip
       records = [records] unless records.kind_of?(Array)
 

data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb

@@ -41,7 +41,7 @@ module Mysql
     end
 
     def emit_rows(type, record)
-      emit_record(type, record) do |table|
+      emit_record(type, record) do |opt|
         records = record["rows"].collect do |row|
           row = yield(row) if block_given?  # Give the caller a chance to generate the correct row
           { ROW => convert_to_flydata_row_format(row) }

data/lib/flydata/parser/mysql/mysql_alter_table.treetop

@@ -20,10 +20,10 @@ grammar MysqlAlterTable
       def tree
         value = {
           type: :alter_table,
-          table_name: tbl_name.table_part.text_value,
+          table_name: tbl_name.table_part.value,
           actions: alter_specs.actions
         }
-        value[:schema_name] = tbl_name.schema_part.text_value if tbl_name.has_schema?
+        value[:schema_name] = tbl_name.schema_part.value if tbl_name.has_schema?
         value
       end
     }
@@ -41,27 +41,34 @@ grammar MysqlAlterTable
     alter_spec ( comma alter_spec )* {
       def actions
         ret = []
-        concat_action!(ret, alter_spec.action)
+        ret.concat(actions_with_query(alter_spec))
         0.upto(elements[1].elements.size - 1) do |i|
-          action = elements[1].elements[i].elements[1].action
-          concat_action!(ret, action)
+          ret.concat(actions_with_query(elements[1].elements[i].elements[1]))
         end
         ret
       end
 
-      def concat_action!(array, action)
+      def actions_with_query(alter_spec)
+        action = alter_spec.action
         if action.kind_of?(Array)
           actions = action
         else
           actions = [action]
         end
-        array.concat(actions)
+        actions.each {|a| a[:query] = alter_spec.text_value }
+        actions
       end
     }
   end
 
   rule alter_spec
-    add_key (sp col_key)? sp col_name_def ( sp pos_def )? {
+    add_key sp key_def {
+      def action
+        action_hash = key_def.action
+        action_hash
+      end
+    }
+    / add_key (sp col_key)? sp col_name_def ( sp pos_def )? {
       def action
         ret = { action: :add_column }
         ret.merge!(col_name_def.column_def)
@@ -71,7 +78,7 @@ grammar MysqlAlterTable
         ret
       end
     }
-    / add_key (sp col_key)? sp '(' nsp col_name_def ( comma col_name_def )* nsp ')' {
+    / add_key (sp col_key)? nsp '(' nsp col_name_def ( comma col_name_def )* nsp ')' {
       def action
         ret = [col_name_def.column_def]
         ret += 0.upto(elements[6].elements.count - 1).collect do |i|
@@ -80,19 +87,423 @@ grammar MysqlAlterTable
         ret.collect{|v| {action: :add_column}.merge(v) }
       end
     }
+    / drop_key sp primary_sym sp key_sym {
+      def action
+        { action: :drop_primary_key }
+      end
+    }
+    / drop_key sp foreign_sym sp key_sym sp field_ident {
+      def action
+        { action: :drop_foreign_key,
+          support_level: :nonbreaking
+        }
+      end
+    }
+    / drop_key sp key_or_index sp field_ident {
+      def action
+        { action: "drop_#{key_or_index.text_value.downcase}".to_sym,
+          support_level: :nonbreaking
+        }
+      end
+    }
     / drop_key (sp col_key)? sp col_name {
       def action
         { action: :drop_column,
-          column: col_name.text_value, }
+          column: col_name.value, }
+      end
+    }
+    / enable_sym sp keys_sym {
+      def action
+        { action: :enable_keys,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+    / [^,]+ {
+      def action
+        raise "unsupported ALTER TABLE query.  Contact FlyData Support"
       end
     }
-    / [^,]+
+  end
+
+  rule key_def
+    normal_key_type key_alg nsp '(' key_list ')' normal_key_options {
+      def action
+        { action: :add_index,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+    / normal_key_type opt_ident key_alg nsp '(' key_list ')' normal_key_options {
+      def action
+        { action: :add_index,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+    / fulltext opt_key_or_index opt_ident init_key_options nsp '(' key_list ')' fulltext_key_options {
+      def action
+        { action: :add_index,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+    / spatial opt_key_or_index opt_ident init_key_options nsp '(' key_list ')' spatial_key_options {
+      def action
+        { action: :add_index,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+    / constraint_sym sp constraint_key_type key_alg nsp '(' key_list ')' normal_key_options {
+      def action
+        case constraint_key_type.value
+        when :unique
+          { action: :add_unique_constraint,
+            support_level: :nonbreaking
+          }
+        when :primary_key
+          { action: :add_primary_key_constraint,
+            keys: key_list.value
+          }
+        else
+          raise "Unsupported constraint key type `#{constraint_key_type.value}"
+        end
+      end
+    }
+    / constraint_sym sp constraint_key_type opt_ident key_alg nsp '(' key_list ')' normal_key_options {
+      def action
+        case constraint_key_type.value
+        when :unique
+          { action: :add_unique_constraint,
+            support_level: :nonbreaking
+          }
+        when :primary_key
+          { action: :add_primary_key_constraint,
+            keys: key_list.value
+          }
+        else
+          raise "Unsupported constraint key type `#{constraint_key_type.value}"
+        end
+      end
+    }
+    / opt_constraint constraint_key_type key_alg nsp '(' key_list ')' normal_key_options {
+      def action
+        case constraint_key_type.value
+        when :unique
+          { action: :add_unique_constraint,
+            support_level: :nonbreaking
+          }
+        when :primary_key
+          { action: :add_primary_key_constraint,
+            keys: key_list.value
+          }
+        else
+          raise "Unsupported constraint key type `#{constraint_key_type.value}"
+        end
+      end
+    }
+    / opt_constraint constraint_key_type opt_ident key_alg nsp '(' key_list ')' normal_key_options {
+      def action
+        case constraint_key_type.value
+        when :unique
+          { action: :add_unique_constraint,
+            support_level: :nonbreaking
+          }
+        when :primary_key
+          { action: :add_primary_key_constraint,
+            keys: key_list.value
+          }
+        else
+          raise "Unsupported constraint key type `#{constraint_key_type.value}"
+        end
+      end
+    }
+    / opt_constraint 'foreign'i key_sym opt_ident nsp '(' key_list ')' sp references
+    / constraint_sym sp check_constraint {
+      def action
+        { action: :add_check_constraint,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+    / opt_constraint check_constraint {
+      def action
+        { action: :add_check_constraint,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+  end
+
+  rule check_constraint
+    'check'i nsp '(' nsp expr nsp ')'
+  end
+
+  rule expr
+    ( !')' . )+
+  end
+
+  rule references
+    'references'i sp table_ident opt_ref_list opt_match_clause opt_on_update_delete
+  end
+
+  rule table_ident
+    ident '.' ident
+    / '.' ident
+    / ident
+  end
+
+  rule opt_ref_list
+    ( nsp '(' nsp ref_list nsp ')' )?
+  end
+
+  rule ref_list
+    ident comma ref_list
+    / ident
+  end
+
+  rule opt_match_clause
+    ( sp ( 'match'i sp 'full'i / 'match'i sp 'partial'i / 'match'i sp 'simple'i ) )?
+  end
+
+  rule opt_on_update_delete
+    ( sp (
+        'on'i sp 'update'i delete_option sp 'on'i sp 'delete'i sp delete_option
+        / 'on'i sp 'delete'i delete_option sp 'on'i sp 'update'i sp delete_option
+        / 'on'i sp 'update'i sp delete_option
+        / 'on'i sp 'delete'i sp delete_option
+      )
+    )?
+  end
+
+  rule delete_option
+    'restrict'i
+    / 'cascade'i
+    / 'set'i sp 'null'i
+    / 'no'i sp 'action'i
+    / 'set'i sp 'default'i
+  end
+
+  rule opt_constraint
+    ( constraint sp )?
+  end
+
+  rule constraint
+    constraint_sym opt_ident
+  end
+
+  rule constraint_sym
+    'constraint'i
+  end
+
+  rule constraint_key_type
+    primary_sym sp key_sym {
+      # #value returns a normalized value of the element while #text_value
+      # returns its text as is
+      def value
+        :primary_key
+      end
+    }
+    / unique_sym opt_key_or_index {
+      def value
+        :unique
+      end
+    }
+  end
+
+  rule primary_sym
+    'primary'i
+  end
+
+  rule key_sym
+    'key'i
+  end
+
+  rule keys_sym
+    'keys'i
+  end
+
+  rule unique_sym
+    'unique'i
+  end
+
+  rule index_sym
+    'index'i
+  end
+
+  rule enable_sym
+    'enable'i
+  end
+
+  rule foreign_sym
+    'foreign'i
+  end
+
+  rule fulltext
+    'fulltext'i
+  end
+
+  rule fulltext_key_options
+    ( sp fulltext_key_opts )?
+  end
+
+  rule fulltext_key_opts
+    fulltext_key_opt sp fulltext_key_opts
+    / fulltext_key_opt
+  end
+
+  rule fulltext_key_opt
+    all_key_opt
+    / 'with parser'i sp ident_sys
+  end
+
+  rule spatial
+    'spatial'i
+  end
+
+  rule spatial_key_options
+    ( sp spatial_key_opts )?
+  end
+
+  rule spatial_key_opts
+    spatial_key_opt sp spatial_key_opts
+    / spatial_key_opt
+  end
+
+  rule spatial_key_opt
+    all_key_opt
+  end
+
+  rule opt_key_or_index
+    ( sp key_or_index )?
+  end
+
+  rule normal_key_type
+    key_or_index
+  end
+
+  rule key_or_index
+    key_sym / index_sym
+  end
+
+  rule opt_ident
+    ( sp field_ident )?
+  end
+
+  rule field_ident
+    ident '.' ident '.' ident
+    / ident '.' ident
+    / '.' ident
+    / ident
+  end
+
+  rule key_alg
+    sp init_key_options key_using_alg
+    / init_key_options
+  end
+
+  rule init_key_options
+    '' # In original, this is doing some initialization
+  end
+
+  rule key_using_alg
+    'using'i sp btree_or_rtree
+    / type_sym sp btree_or_rtree
+  end
+
+  rule btree_or_rtree
+    'btree'i / 'rtree'i / 'hash'i
+  end
+
+  rule type_sym
+    'type'i
+  end
+
+  rule key_list
+    nsp key_part order_dir nsp ',' nsp key_list nsp {
+      # #value returns a normalized value of the element while #text_value
+      # returns its text as is
+      def value
+        order_dir_value = order_dir.value
+        if order_dir_value && !order_dir_value.empty?
+          order_dir_value = " " + order_dir_value
+        end
+        [ "#{key_part.value}#{order_dir_value}" ] + key_list.value
+      end
+    }
+    / nsp key_part order_dir nsp {
+      def value
+        order_dir_value = order_dir.value
+        if order_dir_value && !order_dir_value.empty?
+          order_dir_value = " " + order_dir_value
+        end
+        [ "#{key_part.value}#{order_dir_value}" ]
+      end
+    }
+  end
+
+  rule key_part
+    ident nsp '(' nsp num nsp ')' {
+      # #value returns a normalized value of the element while #text_value
+      # returns its text as is
+      def value
+        "#{ident.value}(#{num.text_value})"
+      end
+    }
+    / ident
+  end
+
+  rule order_dir
+    # Treetop creates an empty SyntaxNode if the rule results in "match none"
+    # The SyntaxNode instance does not have a custom method as deinfed here,
+    # so it causes an issue to other nodes expecting the method.
+    # 1..1 is a magic trick to force Treetop creating a SyntaxNode with
+    # the custom method.
+    (( sp ('asc'i / 'desc'i) )?) 1..1 {
+      def value
+        text_value.strip.downcase
+      end
+    }
+  end
+
+  rule normal_key_options
+    ( sp normal_key_opts )?
+  end
+
+  rule normal_key_opts
+    normal_key_opt sp normal_key_opts
+    / normal_key_opt
+  end
+
+  rule normal_key_opt
+    all_key_opt
+    / key_using_alg
+  end
+
+  rule all_key_opt
+    key_block_size nsp equal nsp ulong_num
+    / key_block_size sp ulong_num
+    / comment_opt
+  end
+
+  rule opt_equal
+    ( nsp equal )?
+  end
+
+  rule equal
+    '='
+    / ':='
+  end
+
+  rule key_block_size
+    'key_block_size'i
   end
 
   rule col_name_def
     col_name sp col_def {
       def column_def
-        {column: col_name.text_value}.merge(col_def.column_def)
+        {column: col_name.value}.merge(col_def.column_def)
       end
     }
   end
@@ -105,7 +516,7 @@ grammar MysqlAlterTable
     }
     / 'after'i sp col_name {
       def pos_def_option
-        { after: col_name.text_value, }
+        { after: col_name.value, }
       end
     }
   end
@@ -158,7 +569,7 @@ grammar MysqlAlterTable
   end
 
   rule data_type_name
-    ident ''
+    ident_sym ''
   end
 
   rule meta_text
@@ -248,6 +659,14 @@ grammar MysqlAlterTable
     [0-9]* '.' [0-9]+ { def text_value; v = super; v.start_with?('.') ? "0#{v}" : v; end }
     / [0-9]+
   end
+  rule num
+    [0-9]+
+  end
+
+  rule ulong_num
+    [0-9]+ # TODO May not be the correct definition
+  end
+
   rule auto_increment_opt
     'auto_increment'i
   end
@@ -325,7 +744,7 @@ grammar MysqlAlterTable
 
   rule value
     quoted_value { def raw_value; text_raw_value; end }
-    / ident { def raw_value; text_value; end }
+    / ident_sym { def raw_value; text_value; end }
   end
 
   rule quoted_value
@@ -333,21 +752,41 @@ grammar MysqlAlterTable
   end
 
   rule text
-    ("\\\\" / "\\'" / !"'" .)*
+    ("\\\\" / "\\'" / !"'" . )*
+  end
+
+  rule ident
+    ident_sys / keyword
+  end
+
+  rule keyword
+    ([a-zA-Z_]+) 1..1 { # TODO list all keywords
+      def value
+        text_value
+      end
+    }
   end
 
   rule ident_sys
-    ident / ident_quoted
+    ident_sym / ident_quoted
   end
 
-  rule ident
-    [0-9a-zA-Z_]+
+  rule ident_sym
+    # 1..1 prevents Treetop from embedding the SyntaxNode to an upper node
+    # without the custom method.
+    ([0-9a-zA-Z_]+) 1..1 {
+      def value
+        text_value
+      end
+    }
   end
 
   rule ident_quoted
-    '`' ident '`' {
-      def text_value
-        ident.text_value
+    '`' ident_sym '`' {
+      # #value returns a normalized value of the element while #text_value
+      # returns its text as is
+      def value
+        ident_sym.value
       end
     }
   end