flydata 0.2.10 → 0.2.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: acf1cc6cb8257d42635142a15b10f63a7bf276b4
-  data.tar.gz: 240b9f073f79c4a0fedc4dc1d454d8fbcba05077
+  metadata.gz: 1cf10dc736cbd2f2712294f21cf074728aab1be8
+  data.tar.gz: aba35db4a5431b33a673f4f1708601eeeaa46507
 SHA512:
-  metadata.gz: 5d4c6dad5c51496f6b1748d112c9ac5851ae8ba1dc8db31045ec11d7bce44deff48f904c80ae04ec736330cb432b2da4a538879e4a87b1b49bc3313f9146758c
-  data.tar.gz: 2b019366d7201f3d1a76367267d3f4a4189408bad14d650240f726eafdf1fd2e5eb8ff5b48b624790b0fb07f1fe705634fc95b4582a233e9699cc8c2eb15f156
+  metadata.gz: 90939617163c3050e124607859340cc0390242a4984451bfcfb383581a8acca374853263b8358c95f5297548984fa0c15fa4bb06a70f171aed0f99e2eca7fa37
+  data.tar.gz: f69a0c50a94ba780577fcc5877e10d4d2059f6de3ffaed1145b6509ca11b21500a7ea5a36cab7f47c559023e4b23ed25612c50c31c6ca7e3ed29c7262fd0c472

data/Gemfile

@@ -25,4 +25,5 @@ group :development do
   gem 'activemodel', '~> 4.0', '>= 4.0.0'
   gem 'activerecord', '~> 4.0', '>= 4.0.0'
   gem 'protected_attributes', '~> 1.0', '>= 1.0.8'
+  gem 'pry'
 end

data/Gemfile.lock

@@ -20,6 +20,7 @@ GEM
     addressable (2.3.6)
     arel (4.0.2)
     builder (3.1.4)
+    coderay (1.1.0)
     cool.io (1.2.4)
     diff-lcs (1.2.5)
     faraday (0.8.9)
@@ -60,6 +61,7 @@ GEM
     jwt (1.0.0)
     kodama (0.1.1)
       ruby-binlog (>= 0.1.9)
+    method_source (0.8.2)
     mime-types (2.3)
     minitest (4.7.5)
     msgpack (0.5.8)
@@ -78,6 +80,10 @@ GEM
     polyglot (0.3.5)
     protected_attributes (1.0.8)
       activemodel (>= 4.0.1, < 5.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
     rack (1.5.2)
     rake (10.3.2)
     rdoc (4.1.1)
@@ -124,6 +130,7 @@ DEPENDENCIES
   json (~> 1.8, >= 1.8.0)
   mysql2 (~> 0.3, >= 0.3.11)
   protected_attributes (~> 1.0, >= 1.0.8)
+  pry
   rest-client (~> 1.6, >= 1.6.7)
   rspec (~> 3.0)
   ruby-binlog (~> 1.0, >= 1.0.1)

data/VERSION

@@ -1 +1 @@
-0.2.10
+0.2.11

data/benchmark/benchmark_helper.rb

@@ -0,0 +1,4 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'flydata'
+require 'benchmark'

data/benchmark/insert_parser_bench.rb

@@ -0,0 +1,28 @@
+require_relative 'benchmark_helper'
+require 'flydata/parser/mysql/dump_parser'
+require 'zlib'
+
+INSERT_TEST_FILE_NAME = case ARGV[0]
+                        when 'num'
+                          'insert_parser_test_data_num_only.sql.gz'
+                        else
+                          'insert_parser_test_data.sql.gz'
+                        end
+INSERT_TEST_FILE_PATH = File.realpath("data/#{INSERT_TEST_FILE_NAME}", File.dirname(__FILE__))
+def readline_gz_file(gz_file_path = INSERT_TEST_FILE_PATH)
+  Zlib::GzipReader.open(gz_file_path) {|f| return f.readline}
+end
+
+@test_line = readline_gz_file
+
+def subject
+  Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
+  Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
+  Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
+  Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
+  Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
+end
+
+Benchmark.bm do |x|
+  x.report { 10.times{ subject } }
+end

data/benchmark/insert_parser_prof.rb

@@ -0,0 +1,24 @@
+require_relative 'benchmark_helper'
+require 'flydata/parser/mysql/dump_parser'
+require 'zlib'
+require 'ruby-prof'
+
+INSERT_TEST_FILE_NAME = case ARGV[0]
+                        when 'num'
+                          'insert_parser_test_data_num_only.sql.gz'
+                        else
+                          'insert_parser_test_data.sql.gz'
+                        end
+INSERT_TEST_FILE_PATH = File.realpath("data/#{INSERT_TEST_FILE_NAME}", File.dirname(__FILE__))
+
+def readline_gz_file(gz_file_path = INSERT_TEST_FILE_PATH)
+  Zlib::GzipReader.open(gz_file_path) {|f| return f.readline}
+end
+
+@test_line = readline_gz_file
+
+def subject
+  Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
+end
+
+subject

data/flydata.gemspec

@@ -2,16 +2,14 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.2.10 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.10"
+  s.version = "0.2.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2014-10-27"
+  s.date = "2014-10-31"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -22,6 +20,11 @@ Gem::Specification.new do |s|
     "Gemfile.lock",
     "Rakefile",
     "VERSION",
+    "benchmark/benchmark_helper.rb",
+    "benchmark/data/insert_parser_test_data.sql.gz",
+    "benchmark/data/insert_parser_test_data_num_only.sql.gz",
+    "benchmark/insert_parser_bench.rb",
+    "benchmark/insert_parser_prof.rb",
     "bin/fdmysqldump",
     "bin/flydata",
     "bin/serverinfo",
@@ -115,7 +118,8 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.2.2"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.0.14"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then
@@ -141,6 +145,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<activemodel>, [">= 4.0.0", "~> 4.0"])
       s.add_development_dependency(%q<activerecord>, [">= 4.0.0", "~> 4.0"])
       s.add_development_dependency(%q<protected_attributes>, [">= 1.0.8", "~> 1.0"])
+      s.add_development_dependency(%q<pry>, [">= 0"])
     else
       s.add_dependency(%q<rest-client>, [">= 1.6.7", "~> 1.6"])
       s.add_dependency(%q<i18n>, [">= 0.6.5", "~> 0.6"])
@@ -161,6 +166,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<activemodel>, [">= 4.0.0", "~> 4.0"])
       s.add_dependency(%q<activerecord>, [">= 4.0.0", "~> 4.0"])
       s.add_dependency(%q<protected_attributes>, [">= 1.0.8", "~> 1.0"])
+      s.add_dependency(%q<pry>, [">= 0"])
     end
   else
     s.add_dependency(%q<rest-client>, [">= 1.6.7", "~> 1.6"])
@@ -182,6 +188,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<activemodel>, [">= 4.0.0", "~> 4.0"])
     s.add_dependency(%q<activerecord>, [">= 4.0.0", "~> 4.0"])
     s.add_dependency(%q<protected_attributes>, [">= 1.0.8", "~> 1.0"])
+    s.add_dependency(%q<pry>, [">= 0"])
   end
 end
 

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -58,7 +58,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     load_custom_conf
     $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
     $log.info "mysql client version: #{`mysql -V`}"
-    server_version = `echo 'select version();' | mysql -h #{@host} --port #{@port} -u #{@username} -p#{@password} 2>/dev/null`
+    server_version = `echo 'select version();' | MYSQL_PWD="#{@password}" mysql -h #{@host} --port #{@port} -u #{@username} 2>/dev/null`
     $log.info "mysql server version: #{server_version}"
 
     @tables = @tables.split(/,\s*/)

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -428,15 +428,6 @@ EOS
         #   ex) INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
         class InsertParser
           #INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
-          module State
-            IN_VALUE = 'IN_VALUE'
-            NEXT_VALUES = 'NEXT_VALUES'
-          end
-
-          def initialize
-            @values = []
-            @values_set = []
-          end
 
           def start_ruby_prof
             RubyProf.start if defined?(RubyProf) and not RubyProf.running?
@@ -466,6 +457,9 @@ EOS
           private
 
           def _parse(target_line)
+            values = []
+            values_set = []
+
             target_line = target_line.strip
             start_index = target_line.index('(')
             target_line = target_line[start_index..-2]
@@ -477,48 +471,49 @@ EOS
             # 'String#each_char' is twice as slow as the current storategy.
             items = target_line.split(',')
             index = 0
-            cur_state = State::NEXT_VALUES
+            cur_state = :next_values
 
             loop do
               case cur_state
-              when State::NEXT_VALUES
+              when :next_values
                 chars = items[index]
                 break unless chars
                 items[index] = chars[1..-1]
-                cur_state = State::IN_VALUE
-              when State::IN_VALUE
+                cur_state = :in_value
+              when :in_value
                 chars = items[index]
                 index += 1
                 if chars.start_with?("'")
+                  chars_size = chars.size
                   # single item (not last item)
                   # size check added below otherwise end_with? matches the single quote which was also used by start_with?
-                  if chars.size > 1 and chars.end_with?("'") and !last_char_escaped?(chars)
-                    @values << replace_escape_char(chars[1..-2])
+                  if chars_size > 1 and chars.end_with?("'") and !(chars.end_with?("\\'") and last_char_escaped?(chars))
+                    values << (chars[1..-2]).gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
                     # single item (last item)
                     # size check added below otherwise end_with? matches the single quote which was also used by start_with?
-                  elsif chars.size > 2 and chars.end_with?("')") and !last_char_escaped?(chars[0..-2])
-                    @values << replace_escape_char(chars[1..-3])
-                    @values_set << @values
-                    @values = []
-                    cur_state = State::NEXT_VALUES
+                  elsif chars_size > 2 and chars.end_with?("')") and !(chars[0..-2].end_with?("\\'") and last_char_escaped?(chars[0..-2]))
+                    values << (chars[1..-3]).gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
+                    values_set << values
+                    values = []
+                    cur_state = :next_values
                     # multi items
                   else
                     cur_value = chars[1..-1]
                     loop do
                       next_chars = items[index]
                       index += 1
-                      if next_chars.end_with?('\'') and !last_char_escaped?(next_chars)
+                      if next_chars.end_with?('\'') and !(next_chars.end_with?("\\'") and last_char_escaped?(next_chars))
                         cur_value << ','
                         cur_value << next_chars[0..-2]
-                        @values << replace_escape_char(cur_value)
+                        values << (cur_value).gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
                         break
-                      elsif next_chars.end_with?("')") and !last_char_escaped?(next_chars[0..-2])
+                      elsif next_chars.end_with?("')") and !(next_chars[0..-2].end_with?("\\'") and last_char_escaped?(next_chars[0..-2]))
                         cur_value << ','
                         cur_value << next_chars[0..-3]
-                        @values << replace_escape_char(cur_value)
-                        @values_set << @values
-                        @values = []
-                        cur_state = State::NEXT_VALUES
+                        values << (cur_value).gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
+                        values_set << values
+                        values = []
+                        cur_state = :next_values
                         break
                       else
                         cur_value << ','
@@ -529,26 +524,28 @@ EOS
                 else
                   if chars.end_with?(')')
                     chars = chars[0..-2]
-                    @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
-                    @values_set << @values
-                    @values = []
-                    cur_state = State::NEXT_VALUES
+                    values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
+                    values_set << values
+                    values = []
+                    cur_state = :next_values
                   else
-                    @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
+                    values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
                   end
                 end
               else
                 raise "Invalid state: #{cur_state}"
               end
             end
-            return @values_set
+            return values_set
           end
 
+          ESCAPE_REGEXP = /\\\\|\\'|\\"|\\n|\\r/
           ESCAPE_HASH_TABLE = {"\\\\" => "\\", "\\'" => "'", "\\\"" => "\"", "\\n" => "\n", "\\r" => "\r"}
 
-          def replace_escape_char(original)
-            original.gsub(/\\\\|\\'|\\"|\\n|\\r/, ESCAPE_HASH_TABLE)
-          end
+          # For the peformance, this function is put inline.
+          #def replace_escape_char(original)
+          #  original.gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
+          #end
 
           # This method assume that the last character is '(single quotation)
           # abcd\' -> true

data/lib/flydata/parser/mysql/mysql_alter_table.treetop

@@ -54,39 +54,46 @@ grammar MysqlAlterTable
          }]
       end
     }
-    /
-    alter_specs (sp partition_opts)? {
+    / ( partition_rules ) 1..1 { # This does not exist in sql_yacc.yy
       def actions
-        alter_specs.actions
+        action = elements[0].action
+        action[:query] = text_value
+        action[:support_level] = :nonbreaking
+        [action]
+      end
+    }
+    / alter_list (sp partition_opts)? {
+      def actions
+        alter_list.actions
       end
     }
   end
 
-  rule alter_specs
-    alter_spec ( comma alter_spec )* {
+  rule alter_list
+    alter_list_item ( comma alter_list_item )* {
       def actions
         ret = []
-        ret.concat(actions_with_query(alter_spec))
+        ret.concat(actions_with_query(alter_list_item))
         0.upto(elements[1].elements.size - 1) do |i|
           ret.concat(actions_with_query(elements[1].elements[i].elements[1]))
         end
         ret
       end
 
-      def actions_with_query(alter_spec)
-        action = alter_spec.action
+      def actions_with_query(alter_list_item)
+        action = alter_list_item.action
         if action.kind_of?(Array)
           actions = action
         else
           actions = [action]
         end
-        actions.each {|a| a[:query] = alter_spec.text_value }
+        actions.each {|a| a[:query] = alter_list_item.text_value }
         actions
       end
     }
   end
 
-  rule alter_spec
+  rule alter_list_item
     add_key sp key_def {
       def action
         action_hash = key_def.action
@@ -730,6 +737,10 @@ grammar MysqlAlterTable
     [0-9]+ # TODO May not be the correct definition
   end
 
+  rule real_ulong_num
+    [0-9]+ # TODO - May have to expand later
+  end
+
   rule auto_increment_opt
     'auto_increment'i
   end
@@ -1018,4 +1029,273 @@ grammar MysqlAlterTable
   rule dot
     '.'
   end
+
+  ## Partition related rules
+
+  rule partition_rules
+    add_partition_rule {
+      def action
+        { action: :add_partition }
+      end
+    }
+    / drop_sym sp partition_sym sp alt_part_name_list {
+      def action
+        { action: :drop_partition }
+      end
+    }
+    / rebuild_sym sp partition_sym opt_no_write_to_binlog sp all_or_alt_part_name_list {
+      def action
+        { action: :rebuild_partition }
+      end
+    }
+    / optimize_sym sp partition_sym opt_no_write_to_binlog sp all_or_alt_part_name_list {
+      def action
+        { action: :optimize_partition }
+      end
+    }
+    / analyze_sym sp partition_sym opt_no_write_to_binlog sp all_or_alt_part_name_list {
+      def action
+        { action: :analyze_partition }
+      end
+    }
+    / check_sym sp partition_sym sp all_or_alt_part_name_list opt_mi_check_type {
+      def action
+        { action: :check_partition }
+      end
+    }
+    / repair_sym sp partition_sym opt_no_write_to_binlog sp all_or_alt_part_name_list opt_mi_repair_type {
+      def action
+        { action: :repair_partition }
+      end
+    }
+    / coalesce_sym sp partition_sym opt_no_write_to_binlog sp real_ulong_num {
+      def action
+        { action: :coalesce_partition }
+      end
+    }
+    / truncate_sym sp partition_sym sp all_or_alt_part_name_list {
+      def action
+        { action: :truncate_partition }
+      end
+    }
+    / reorg_partition_rule {
+      def action
+        { action: :reorganize_partition }
+      end
+    }
+    / exchange_sym sp partition_sym sp alt_part_name_item sp
+      with_sym sp table_sym sp table_ident opt_validation {
+      def action
+        { action: :exchange_partition }
+      end
+    }
+    / discard_sym sp partition_sym sp all_or_alt_part_name_list sp tablespace_sym {
+      def action
+        { action: :discard_partition_tablespace }
+      end
+    }
+    / import_sym sp partition_sym sp all_or_alt_part_name_list sp tablespace_sym {
+      def action
+        { action: :import_partition_tablespace }
+      end
+    }
+    / remove_partitioning {
+      def action
+        { action: :remove_partitioning }
+      end
+    }
+  end
+
+  rule remove_partitioning
+    remove_sym sp partitioning_sym
+  end
+
+  rule add_partition_rule
+    add_sym sp partition_sym opt_no_write_to_binlog nsp add_part_extra
+  end
+
+  rule reorg_partition_rule
+    reorganize_sym sp partition_sym opt_no_write_to_binlog nsp reorg_parts_rule
+  end
+
+  rule opt_validation
+    ( sp (with_sym / without_sym) sp validation_sym )?
+  end
+
+  rule remove_sym
+    'remove'i ![A-Za-z0-9_]
+  end
+
+  rule all_sym
+    'all'i ![A-Za-z0-9_]
+  end
+
+  rule check_sym
+    'check'i ![A-Za-z0-9_]
+  end
+
+  rule table_sym
+    'table'i ![A-Za-z0-9_]
+  end
+
+  rule with_sym
+    'with'i ![A-Za-z0-9_]
+  end
+
+  rule without_sym
+    'without'i ![A-Za-z0-9_]
+  end
+
+  rule validation_sym
+    'validation'i ![A-Za-z0-9_]
+  end
+
+  rule exchange_sym
+    'exchange'i ![A-Za-z0-9_]
+  end
+
+  rule reorganize_sym
+    'reorganize'i ![A-Za-z0-9_]
+  end
+
+  rule truncate_sym
+    'truncate'i ![A-Za-z0-9_]
+  end
+
+  rule coalesce_sym
+    'coalesce'i ![A-Za-z0-9_]
+  end
+
+  rule repair_sym
+    'repair'i ![A-Za-z0-9_]
+  end
+
+  rule analyze_sym
+    'analyze'i ![A-Za-z0-9_]
+  end
+
+  rule optimize_sym
+    'optimize'i ![A-Za-z0-9_]
+  end
+
+  rule rebuild_sym
+    'rebuild'i ![A-Za-z0-9_]
+  end
+
+  rule drop_sym
+    'drop'i ![A-Za-z0-9_]
+  end
+
+  rule add_sym
+    'add'i ![A-Za-z0-9_]
+  end
+
+  rule partition_sym
+    'partition'i ![A-Za-z0-9_]
+  end
+
+  rule partitioning_sym
+    'partitioning'i ![A-Za-z0-9_]
+  end
+
+  rule local_sym
+    'local'i ![A-Za-z0-9_]
+  end
+
+  rule for_sym
+    'for'i ![A-Za-z0-9_]
+  end
+
+  rule upgrade_sym
+    'upgrade'i ![A-Za-z0-9_]
+  end
+
+  rule changed_sym
+    'changed'i ![A-Za-z0-9_]
+  end
+
+  rule extended_sym
+    'extended'i ![A-Za-z0-9_]
+  end
+
+  rule medium_sym
+    'medium'i ![A-Za-z0-9_]
+  end
+
+  rule fast_sym
+    'fast'i ![A-Za-z0-9_]
+  end
+
+  rule quick_sym
+    'quick'i ![A-Za-z0-9_]
+  end
+
+  rule use_frm_sym
+    'use_frm'i ![A-Za-z0-9_]
+  end
+
+  # query we parse may never have this, including for the sake of completeness
+  rule no_write_to_binlog_sym
+    'no_write_to_binlog'i ![A-Za-z0-9_]
+  end
+
+  rule opt_no_write_to_binlog
+    ( sp (no_write_to_binlog_sym / local_sym) )?
+  end
+
+  # Generic implementation(matches anything). Should be modified when required
+  rule add_part_extra
+    .*
+  end
+
+  # Generic implementation (matches anything). Should be modified when required
+  rule reorg_parts_rule
+    .*
+  end
+
+  rule all_or_alt_part_name_list
+    all_sym / alt_part_name_list
+  end
+
+  rule alt_part_name_list
+    alt_part_name_item comma alt_part_name_list
+    / alt_part_name_item
+  end
+
+  rule alt_part_name_item
+    ident
+  end
+
+  rule opt_mi_check_type
+    ( sp mi_check_types )?
+  end
+
+  rule mi_check_types
+    mi_check_type sp mi_check_types
+    / mi_check_type
+  end
+
+  rule mi_check_type
+    quick_sym
+    / fast_sym
+    / medium_sym
+    / extended_sym
+    / changed_sym
+    / for_sym sp upgrade_sym
+  end
+
+  rule opt_mi_repair_type
+    ( sp mi_repair_types )?
+  end
+
+  rule mi_repair_types
+    mi_repair_type sp mi_repair_types
+    / mi_repair_type
+  end
+
+  rule mi_repair_type
+    quick_sym
+    / extended_sym
+    / use_frm_sym
+  end
 end

data/lib/flydata/table_def/redshift_table_def.rb

@@ -119,13 +119,21 @@ EOS
     line
   end
 
+  NULL_STR = "NULL"
+
   def self.replace_default_value(type, default_value)
-    val = default_value ? "'#{default_value}'" : "NULL"
+    return NULL_STR if default_value.nil?
     case type
     when 'timestamp'
-      (val.upcase == "'CURRENT_TIMESTAMP'") ? 'SYSDATE' : val
+      if default_value.upcase == "CURRENT_TIMESTAMP"
+        'SYSDATE'
+      else
+        "'#{self.parse_timestamp(default_value)}'"
+      end
+    when 'date'
+      "'#{self.parse_date(default_value)}'"
     else
-      val
+      "'#{default_value}'"
     end
   end
 
@@ -165,7 +173,7 @@ EOS
   def self.escape(text)
     text.gsub("'", "\\\\'")
   end
-  
+
   def self.check_and_replace_max(params, max_size_a)
       final_params = []
       params.split(",").each_with_index do |param, i|
@@ -174,6 +182,38 @@ EOS
       end
       final_params.join(",")
   end
+
+  APACHE_TIMESTAMP_REGEXP = Regexp.new('^(?<apache_time_format>\[[0-3]\d\/\D{3}\/[1-2]\d{3}:[0-2]\d:[0-5]\d:[0-5]\d ?[\+\-]\d{2}:?\d{2}\])$')
+
+  def self.parse_timestamp(value)
+    if value.kind_of?(Integer) or /^\d+$/ === value
+      # Unix epoch in UTC
+      t = DateTime.strptime(value.to_s, '%s')
+    elsif APACHE_TIMESTAMP_REGEXP.match(value)
+      # apache time format
+      t = DateTime.strptime(value, "[%d/%b/%Y:%H:%M:%S %Z]")
+    else
+      t = DateTime.parse(value)
+    end
+    t = t.new_offset(0) # Redshift Plug-in uses UTC
+    t.strftime('%Y-%m-%d %H:%M:%S.%6N')
+  rescue ArgumentError => ae
+    # '0000-00-00 00:00:00' is valid for mysql datetime column
+    if value.start_with?('0000-00-00 00:00:00')
+      return '0001-01-01 00:00:00.000000'
+    else
+      raise ae
+    end
+  end
+
+  def self.parse_date(value)
+    dt = Date.parse(value)
+    dt.strftime('%Y-%m-%d')
+  rescue ArgumentError => ae
+    # '0000-00-00' is valid for mysql date column
+    return '0001-01-01' if value == '0000-00-00'
+    raise ae
+  end
 end
 
 end

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -225,6 +225,8 @@ EOT
       setup_initial_flydata_files
       allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
       allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
+      allow(plugin).to receive(:`).with("mysql -V").and_return("mysql  Ver 14.14 Distrib 5.5.40")
+      allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
       Timecop.freeze(now)
     end
 

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -860,6 +860,90 @@ describe 'MysqlAlterTableParser' do
         end
       end
     end
+    shared_examples "test optional no_write_to_binlog" do |*examples|
+      context "with no_write_to_binlog" do
+        it_behaves_like *examples do
+          let(:no_write_to_binlog){" no_write_to_binlog"}
+        end
+      end
+      context "with local" do
+        it_behaves_like *examples do
+          let(:no_write_to_binlog){" LOCAL"}
+        end
+      end
+      context "without local or no_write_to_binlog flag" do
+        it_behaves_like *examples do
+          let(:no_write_to_binlog){""}
+        end
+      end
+    end
+    shared_examples "test partition names" do |*examples|
+      context "ALL" do
+        it_behaves_like *examples do
+          let(:partition_names){"ALL"}
+        end
+      end
+      context "ALL lower case" do
+        it_behaves_like *examples do
+          let(:partition_names){"all"}
+        end
+      end
+      context "single partition" do
+        it_behaves_like *examples do
+          let(:partition_names){"p0"}
+        end
+      end
+      context "multiple partitions" do
+        it_behaves_like *examples do
+          let(:partition_names){"p0,P1, p2"}
+        end
+      end
+    end
+    shared_examples "test check types" do |*examples|
+      ["quick", "FAST", "medium", "EXTENDED", "CHANGED", "for upgrade"].each do |check_type|
+        context "check_type is #{check_type}" do
+          it_behaves_like *examples do
+            let(:check_type) {check_type}
+          end
+        end
+      end
+      context "multiple check types" do
+        it_behaves_like *examples do
+          let(:check_type) {"fast for upgrade"}
+        end
+      end
+    end
+    shared_examples "test repair types" do |*examples|
+      ["quick", "EXTENDED", "use_frm"].each do |repair_type|
+        context "repair_type is #{repair_type}" do
+          it_behaves_like *examples do
+            let(:repair_type) {repair_type}
+          end
+        end
+      end
+      context "multiple repair types" do
+        it_behaves_like *examples do
+          let(:repair_type) {"quick USE_FRM"}
+        end
+      end
+    end
+    shared_examples "test optional validation" do |*examples|
+      context "with validation" do
+        it_behaves_like *examples do
+          let(:validation){" with validation"}
+        end
+      end
+      context "without validation" do
+        it_behaves_like *examples do
+          let(:validation){" WITHOUT VALIDATION"}
+        end
+      end
+      context "none" do
+        it_behaves_like *examples do
+          let(:validation){""}
+        end
+      end
+    end
     context 'add index' do
       let(:action) { :add_index }
       context "with a simple add index" do
@@ -1138,5 +1222,116 @@ describe 'MysqlAlterTableParser' do
       let(:action) { :import_tablespace }
       it_behaves_like "a parser parsing a breaking query"
     end
+    context "add partition" do
+      let(:action) { :add_partition }
+      context "simple definition with spaces" do
+        let(:alter_table_action) { "add partition#{no_write_to_binlog} (PARTITION p2)" }
+        it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+      end
+      context "complex partition definition without spaces" do
+        let(:alter_table_action) { "ADD partition#{no_write_to_binlog}(PARTITION p2 VALUES LESS THAN (800))" }
+        it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+      end
+    end
+    context "drop partition" do
+      let(:action) { :drop_partition }
+      context "drop single partition" do
+        let(:alter_table_action) { "DROP partition p0" }
+        it_behaves_like "a parser parsing a nonbreaking query"
+      end
+      context "drop multiple partitions" do
+        context "upcase and spaces between partition names" do
+          let(:alter_table_action) { "DROP PARTITION p0, p1" }
+          it_behaves_like "a parser parsing a nonbreaking query"
+        end
+        context "lowercase and no spaces between partition names" do
+          let(:alter_table_action) { "drop partition p0,p1,p2" }
+          it_behaves_like "a parser parsing a nonbreaking query"
+        end
+      end
+    end
+    context "rebuild, optimize, analyze partitions" do
+      ["rebuild", "optimize", "analyze"].each do |op|
+        context "#{op} partition" do
+          let(:action) { "#{op}_partition".to_sym }
+          context "with upcase op" do
+            let(:alter_table_action) { "#{op.upcase} PARTITION#{no_write_to_binlog} #{partition_names}" }
+            it_behaves_like "test partition names", "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+          end
+          context "with lowercase op" do
+            let(:alter_table_action) { "#{op} partition#{no_write_to_binlog} #{partition_names}" }
+            it_behaves_like "test partition names", "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+          end
+        end
+      end
+    end
+    context "check partition" do
+      let(:action) { :check_partition }
+      context "without check_type" do
+        let(:alter_table_action) { "CHECK PARTITION #{partition_names}" }
+        it_behaves_like "test partition names", "a parser parsing a nonbreaking query"
+      end
+      context "with check type" do
+        let(:alter_table_action) { "check partition #{partition_names} #{check_type}" }
+        it_behaves_like "test check types", "test partition names", "a parser parsing a nonbreaking query"
+      end
+    end
+    context "repair partition" do
+      let(:action) { :repair_partition }
+      context "without repair_type" do
+        let(:alter_table_action) { "REPAIR PARTITION#{no_write_to_binlog} #{partition_names}" }
+        it_behaves_like "test partition names", "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+      end
+      context "with repair type" do
+        let(:alter_table_action) { "repair partition#{no_write_to_binlog} #{partition_names} #{repair_type}" }
+        it_behaves_like "test repair types", "test partition names", "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+      end
+    end
+    context "coalesce partition" do
+      let(:action) { :coalesce_partition }
+      context "single digit partition number" do
+        let(:alter_table_action) { "coalesce partition#{no_write_to_binlog} 1" }
+        it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+      end
+      context "double digit partition number" do
+        let(:alter_table_action) { "COALESCE PARTITION#{no_write_to_binlog} 22" }
+        it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+      end
+    end
+    context "truncate partition" do
+      let(:action) { :truncate_partition }
+      let(:alter_table_action) { "TRUNCATE PARTITION #{partition_names}" }
+      it_behaves_like "test partition names", "a parser parsing a nonbreaking query"
+    end
+    context "reorganize partition" do
+      let(:action) { :reorganize_partition }
+      context "simple definition without partition names and partition definition" do
+        let(:alter_table_action) { "reorganize partition#{no_write_to_binlog}" }
+        it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+      end
+      context "complex partition definition" do
+        let(:alter_table_action) { "REORGANIZE PARTITION#{no_write_to_binlog} p0 INTO (PARTITION n0 VALUES LESS THAN (1960))" }
+        it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
+      end
+    end
+    context "exchange partition" do
+      let(:action) { :exchange_partition }
+      let(:alter_table_action) { "EXCHANGE PARTITION #{ident} with table#{identifier}#{validation}" }
+      it_behaves_like "test table identifier", "test optional validation", "a parser parsing a nonbreaking query"
+    end
+    context "discard and import partition tablespace" do
+      ["discard", "import"].each do |op|
+        context "#{op}" do
+          let(:action) {"#{op}_partition_tablespace".to_sym}
+          let(:alter_table_action) {"#{op} PARTITION #{partition_names} TABLESPACE"}
+          it_behaves_like "test partition names", "a parser parsing a nonbreaking query"
+        end
+      end
+    end
+    context "remove partitioning" do
+      let(:action) {:remove_partitioning}
+      let(:alter_table_action) { "REMOVE PARTITIONING" }
+      it_behaves_like "a parser parsing a nonbreaking query"
+    end
   end
 end

data/spec/flydata/table_def/redshift_table_def_spec.rb

@@ -262,17 +262,52 @@ EOT
         it_behaves_like *examples
       end
 
-      context 'with datetime column def' do
-        let(:default_value) { '2014-10-17 22:22:22' }
-        let(:default_value_sql) { "'#{default_value}'" }
+      context 'with date column def' do
+        before do
+          column[:column] = "value_date"
+          column[:type] = "date"
+        end
+        let(:type_sql) { %Q|"value_date" date| }
         let(:not_null_default_sql) { " DEFAULT '0000-01-01'" }
+
+        context 'when default_value is normal' do
+          let(:default_value) { '2014-10-17' }
+          let(:default_value_sql) { "'#{default_value}'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 0000-00-00' do
+          let(:default_value) { '0000-00-00' }
+          let(:default_value_sql) { "'0001-01-01'" }
+          it_behaves_like *examples
+        end
+      end
+
+      context 'with datetime column def' do
         before do
           column[:column] = "value_datetime"
           column[:type] = "datetime"
         end
         let(:type_sql) { %Q|"value_datetime" timestamp| }
+        let(:not_null_default_sql) { " DEFAULT '0000-01-01'" }
 
-        it_behaves_like *examples
+        context 'when default_value is normal' do
+          let(:default_value) { '2014-10-17 22:22:22' }
+          let(:default_value_sql) { "'#{default_value}.000000'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is CURRENT_TIMESTAMP' do
+          let(:default_value) { 'CURRENT_TIMESTAMP' }
+          let(:default_value_sql) { "SYSDATE" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 0000-00-00 00:00:00' do
+          let(:default_value) { '0000-00-00 00:00:00' }
+          let(:default_value_sql) { "'0001-01-01 00:00:00.000000'" }
+          it_behaves_like *examples
+        end
       end
 
       context 'with decimal column def' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.2.11
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-27 00:00:00.000000000 Z
+date: 2014-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -376,6 +376,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: FlyData Agent
 email: sysadmin@flydata.com
 executables:
@@ -391,6 +405,11 @@ files:
 - Gemfile.lock
 - Rakefile
 - VERSION
+- benchmark/benchmark_helper.rb
+- benchmark/data/insert_parser_test_data.sql.gz
+- benchmark/data/insert_parser_test_data_num_only.sql.gz
+- benchmark/insert_parser_bench.rb
+- benchmark/insert_parser_prof.rb
 - bin/fdmysqldump
 - bin/flydata
 - bin/serverinfo
@@ -501,7 +520,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: FlyData Agent