flydata 0.1.0 → 0.1.1

data/Gemfile

@@ -1,4 +1,5 @@
 source "http://rubygems.org"
+source "https://rubygems.flydata.com"
 
 gem "rest-client", "~> 1.6.7"
 gem "i18n", "~> 0.6.5"
@@ -6,7 +7,7 @@ gem "activesupport", "~> 4.0.0"
 gem "json", "~> 1.8.0"
 gem "highline", "~> 1.6.19"
 gem "fluentd", "0.10.35"
-gem "ruby-binlog", ">= 1.0.0", :git => "https://github.com/hapyrus/ruby-binlog.git"
+gem "ruby-binlog", ">= 1.0.0"
 gem "fluent-plugin-mysql-binlog", "~> 0.0.2"
 gem "mysql2", "~> 0.3.11"
 

data/Gemfile.lock

@@ -1,11 +1,6 @@
-GIT
-  remote: https://github.com/hapyrus/ruby-binlog.git
-  revision: e381b4fc411ca2c2402aefc25809ac3c97d75f94
-  specs:
-    ruby-binlog (1.0.0)
-
 GEM
   remote: http://rubygems.org/
+  remote: https://rubygems.flydata.com/
   specs:
     activemodel (4.0.0)
       activesupport (= 4.0.0)
@@ -100,6 +95,7 @@ GEM
     rspec-expectations (2.14.3)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.3)
+    ruby-binlog (1.0.0)
     ruby-prof (0.14.2)
     sqlite3 (1.3.8)
     thread_safe (0.1.3)
@@ -126,7 +122,7 @@ DEPENDENCIES
   protected_attributes
   rest-client (~> 1.6.7)
   rspec
-  ruby-binlog (>= 1.0.0)!
+  ruby-binlog (>= 1.0.0)
   ruby-prof
   sqlite3
   timecop

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/bin/fdmysqldump

@@ -46,13 +46,13 @@ command += " " + ARGV.join(' ')
 IO.popen(command, 'r') do |io|
   create_flydata_ctl_table = true
   while !io.eof?
-    mysql_tabledef = Flydata::Sync::MysqlTableDef.create(io)
+    mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(io)
     if mysql_tabledef.nil?
       # stream had no more create table definition
       break
     end
     flydata_tabledef = mysql_tabledef.to_flydata_tabledef
-    puts Flydata::Sync::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table)
+    puts Flydata::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table)
     create_flydata_ctl_table = false
   end
 end

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-04-18"
+  s.date = "2014-05-01"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
   s.executables = ["fdmysqldump", "flydata"]
@@ -43,6 +43,7 @@ Gem::Specification.new do |s|
     "lib/flydata/command/setlogdel.rb",
     "lib/flydata/command/setup.rb",
     "lib/flydata/command/start.rb",
+    "lib/flydata/command/status.rb",
     "lib/flydata/command/stop.rb",
     "lib/flydata/command/sync.rb",
     "lib/flydata/command/version.rb",
@@ -71,6 +72,12 @@ Gem::Specification.new do |s|
     "spec/flydata/command/sync_spec.rb",
     "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
     "spec/flydata/heroku_spec.rb",
+    "spec/flydata/table_def/mysql_table_def_spec.rb",
+    "spec/flydata/table_def/mysqldump_test_table_all.dump",
+    "spec/flydata/table_def/mysqldump_test_table_column_comment.dump",
+    "spec/flydata/table_def/mysqldump_test_table_enum.dump",
+    "spec/flydata/table_def/mysqldump_test_table_multi_pk.dump",
+    "spec/flydata/table_def/mysqldump_test_table_no_pk.dump",
     "spec/flydata/util/encryptor_spec.rb",
     "spec/flydata_spec.rb",
     "spec/spec_helper.rb",
@@ -79,7 +86,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://flydata.co/"
   s.licenses = ["All right reserved."]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.24"
+  s.rubygems_version = "1.8.23"
   s.summary = "FlyData CLI"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/sender.rb

@@ -53,6 +53,13 @@ EOF
           start
         end
       end
+      def status
+        if process_exist?
+          say("Process is running.")
+        else
+          say("Process is not running.")
+        end
+      end
       def kill_all
         if system("ps ax | grep 'flydata' | grep -v grep | awk '{print \"kill  \" $1}' | sh")
           say("Done.")

data/lib/flydata/command/status.rb

@@ -0,0 +1,10 @@
+module Flydata
+  module Command
+    class Status < Base
+      def run
+        sender = Flydata::Command::Sender.new
+        sender.status
+      end
+    end
+  end
+end

data/lib/flydata/helpers.rb

@@ -20,6 +20,7 @@ Usage: flydata COMMAND
   start                    # start flydata process
   stop                     # stop flydata process
   restart                  # restart flydata process
+  status                   # check flydata process
   conf                     # show configuration
   sync                     # initial sync(only for mysql database)
 

data/lib/flydata/table_def/mysql_table_def.rb

@@ -1,5 +1,5 @@
 module Flydata
-module Sync
+module TableDef
 
 class MysqlTableDef
   TYPE_MAP_M2F = {
@@ -15,6 +15,7 @@ class MysqlTableDef
     'decimal' => 'numeric',
     'double' => 'float8',
     'double precision' => 'float8',
+    'enum' => 'enum',
     'fixed' => 'numeric',
     'float' => 'float4',
     'int' => 'int4',
@@ -49,7 +50,6 @@ class MysqlTableDef
   end
 
   def self._create(io)
-    
     table_def = ''
     table_name = nil
     columns = []
@@ -57,24 +57,88 @@ class MysqlTableDef
     comment = nil
 
     position = :before_create_table
+
     io.each_line do |line|
-      if line =~ /CREATE TABLE `(.*?)`/
-        position = :in_create_table
-        table_name = $1
+      case position
+      when :before_create_table
+        if line =~ /CREATE TABLE `(.*?)`/
+          position = :in_create_table
+          table_name = $1
+          table_def += line.chomp
+          next
+        end
+
+      when :in_create_table
+        table_def += line.chomp
+
+        # `col_smallint` smallint(6) DEFAULT NULL,
+        if line.strip.start_with?('`')
+          columns << parse_column_line(line)
+        # PRIMARY KEY (`id`)
+        elsif line.strip.start_with?("PRIMARY KEY")
+          parse_primary_key(line, columns)
+        #) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table';
+        elsif line.strip.start_with?(')')
+          default_charset = $1 if line =~ /DEFAULT CHARSET\s*=\s*([^\s]+)/
+          comment = $1 if /COMMENT='((?:\\'|[^'])*)'/.match(line)
+          position = :after_create_table
+        else
+          raise "Invalid mysqldump file. It must be a bug...\n#{line}"
+        end
+
+      when :after_create_table
+        unless columns.any? {|column| column[:primary_key]}
+          raise "Primary key must be defined."
+        end
+        break
       end
+    end
+    position == :after_create_table ? [table_def, table_name, columns, default_charset, comment] : nil
+  end
+  attr_reader :columns, :table_name
+
+  def to_flydata_tabledef
+    tabledef = { table_name: @table_name,
+                 columns: @columns,
+               }
+    tabledef[:default_charset] = @default_charset if @default_charset
+    tabledef[:comment] = @comment if @comment
 
-      next unless position == :in_create_table
+    tabledef
+  end
 
-      table_def += line.chomp
+  private
 
-      if line =~ /^\s*`(.*?)`\s+([a-z()0-9,\s]+)/ # column def
-        name = $1
-        type = $2.strip
-        type = type[0..-2] if type.end_with?(',')
+  def self.parse_column_line(line)
+    line = line.strip
+    line = line[0..-2] if line.end_with?(',')
+    pos = 0
+    cond = :column_name
+    column = {}
+
+    while pos < line.length
+      #cur_char = line[pos]
+      case cond
+      when :column_name  #`column_name` ...
+        pos = line.index('`', 1)
+        column[:name] = line[1..pos-1]
+        cond = :column_type
+        pos += 1
+      when :column_type  #... formattype(,,,) ...
+        pos += 1 until line[pos] != ' '
+        start_pos = pos
+        pos += 1 until line[pos].nil? || line[pos] =~ /\s|\(/
+        if line[pos] == '('
+          pos = parse_column_type_values(line, pos)
+          pos += 1
+        end
+        type = line[start_pos..pos-1]
         TYPE_MAP_M2F.each do |mysql_type, flydata_type|
           type.gsub!(/\b#{mysql_type}\b/, flydata_type)
         end
-        column = {name: name, type: type}
+        column[:type] = type
+        cond = :options
+      when :options
         column[:auto_increment] = true if line =~ /AUTO_INCREMENT/
         column[:not_null] = true if line =~ /NOT NULL/
         if /DEFAULT\s+((?:[^'\s]+\b)|(?:'(?:\\'|[^'])*'))/.match(line)
@@ -86,41 +150,28 @@ class MysqlTableDef
         if /COMMENT\s+'(((?:\\'|[^'])*))'/.match(line)
           column[:comment] = $1
         end
-        columns << column
-      end
-
-      if line =~ /PRIMARY KEY/
-        primary_keys = line.scan(/`(.*?)`/).collect{|item| item[0]}
-        primary_keys.each do |primary_key|
-          column = columns.detect {|column|
-            column[:name] === primary_key
-          }
-          raise "PK #{primary_key} must exist in the definition " if column.nil?
-          column[:primary_key] = true
-        end
-      end
-
-      if line =~ /DEFAULT CHARSET\s*=\s*([^\s]+)/
-        default_charset = $1
-      end
-
-      if line =~ /ENGINE=.*/  # TODO Need better parsing.
-        position = :after_create_table
         break
+      else
+        raise "Invalid condition. It must be a bug..."
       end
     end
-    position == :after_create_table ? [table_def, table_name, columns, default_charset, comment] : nil
+    column
   end
-  attr_reader :columns, :table_name
 
-  def to_flydata_tabledef
-    tabledef = { table_name: @table_name,
-                 columns: @columns,
-               }
-    tabledef[:default_charset] = @default_charset if @default_charset
-    tabledef[:comment] = @comment if @comment
+  def self.parse_primary_key(line, columns)
+    primary_keys = line.scan(/`(.*?)`/).collect{|item| item[0]}
+    primary_keys.each do |primary_key|
+      column = columns.detect {|column|
+        column[:name] === primary_key
+      }
+      raise "PK #{primary_key} must exist in the definition " if column.nil?
+      column[:primary_key] = true
+    end
+  end
 
-    tabledef
+  def self.parse_column_type_values(line, pos)
+    #TODO: more better parse
+    line.index(')', pos)
   end
 end
 

data/lib/flydata/table_def/redshift_table_def.rb

@@ -1,5 +1,5 @@
 module Flydata
-module Sync
+module TableDef
 
 class RedshiftTableDef
   TYPE_MAP_F2R = {
@@ -7,6 +7,7 @@ class RedshiftTableDef
     'char' => {type: 'char', use_params: true},
     'date' => {type: 'date'},
     'datetime' => {type: 'timestamp'},
+    'enum' => {type: 'varchar encode bytedict'},
     'float4' => {type: 'float4'},
     'float4 unsigned' => {type: 'float4'},
     'float8' => {type: 'float8'},
@@ -132,11 +133,15 @@ EOS
     sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_tabledef[:table_name] ]
     values = []
     flydata_tabledef[:columns].each do |col|
-      values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{col[:type]}')"
+      values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{escape(col[:type])}')"
     end
     sql += values.join(",\n") + ';'
     sql
   end
+
+  def self.escape(text)
+    text.gsub("'", "\\\\'")
+  end
 end
 
 end

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'fluent_plugins_spec_helper'
 require 'in_mysql_binlog_flydata'
 require 'timecop'
@@ -25,14 +27,32 @@ EOT
   # - 17 WRITE_ROWS_EVENT
   TEST_EVENT_INSERT=<<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>23, "server_id"=>1, "event_length"=>39, "next_position"=>667, "flags"=>1, "event_type"=>"Write_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 3, 0, 0, 0, 3, 0, 102, 111, 111], "rows"=>[["1", "foo"],["2","var"],["3","hoge"]]}
+EOT
+  TEST_EVENT_TWO_BYTE_INSERT=<<EOT
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>23, "server_id"=>1, "event_length"=>39, "next_position"=>667, "flags"=>1, "event_type"=>"Write_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 3, 0, 0, 0, 3, 0, 102, 111, 111], "rows"=>[["1", "føø"],["2","vår"],["3","høgé"]]}
+EOT
+  TEST_EVENT_THREE_BYTE_INSERT=<<EOT
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>23, "server_id"=>1, "event_length"=>39, "next_position"=>667, "flags"=>1, "event_type"=>"Write_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 3, 0, 0, 0, 3, 0, 102, 111, 111], "rows"=>[["1", "富无无"],["2","易变的"],["3","切实切实"]]}
 EOT
   # - 18 UPDATE_ROWS_EVENT
   TEST_EVENT_UPDATE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>24, "server_id"=>1, "event_length"=>78, "next_position"=>2606, "flags"=>1, "event_type"=>"Update_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[255], "raw_used_columns"=>[255], "raw_row"=>[252, 6, 0, 0, 0, 1, 0, 97, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 98, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 99, 252, 6, 0, 0, 0, 1, 0, 100], "rows"=>[[["1", "foo"], ["1", "wow"]],[["3", "hoge"], ["3", "fuga"]]]}
+EOT
+  TEST_EVENT_TWO_BYTE_UPDATE = <<EOT
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>24, "server_id"=>1, "event_length"=>78, "next_position"=>2606, "flags"=>1, "event_type"=>"Update_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[255], "raw_used_columns"=>[255], "raw_row"=>[252, 6, 0, 0, 0, 1, 0, 97, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 98, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 99, 252, 6, 0, 0, 0, 1, 0, 100], "rows"=>[[["1", "føø"], ["1", "∑ø∑"]],[["3", "høgé"], ["3", "fügå"]]]}
+EOT
+  TEST_EVENT_THREE_BYTE_UPDATE = <<EOT
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>24, "server_id"=>1, "event_length"=>78, "next_position"=>2606, "flags"=>1, "event_type"=>"Update_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[255], "raw_used_columns"=>[255], "raw_row"=>[252, 6, 0, 0, 0, 1, 0, 97, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 98, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 99, 252, 6, 0, 0, 0, 1, 0, 100], "rows"=>[[["1", "富无无"], ["1", "很兴奋"]],[["3", "切实切实"], ["3", "興奮虎"]]]}
 EOT
   # - 19 DELETE_ROWS_EVENT
   TEST_EVENT_DELETE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>25, "server_id"=>1, "event_length"=>41, "next_position"=>5365, "flags"=>1, "event_type"=>"Delete_rows", "table_id"=>170, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 2, 0, 0, 0, 5, 0, 104, 111, 104, 111, 103], "rows"=>[["2", "var"],["3","hoge"]]}
+EOT
+  TEST_EVENT_TWO_BYTE_DELETE = <<EOT
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>25, "server_id"=>1, "event_length"=>41, "next_position"=>5365, "flags"=>1, "event_type"=>"Delete_rows", "table_id"=>170, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 2, 0, 0, 0, 5, 0, 104, 111, 104, 111, 103], "rows"=>[["2", "vår"],["3","høgé"]]}
+EOT
+  TEST_EVENT_THREE_BYTE_DELETE = <<EOT
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>25, "server_id"=>1, "event_length"=>41, "next_position"=>5365, "flags"=>1, "event_type"=>"Delete_rows", "table_id"=>170, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 2, 0, 0, 0, 5, 0, 104, 111, 104, 111, 103], "rows"=>[["2", "易变的"],["3","切实切实"]]}
 EOT
   # Unsupported event
   # - 02 QUERY_EVENT
@@ -120,8 +140,14 @@ EOT
 
     let(:plugin) { MysqlBinlogFlydataInput.new }
     let(:insert_event) { create_event(TEST_EVENT_INSERT) }
+    let(:insert_two_byte_event) { create_event(TEST_EVENT_TWO_BYTE_INSERT) }
+    let(:insert_three_byte_event) { create_event(TEST_EVENT_THREE_BYTE_INSERT) }
     let(:delete_event) { create_event(TEST_EVENT_DELETE) }
+    let(:delete_two_byte_event) { create_event(TEST_EVENT_TWO_BYTE_DELETE) }
+    let(:delete_three_byte_event) { create_event(TEST_EVENT_THREE_BYTE_DELETE) }
     let(:update_event) { create_event(TEST_EVENT_UPDATE) }
+    let(:update_two_byte_event) { create_event(TEST_EVENT_TWO_BYTE_UPDATE) }
+    let(:update_three_byte_event) { create_event(TEST_EVENT_THREE_BYTE_UPDATE) }
 
     let(:query_event) { create_event(TEST_EVENT_QUERY_CREATE_DATABSE) }
     let(:table_map_event) { create_event(TEST_EVENT_TABLE_MAP) }
@@ -160,6 +186,22 @@ EOT
         end
       end
 
+      context 'when received insert event containing two byte UTF8 chars' do
+        it do
+          table_seq_file.should_receive(:write).exactly(3).with(2)
+          expect_emitted_records_with_rows(insert_two_byte_event, :insert, TEST_TABLE,
+            [{"1"=>"0SL00000001", "2"=>"føø"}, {"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
+        end
+      end
+
+      context 'when received insert event containing three byte UTF8 chars' do
+        it do
+          table_seq_file.should_receive(:write).exactly(3).with(2)
+          expect_emitted_records_with_rows(insert_three_byte_event, :insert, TEST_TABLE,
+            [{"1"=>"0SL00000001", "2"=>"富无无"}, {"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
+        end
+      end
+
       context 'when received delete event' do
         it do
           table_seq_file.should_receive(:write).twice.with(2)
@@ -168,6 +210,22 @@ EOT
         end
       end
 
+      context 'when received delete event containing two byte UTF8 chars' do
+        it do
+          table_seq_file.should_receive(:write).twice.with(2)
+          expect_emitted_records_with_rows(delete_two_byte_event, :delete, TEST_TABLE,
+            [{"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
+        end
+      end
+
+      context 'when received delete event with containing byte UTF8 chars' do
+        it do
+          table_seq_file.should_receive(:write).twice.with(2)
+          expect_emitted_records_with_rows(delete_three_byte_event, :delete, TEST_TABLE,
+            [{"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
+        end
+      end
+
       context 'when received update event' do
         it do
           table_seq_file.should_receive(:write).twice.with(2)
@@ -176,6 +234,22 @@ EOT
         end
       end
 
+      context 'when received update event with two byte utf8 chars' do
+        it do
+          table_seq_file.should_receive(:write).twice.with(2)
+          expect_emitted_records_with_rows(update_two_byte_event, :update, TEST_TABLE,
+            [{"1"=>"0SL00000001", "2"=>"∑ø∑"}, {"1"=>"0SL00000003", "2"=>"fügå"}])
+        end
+      end
+
+      context 'when received update event with three byte utf8 chars' do
+        it do
+          table_seq_file.should_receive(:write).twice.with(2)
+          expect_emitted_records_with_rows(update_three_byte_event, :update, TEST_TABLE,
+            [{"1"=>"0SL00000001", "2"=>"很兴奋"}, {"1"=>"0SL00000003", "2"=>"興奮虎"}])
+        end
+      end
+
       context 'when received event with another database name' do
         it do
           event = insert_event

data/spec/flydata/table_def/mysql_table_def_spec.rb

@@ -0,0 +1,123 @@
+require 'spec_helper'
+require 'flydata/table_def'
+
+module Flydata
+module TableDef
+
+
+describe MysqlTableDef do
+
+  # file full path which exists in same directory.
+  def full_path(file_name)
+    File.join(File.dirname(__FILE__), file_name)
+  end
+
+  def file_io(file_name)
+    File.new(full_path(file_name))
+  end
+
+  describe '.to_flydata_tabledef' do
+    let(:dump_file_io) { raise "dump_file_io must be override on context." }
+    subject { MysqlTableDef.create(dump_file_io).to_flydata_tabledef }
+
+    context 'when dump file includes single table with all kind of format types' do
+      let(:dump_file_io) { file_io('mysqldump_test_table_all.dump') }
+
+      it 'should set correct table name' do
+        expect(subject[:table_name]).to eq('test_table_all')
+      end
+
+      it 'should set correct table name' do
+        expect(subject[:columns]).to eq(
+          [
+            {:name=>"id", :type=>"int8(20)", :auto_increment=>true, :not_null=>true, :primary_key=>true},
+            {:name=>"col_binary", :type=>"binary(100)", :default=>nil},
+            {:name=>"col_blob", :type=>"varbinary"},
+            {:name=>"col_bool", :type=>"int1(1)", :default=>"0"},
+            {:name=>"col_char", :type=>"varchar(6)", :default=>nil},
+            {:name=>"col_date", :type=>"date", :default=>nil},
+            {:name=>"col_datetime", :type=>"datetime", :default=>nil},
+            {:name=>"col_decimal", :type=>"numeric(5,2)", :default=>nil},
+            {:name=>"col_double", :type=>"float8", :default=>nil},
+            {:name=>"col_float", :type=>"float4", :default=>nil},
+            {:name=>"col_float_4_2", :type=>"float4(4,2)", :default=>nil},
+            {:name=>"col_int", :type=>"int4(11)", :default=>nil},
+            {:name=>"col_int_6", :type=>"int4(6)", :default=>nil},
+            {:name=>"col_longblob", :type=>"varbinary"},
+            {:name=>"col_longtext", :type=>"text"},
+            {:name=>"col_mediumblob", :type=>"varbinary"},
+            {:name=>"col_mediumint", :type=>"int3(9)", :default=>nil},
+            {:name=>"col_mediumtext", :type=>"text"},
+            {:name=>"col_smallint", :type=>"int2(6)", :default=>nil},
+            {:name=>"col_text", :type=>"text"},
+            {:name=>"col_time", :type=>"time", :default=>nil},
+            {:name=>"col_timestamp", :type=>"datetime", :not_null=>true, :default=>"CURRENT_TIMESTAMP"},
+            {:name=>"col_tinyblob", :type=>"varbinary"},
+            {:name=>"col_tinyint", :type=>"int1(4)", :default=>nil},
+            {:name=>"col_tinytext", :type=>"text"},
+            {:name=>"col_varbinary", :type=>"varbinary(255)", :default=>nil},
+            {:name=>"col_varchar", :type=>"varchar(124)", :default=>nil}
+          ]
+        )
+      end
+
+      it 'default_charset should be set' do
+        expect(subject[:default_charset]).to eq('utf8')
+      end
+
+      it 'comment should be set' do
+        expect(subject[:comment]).to eq('test table includes all kind of format type')
+      end
+    end
+
+    context 'when table does not have primary key' do
+      let(:dump_file_io) { file_io('mysqldump_test_table_no_pk.dump') }
+      it 'should raise an error' do
+        expect{subject}.to raise_error('Primary key must be defined.')
+      end
+    end
+
+    context 'when column has comment' do
+      let(:dump_file_io) { file_io('mysqldump_test_table_column_comment.dump') }
+      it 'comment should be set' do
+        expect(subject[:columns]).to eq(
+          [
+            {:name=>"id", :type=>"int4(11)", :not_null=>true, :default=>"0",
+             :comment=>"this is primary key", :primary_key=>true},
+            {:name=>"value", :type=>"text"}
+          ]
+        )
+      end
+    end
+
+    context 'when table has multiple pk' do
+      let(:dump_file_io) { file_io('mysqldump_test_table_multi_pk.dump') }
+      it 'multi pk should be set' do
+        expect(subject[:columns]).to eq(
+          [
+            {:name=>"id1", :type=>"int4(11)", :not_null=>true, :default=>"0", :primary_key=>true},
+            {:name=>"id2", :type=>"int4(11)", :not_null=>true, :default=>"0", :primary_key=>true},
+            {:name=>"value", :type=>"text"}
+          ]
+        )
+      end
+    end
+
+    context 'when table has enum column' do
+      let(:dump_file_io) { file_io('mysqldump_test_table_enum.dump') }
+      it 'enum columns should be parsed' do
+        expect(subject[:columns]).to eq(
+          [
+            {:name=>"id", :type=>"int4(11)", :not_null=>true, :primary_key=>true},
+            {:name=>"enum_1", :type=>"enum('apple','orange','banana')", :default=>nil},
+            {:name=>"enum_2", :type=>"enum('a','b','c')", :default=>"a"},
+            {:name=>"enum_3", :type=>"enum('e','f','g')", :not_null=>true},
+          ]
+        )
+      end
+    end
+  end
+end
+
+end
+end

data/spec/flydata/table_def/mysqldump_test_table_all.dump

@@ -0,0 +1,66 @@
+-- MySQL dump 10.13  Distrib 5.6.13, for osx10.9 (x86_64)
+--
+-- Host: localhost    Database: masashi_test
+-- ------------------------------------------------------
+-- Server version	5.6.13-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `test_table_all`
+--
+
+DROP TABLE IF EXISTS `test_table_all`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `test_table_all` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `col_binary` binary(100) DEFAULT NULL,
+  `col_blob` blob,
+  `col_bool` tinyint(1) DEFAULT '0',
+  `col_char` char(6) DEFAULT NULL,
+  `col_date` date DEFAULT NULL,
+  `col_datetime` datetime DEFAULT NULL,
+  `col_decimal` decimal(5,2) DEFAULT NULL,
+  `col_double` double DEFAULT NULL,
+  `col_float` float DEFAULT NULL,
+  `col_float_4_2` float(4,2) DEFAULT NULL,
+  `col_int` int(11) DEFAULT NULL,
+  `col_int_6` int(6) DEFAULT NULL,
+  `col_longblob` longblob,
+  `col_longtext` longtext,
+  `col_mediumblob` mediumblob,
+  `col_mediumint` mediumint(9) DEFAULT NULL,
+  `col_mediumtext` mediumtext,
+  `col_smallint` smallint(6) DEFAULT NULL,
+  `col_text` text,
+  `col_time` time DEFAULT NULL,
+  `col_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  `col_tinyblob` tinyblob,
+  `col_tinyint` tinyint(4) DEFAULT NULL,
+  `col_tinytext` tinytext,
+  `col_varbinary` varbinary(255) DEFAULT NULL,
+  `col_varchar` varchar(124) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table includes all kind of format type';
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-04-25 12:18:04

data/spec/flydata/table_def/mysqldump_test_table_column_comment.dump

@@ -0,0 +1,41 @@
+-- MySQL dump 10.13  Distrib 5.6.13, for osx10.9 (x86_64)
+--
+-- Host: localhost    Database: masashi_test
+-- ------------------------------------------------------
+-- Server version	5.6.13-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `test_table_column_comment`
+--
+
+DROP TABLE IF EXISTS `test_table_column_comment`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `test_table_column_comment` (
+  `id` int(11) NOT NULL DEFAULT '0' COMMENT 'this is primary key',
+  `value` text,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='column comment test';
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-04-25 13:24:35

data/spec/flydata/table_def/mysqldump_test_table_enum.dump

@@ -0,0 +1,43 @@
+-- MySQL dump 10.13  Distrib 5.6.13, for osx10.9 (x86_64)
+--
+-- Host: localhost    Database: masashi_test
+-- ------------------------------------------------------
+-- Server version	5.6.13-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `test_table_enum`
+--
+
+DROP TABLE IF EXISTS `test_table_enum`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `test_table_enum` (
+  `id` int(11) NOT NULL,
+  `enum_1` enum('apple','orange','banana') DEFAULT NULL,
+  `enum_2` enum('a','b','c') DEFAULT 'a',
+  `enum_3` enum('e','f','g') NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='with enum column';
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-04-25 13:55:12

data/spec/flydata/table_def/mysqldump_test_table_multi_pk.dump

@@ -0,0 +1,42 @@
+-- MySQL dump 10.13  Distrib 5.6.13, for osx10.9 (x86_64)
+--
+-- Host: localhost    Database: masashi_test
+-- ------------------------------------------------------
+-- Server version	5.6.13-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `test_table_multi_pk`
+--
+
+DROP TABLE IF EXISTS `test_table_multi_pk`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `test_table_multi_pk` (
+  `id1` int(11) NOT NULL DEFAULT '0',
+  `id2` int(11) NOT NULL DEFAULT '0',
+  `value` text,
+  PRIMARY KEY (`id1`,`id2`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='multiple primary key';
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-04-25 13:39:30

data/spec/flydata/table_def/mysqldump_test_table_no_pk.dump

@@ -0,0 +1,40 @@
+-- MySQL dump 10.13  Distrib 5.6.13, for osx10.9 (x86_64)
+--
+-- Host: localhost    Database: masashi_test
+-- ------------------------------------------------------
+-- Server version	5.6.13-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `test_table_no_pk`
+--
+
+DROP TABLE IF EXISTS `test_table_no_pk`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `test_table_no_pk` (
+  `id` int(11) DEFAULT NULL,
+  `value` text
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='no primary key';
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-04-25 13:48:49

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-18 00:00:00.000000000 Z
+date: 2014-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -336,6 +336,7 @@ files:
 - lib/flydata/command/setlogdel.rb
 - lib/flydata/command/setup.rb
 - lib/flydata/command/start.rb
+- lib/flydata/command/status.rb
 - lib/flydata/command/stop.rb
 - lib/flydata/command/sync.rb
 - lib/flydata/command/version.rb
@@ -364,6 +365,12 @@ files:
 - spec/flydata/command/sync_spec.rb
 - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
 - spec/flydata/heroku_spec.rb
+- spec/flydata/table_def/mysql_table_def_spec.rb
+- spec/flydata/table_def/mysqldump_test_table_all.dump
+- spec/flydata/table_def/mysqldump_test_table_column_comment.dump
+- spec/flydata/table_def/mysqldump_test_table_enum.dump
+- spec/flydata/table_def/mysqldump_test_table_multi_pk.dump
+- spec/flydata/table_def/mysqldump_test_table_no_pk.dump
 - spec/flydata/util/encryptor_spec.rb
 - spec/flydata_spec.rb
 - spec/spec_helper.rb
@@ -383,7 +390,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1795094757284592242
+      hash: -2999852327908013183
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -392,7 +399,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: FlyData CLI