flydata 0.2.30 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: acad4ae7202b7fc525478de91925ef4ee0a24fdd
-  data.tar.gz: 63cf3ac658afa5be37ed7b2a85be476de280c56f
+  metadata.gz: 68f311d20207752e80dd5fb47b905b085e840520
+  data.tar.gz: 6bfa38fa63a7db7e1f4b7109324017e99c82c117
 SHA512:
-  metadata.gz: fb7391185d4f9797817f0628ec46f5b9cce0ef1680fb843d0181f97dd18450c059d6b6614b7ef98e8ee88dfac171933561e448a2ca1b403665563b27b5d005a8
-  data.tar.gz: 7407fd538bbd7678ae19d5ddaa9dcc5b13159555b8c7b82a788c821fb70855594f49848eb141ec7a44982eb164dd63738e63a9714ed132b5b02ff2caab948d4f
+  metadata.gz: 01469b9c9b268bc8dad0bda81a3dbcc3a98a55901108cac17b50a6688313485f1d7cdd8a01b26ea20cbea0d8fa6f4c096055394f55cea58ee59a949acb2963fe
+  data.tar.gz: cde53114405b00f034e7444fd0b9ba9df9519a2cd3979eade6a99d48e80e326615a1ba16fb0d5c014c51d23272ca5cf2257b2f318135ad6efef858f4d6d4479b

data/VERSION

@@ -1 +1 @@
-0.2.30
+0.3.0

data/bin/flydata

@@ -8,4 +8,9 @@ require 'flydata'
 
 args = ARGV.dup
 ARGV.clear # for 'gets'
-Flydata::Cli.new(args).run
+begin
+  Flydata::Cli.new(args).run
+rescue Interrupt
+  # Suppress stacktrace upon Ctrl-C quit
+  puts
+end

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -28,37 +28,37 @@ class MysqlTableDef
   end
 
   TYPE_MAP_M2F = {
-    'bigint' => {type: 'int8'},
-    'binary' => {type: 'binary', override: PROC_override_varbinary},
-    'bit' => {type: 'bit'},
+    'bigint' => {type: 'int8', def_width: '20'},
+    'binary' => {type: 'binary', def_width: '1', override: PROC_override_varbinary},
+    'bit' => {type: 'bit', def_width: '1'},
     'blob' => {type: 'varbinary(65535)'},
-    'bool' => {type: 'int1'},
-    'boolean' => {type: 'int1'},
-    'char' => {type: 'varchar', override: PROC_override_varchar},
+    'bool' => {type: 'int1', def_width: '1'},
+    'boolean' => {type: 'int1', def_width: '1'},
+    'char' => {type: 'varchar', def_width: '1', override: PROC_override_varchar},
     'date' => {type: 'date'},
     'datetime' => {type: 'datetime'},
-    'dec' => {type: 'numeric'},
-    'decimal' => {type: 'numeric'},
+    'dec' => {type: 'numeric', def_width: '10,0'},
+    'decimal' => {type: 'numeric', def_width: '10,0'},
     'double' => {type: 'float8'},
     'double precision' => {type: 'float8'},
     'enum' => {type: 'enum'},
-    'fixed' => {type: 'numeric'},
+    'fixed' => {type: 'numeric', def_width: '10,0'},
     'float' => {type: 'float4'},
-    'int' => {type: 'int4'},
-    'integer' => {type: 'int4'},
+    'int' => {type: 'int4', def_width: '11'},
+    'integer' => {type: 'int4', def_width: '11'},
     'longblob' => {type: 'varbinary(4294967295)'},
     'longtext' => {type: 'text'},
     'mediumblob' => {type: 'varbinary(16777215)'},
-    'mediumint' => {type: 'int3'},
+    'mediumint' => {type: 'int3', def_width: '9'},
     'mediumtext' => {type: 'text'},
-    'numeric' => {type: 'numeric'},
+    'numeric' => {type: 'numeric', def_width: '10,0'},
     'set' => {type: 'set'},
-    'smallint' => {type: 'int2'},
+    'smallint' => {type: 'int2', def_width: '6'},
     'text' => {type: 'text'},
     'time' => {type: 'time'},
     'timestamp' => {type: 'datetime'},
     'tinyblob' => {type: 'varbinary(255)'},
-    'tinyint' => {type: 'int1'},
+    'tinyint' => {type: 'int1', def_width: '4'},
     'tinytext' => {type: 'text'},
     'varbinary' => {type: 'varbinary', override: PROC_override_varbinary},
     'varchar' => {type: 'varchar', override: PROC_override_varchar},
@@ -69,6 +69,9 @@ class MysqlTableDef
       flydata_type = type_hash[:type]
       if /^#{mysql_type}\(|^#{mysql_type}$/.match(type)
         ret_type = type.gsub(/^#{mysql_type}/, flydata_type)
+        if type_hash.has_key?(:def_width) && !/\(.+\)/.match(ret_type)
+          ret_type += "(#{type_hash[:def_width]})"
+        end
         if type_hash[:override]
           ret_type = type_hash[:override].call(ret_type, mysql_type, flydata_type)
         end

data/flydata-core/spec/table_def/mysql_table_def_spec.rb

@@ -199,6 +199,138 @@ describe MysqlTableDef do
       end
     end
   end
+
+  describe '.convert_to_flydata_type' do
+    subject { described_class.convert_to_flydata_type(mysql_type) }
+    shared_examples "converting the mysql data type to the expected flydata type" do
+      # Use the context description as the value of "mysql_type"
+      # RSpec.current_example.meatadata[:example_group] => the shared_examples "converting the mysql data type to the expected flydata type"
+      # [:parent_example_group] => the context including the shared examples (e.g. context "bigint")
+      # [:description] => the name of the example group (e.g. "bigint")
+      let(:mysql_type) { RSpec.current_example.metadata[:example_group][:parent_example_group][:description] }
+      it do
+        expect(subject).to eq flydata_type
+      end
+    end
+    context "with no-width mysql data type" do
+      context "bigint" do
+        let(:flydata_type) { 'int8(20)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "binary" do
+        let(:flydata_type) { 'binary(4)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "bit" do
+        let(:flydata_type) { 'bit(1)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "blob" do
+        let(:flydata_type) { 'varbinary(65535)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "bool" do
+        let(:flydata_type) { 'int1(1)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "boolean" do
+        let(:flydata_type) { 'int1(1)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "char" do
+        let(:flydata_type) { 'varchar(3)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "date" do
+        let(:flydata_type) { 'date' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "datetime" do
+        let(:flydata_type) { 'datetime' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "dec" do
+        let(:flydata_type) { 'numeric(10,0)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "decimal" do
+        let(:flydata_type) { 'numeric(10,0)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "double" do
+        let(:flydata_type) { 'float8' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "fixed" do
+        let(:flydata_type) { 'numeric(10,0)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "float" do
+        let(:flydata_type) { 'float4' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "int" do
+        let(:flydata_type) { 'int4(11)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "integer" do
+        let(:flydata_type) { 'int4(11)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "longblob" do
+        let(:flydata_type) { 'varbinary(4294967295)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "longtext" do
+        let(:flydata_type) { 'text' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "mediumblob" do
+        let(:flydata_type) { 'varbinary(16777215)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "mediumint" do
+        let(:flydata_type) { 'int3(9)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "mediumtext" do
+        let(:flydata_type) { 'text' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "numeric" do
+        let(:flydata_type) { 'numeric(10,0)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "smallint" do
+        let(:flydata_type) { 'int2(6)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "text" do
+        let(:flydata_type) { 'text' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "time" do
+        let(:flydata_type) { 'time' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "timestamp" do
+        let(:flydata_type) { 'datetime' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "tinyblob" do
+        let(:flydata_type) { 'varbinary(255)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "tinyint" do
+        let(:flydata_type) { 'int1(4)' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+      context "tinytext" do
+        let(:flydata_type) { 'text' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
+    end
+  end
 end
 
 end

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.2.30 ruby lib
+# stub: flydata 0.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.30"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-01-26"
+  s.date = "2015-02-11"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -93,6 +93,7 @@ Gem::Specification.new do |s|
     "lib/flydata/compatibility_check.rb",
     "lib/flydata/credentials.rb",
     "lib/flydata/cron.rb",
+    "lib/flydata/errors.rb",
     "lib/flydata/fluent-plugins/idle_event_detector.rb",
     "lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb",
     "lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb",
@@ -149,7 +150,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.2.2"
+  s.rubygems_version = "2.4.3"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/cli.rb

@@ -1,5 +1,6 @@
 require 'slop'
 require_relative 'command_logger'
+require_relative 'errors'
 
 module Flydata
   class Cli
@@ -42,15 +43,16 @@ module Flydata
           cmd_obj = cmd_cls.new(options)
           sub_cmd ? cmd_obj.send(sub_cmd,*@args) : cmd_obj.run(*@args)
         else
-          $stderr.puts usage_text(false)
+          $stderr.puts usage_text
           return
         end
       rescue => e
         #raise e
-        $stderr.puts "! #{e.to_s}"
         $stderr.puts
         $stderr.puts
-        $stderr.puts usage_text
+        $stderr.puts "!! #{e.to_s}"
+        $stderr.puts
+        $stderr.puts e.respond_to?(:description) ? e.description : AgentError.description
         log_error_with_backtrace('Error occured during command.', {error: e, args: @args})
         raise e if FLYDATA_DEBUG
         exit 1

data/lib/flydata/command/setup.rb

@@ -93,8 +93,13 @@ What's next?
           elsif has_registered_redshift_mysql_data_entries?
             de = retrieve_data_entries.first
             if File.exists?(Flydata::FileUtil::SyncFileManager.new(de).binlog_path)
-              Flydata::Command::Sender.new.stop(quiet: true)
-              true
+              sender = Flydata::Command::Sender.new
+              if sender.process_exist?
+                sender.stop(quiet: true)
+                true
+              else
+                false
+              end
             else
               last_message = INITIAL_SYNC_MESSAGE_TEMPLATE
               false

data/lib/flydata/command/sync.rb

@@ -13,9 +13,11 @@ require 'flydata-core/table_def'
 
 module Flydata
   module Command
+
     class Sync < Base
       include Helpers
       INSERT_PROGRESS_INTERVAL = 1000
+      SERVER_DATA_PROCESSING_TIMEOUT = 600 # seconds
 
       # for dump.pos file
       STATUS_PARSING = 'PARSING'
@@ -102,8 +104,21 @@ module Flydata
         de = retrieve_sync_data_entry unless de
         de = load_sync_info(de,tables)
         validate_initial_sync_status(de, tables)
-        flush_buffer_and_stop  unless @initial_sync
-        sync_mysql_to_redshift(de)
+        begin
+          flush_buffer_and_stop  unless @initial_sync
+          sync_mysql_to_redshift(de)
+        rescue ServerDataProcessingTimeout => e
+          ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
+          ee.description = <<EOS
+  Data processing is taking more than expected.  Please contact support@flydata.com to check the system status.
+  Once checked, you can resume sync with the following command.
+
+    flydata start
+
+EOS
+          ee.set_backtrace e.backtrace
+          raise ee
+        end
         complete
       end
 
@@ -114,7 +129,17 @@ module Flydata
       end
 
       def flush
-        flush_buffer_and_stop
+        begin
+          flush_buffer_and_stop
+        rescue ServerDataProcessingTimeout => e
+          ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
+          ee.description = <<EOS
+  Data processing is taking more than expected.  Please contact support@flydata.com to check the system status.
+
+EOS
+          ee.set_backtrace e.backtrace
+          raise ee
+        end
         log_info_stdout("Buffers have been flushed and the sender process has been stopped.")
       end
 
@@ -136,7 +161,20 @@ module Flydata
         all_tables = de['mysql_data_entry_preference']['tables']
         verify_input_tables(tables, all_tables)
 
-        wait_for_server_buffer
+        begin
+          wait_for_server_buffer(SERVER_DATA_PROCESSING_TIMEOUT)
+        rescue ServerDataProcessingTimeout => e
+          ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
+          ee.description = <<EOS
+  Data processing is taking more than expected.  Please contact support@flydata.com to check the system status.
+  Once checked, you can continue sync reset with the following command
+
+    flydata sync:reset #{tables.empty? ? '' : tables.join(" ")}
+
+EOS
+          ee.set_backtrace e.backtrace
+          raise ee
+        end
         cleanup_sync_server(de, tables) unless opts.client?
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         delete_files = [
@@ -158,25 +196,41 @@ module Flydata
         log_info_stdout("Reset completed successfully.")
       end
 
-      def wait_for_server_buffer
+      def wait_for_server_buffer(timeout = 0)
+        start_time = Time.now
         log_info_stdout("Waiting for the server buffer to get empty.")
+        prev_message =nil
         while (status = check) && (status['state'] == 'processing')
+          prev_message = status['message']
+          if timeout > 0 && Time.now - start_time > timeout
+            raise ServerDataProcessingTimeout.new
+          end
           print_progress(status)
           sleep 10
         end
       end
 
-      def wait_for_server_data_processing
+      def wait_for_server_data_processing(timeout = 0)
         state = :PROCESS
+        start_time = Time.now
         log_info_stdout("Uploading data to Redshift...")
         sleep 10
         status = nil
+        prev_message =nil
         while (status = check)
           if state == :PROCESS && status['state'] == 'uploading'
             log_info_stdout("  -> Done")
             state = :UPLOAD
             log_info_stdout("Finishing data upload...")
           end
+          if status['message'] != prev_message
+            # making some progress.  Reset timer
+            start_time = Time.now
+          end
+          prev_message = status['message']
+          if timeout > 0 && Time.now - start_time > timeout
+            raise ServerDataProcessingTimeout.new
+          end
           print_progress(status)
           sleep 10
         end
@@ -606,7 +660,7 @@ EOM
         return unless dump_pos_info[:status] == STATUS_WAITING
         binlog_pos = dump_pos_info[:binlog_pos]
 
-        wait_for_server_data_processing
+        wait_for_server_data_processing(SERVER_DATA_PROCESSING_TIMEOUT)
         tables = de['mysql_data_entry_preference']['tables']
         sync_fm.save_table_binlog_pos(tables, binlog_pos)
         sync_fm.save_dump_pos(STATUS_COMPLETE, '', -1, binlog_pos)
@@ -670,7 +724,7 @@ Thank you for using FlyData!
       def flush_buffer_and_stop
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer
-        wait_for_server_data_processing
+        wait_for_server_data_processing(SERVER_DATA_PROCESSING_TIMEOUT)
         sender.stop(quiet: true)
       end
     end

data/lib/flydata/compatibility_check.rb

@@ -137,8 +137,8 @@ module Flydata
       end
     end
 
-    def check_mysql_row_mode_compat
-      sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1}
+    def check_mysql_parameters_compat
+      sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1, '@@log_slave_updates'=>1}
       errors={}
 
       client = Mysql2::Client.new(@db_opts)

data/lib/flydata/errors.rb

@@ -0,0 +1,34 @@
+module Flydata
+
+class AgentError < StandardError
+    def self.description
+      text = <<-EOM
+  Fix the issue and try again.  If the problem continues, please contact support@flydata.com
+
+EOM
+
+      flydata_log = File.join(FLYDATA_HOME, 'flydata.log')
+      if File.exists?(flydata_log)
+        text += <<-EOM
+Also check the Agent log.
+   Log path: #{flydata_log}
+EOM
+      end
+      text
+    end
+
+    def description
+      if instance_variable_defined?(:@description) && @description
+        @description
+      else
+        self.class.description
+      end
+    end
+
+    attr_writer :description
+end
+
+class ServerDataProcessingTimeout < AgentError
+end
+
+end

data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb

@@ -3,6 +3,7 @@ require_relative 'binlog_record_handler'
 module Mysql
   class DmlRecordHandler < BinlogRecordHandler
     ROW = :row
+    OLD = :old
     INTEGER_TYPES = {'TINY' => 1,
                      'SHORT' => 2,
                      'INT24' => 3,
@@ -36,7 +37,8 @@ module Mysql
 
     def emit_update(record)
       emit_rows(:update, record) do |row|
-        row.last # For update, row has two arrays (old and new values)  Use new values
+        # For update, row has two arrays (old and new values)
+        { OLD => row.first, ROW => row.last }
       end
     end
 
@@ -44,10 +46,13 @@ module Mysql
       emit_record(type, record) do |opt|
         table_name = record['table_name']
         records = record["rows"].collect do |row|
-          row = yield(row) if block_given?  # Give the caller a chance to generate the correct row
-          { ROW => convert_to_flydata_row_format(table_name, row) }
+          row_values = { ROW => row }
+          row_values = yield(row) if block_given?  # Give the caller a chance to generate the correct row values
+          row_types = row_values.keys
+          row_kinds = row_types.inject({}) {|m, k| m[k] = convert_to_flydata_row_format(table_name, row_values[k]); m}
+          encode_signless_integer(row_kinds, record["columns"], row_types)
+          row_kinds
         end
-        encode_signless_integer(records, record["columns"])
         records
       end
     end
@@ -70,19 +75,19 @@ module Mysql
       end
     end
 
-    def encode_signless_integer(records, column_types)
-      records.each do |record|
-        record[ROW].keys.each do |position|
+    def encode_signless_integer(record, column_types, row_types)
+      row_types.each do |row_type|
+        record[row_type].keys.each do |position|
           index = position.to_i - 1
           column_type = column_types[index]
           if INTEGER_TYPES.keys.include?(column_type)
             # It's a signless integer.
-            intval = record[ROW][position]
+            intval = record[row_type][position]
             next unless (intval.kind_of?(Numeric) || intval =~ /^-?[\d]+$/)
             width = INTEGER_TYPES[column_type] * 2 # * 2 because a single byte requires two characters (e.g. ff)
             signless_val = SIGNLESS_INTEGER_PREFIX
             signless_val += sprintf("%0#{width}x", intval).gsub(/\.\.f/, 'f' * width).slice(-width..-1)
-            record[ROW][position] = signless_val
+            record[row_type][position] = signless_val
           end
         end
       end

data/lib/flydata/helpers.rb

@@ -12,25 +12,9 @@ module Flydata
       [klass, method]
     end
 
-    def usage_text(err = true)
+    def usage_text
       text = ""
 
-      if err
-        text += <<-EOM
-Fix the issue and try again.  If the problem continues, please contact support@flydata.com
-
-EOM
-
-        flydata_log = File.join(FLYDATA_HOME, 'flydata.log')
-        if File.exists?(flydata_log)
-          text += <<-EOM
-Also check the Agent log.
-   Log path: #{flydata_log}
-EOM
-        end
-      end
-
-      text += ""
       text += '-' * 64
       text += "\n"
       text += <<-EOM

data/spec/flydata/cli_spec.rb

@@ -27,7 +27,7 @@ module Flydata
           context 'with options' do
             let(:options) { %w(-a --host-name localhost) }
             it 'returns with an error' do
-              expect($stderr).to receive(:puts).with('! Unknown options -a, --host-name').once
+              expect($stderr).to receive(:puts).with('!! Unknown options -a, --host-name').once
               allow($stderr).to receive(:puts)
 
               expect{ subject.run }.to terminate.with_code(1)
@@ -62,7 +62,7 @@ module Flydata
           context 'with optionss' do
             let(:options) { %w(-a --host-name localhost) }
             it 'returns an error' do
-              expect($stderr).to receive(:puts).with('! Unknown options -a, --host-name').once
+              expect($stderr).to receive(:puts).with('!! Unknown options -a, --host-name').once
               allow($stderr).to receive(:puts)
 
               expect{ subject.run }.to terminate.with_code(1)

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -164,9 +164,10 @@ EOT
       rows = [rows] unless rows.kind_of?(Array)
       @seq ||= TEST_SEQUENCE_NUM
       records = rows.collect do |row|
+        record = row.kind_of?(Hash) && row.keys.include?(:row) ? row : { row: row }
         @seq += 1
-        { :type=>type, :table_name=>table, :respect_order=>true, :seq=>@seq,
-          :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1, :row=>row }
+        record.merge({ :type=>type, :table_name=>table, :respect_order=>true,
+          :seq=>@seq, :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1 })
       end
       expect_emitted_records(event, records)
     end
@@ -289,21 +290,30 @@ EOT
         context 'when received update event' do
           it do
             expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
-              [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
+              [{old:{"1"=>"0SL00000001", "2"=>"foo"},
+              row:{"1"=>"0SL00000001", "2"=>"wow"}},
+             {old:{"1"=>"0SL00000003", "2"=>"hoge"},
+              row:{"1"=>"0SL00000003", "2"=>"fuga"}}])
           end
         end
 
         context 'when received update event with two byte utf8 chars' do
           it do
             expect_emitted_records_with_rows(update_two_byte_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
-              [{"1"=>"0SL00000001", "2"=>"∑ø∑"}, {"1"=>"0SL00000003", "2"=>"fügå"}])
+              [{old:{"1"=>"0SL00000001", "2"=>"føø"},
+              row:{"1"=>"0SL00000001", "2"=>"∑ø∑"}},
+             {old:{"1"=>"0SL00000003", "2"=>"høgé"},
+              row:{"1"=>"0SL00000003", "2"=>"fügå"}}])
           end
         end
 
         context 'when received update event with three byte utf8 chars' do
           it do
             expect_emitted_records_with_rows(update_three_byte_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
-              [{"1"=>"0SL00000001", "2"=>"很兴奋"}, {"1"=>"0SL00000003", "2"=>"興奮虎"}])
+              [{old:{"1"=>"0SL00000001", "2"=>"富无无"},
+              row:{"1"=>"0SL00000001", "2"=>"很兴奋"}},
+             {old:{"1"=>"0SL00000003", "2"=>"切实切实"},
+              row:{"1"=>"0SL00000003", "2"=>"興奮虎"}}])
           end
         end
 
@@ -317,7 +327,7 @@ EOT
               table_rev: 2,  # increment revision
               seq: 2,
               actions: [{
-                action: :add_column, column: "sum", :type=>'int4', :query=>'add column sum integer'}],
+                action: :add_column, column: "sum", :type=>'int4(11)', :query=>'add column sum integer'}],
             })
           end
         end
@@ -395,7 +405,10 @@ EOT
         it 'logs a warning and emits FET with a blank binlog file name, when it receives an update event'  do
           expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
           expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "",
-            [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
+            [{old:{"1"=>"0SL00000001", "2"=>"foo"},
+              row:{"1"=>"0SL00000001", "2"=>"wow"}},
+             {old:{"1"=>"0SL00000003", "2"=>"hoge"},
+              row:{"1"=>"0SL00000003", "2"=>"fuga"}}])
         end
 
         it 'logs a warning emits FET with a blank binlog file name, when it receives a delete event' do
@@ -409,7 +422,10 @@ EOT
           expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "",
             [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
           expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "",
-            [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
+            [{old:{"1"=>"0SL00000001", "2"=>"foo"},
+              row:{"1"=>"0SL00000001", "2"=>"wow"}},
+             {old:{"1"=>"0SL00000003", "2"=>"hoge"},
+              row:{"1"=>"0SL00000003", "2"=>"fuga"}}])
           expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "",
             [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
           expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "",
@@ -475,7 +491,7 @@ EOT
             event = update_event
             event['table_name'] = TEST_TABLE_APPEND_ONLY
             expect_emitted_records_with_rows(event, :update, TEST_TABLE_APPEND_ONLY, 2528, "mysql-bin.000048",
-              [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
+              [{old:{"1"=>"0SL00000001", "2"=>"foo"}, row:{"1"=>"0SL00000001", "2"=>"wow"}}, {old:{"1"=>"0SL00000003", "2"=>"hoge"}, row:{"1"=>"0SL00000003", "2"=>"fuga"}}])
           end
 
           it 'emits a record when it receives a delete event for non-append only table' do

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -136,7 +136,7 @@ describe 'MysqlAlterTableParser' do
             },{
               action: :add_column,
               column: "value2",
-              type: "int4",
+              type: "int4(11)",
               auto_increment: true,
               query: "add (value1 varchar(26) not null default 'flydata', value2 int auto_increment)"
             }]
@@ -160,7 +160,7 @@ describe 'MysqlAlterTableParser' do
             },{
               action: :add_column,
               column: "value2",
-              type: "int4",
+              type: "int4(11)",
               auto_increment: true,
               query: "add (`value1` varchar(26) not null default 'flydata', `value2` int auto_increment)"
             }]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.30
+  version: 0.3.0
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-26 00:00:00.000000000 Z
+date: 2015-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -496,6 +496,7 @@ files:
 - lib/flydata/compatibility_check.rb
 - lib/flydata/credentials.rb
 - lib/flydata/cron.rb
+- lib/flydata/errors.rb
 - lib/flydata/fluent-plugins/idle_event_detector.rb
 - lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb
 - lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb
@@ -569,7 +570,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent