flydata 0.5.13 → 0.5.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d4b03db81cb7465b262dce0c4c8e33a14b5e290b
-  data.tar.gz: 42c08437e1b5f0d8452f2f2504666d6ce83006a7
+  metadata.gz: b2ba68dee697e96ae4e45d7c4ab93da8bfdd0bd9
+  data.tar.gz: 5a85b888bf9151233e6163bf825ce52865b5d218
 SHA512:
-  metadata.gz: 6de4243e12d2ec8d0c0c62a68a21fd9f1bef9f5a40caebe494687656ab5714d1d88e56ff7aa091796c52a57321e74e6c287ddab02e3968a0029e0203ff3ea47a
-  data.tar.gz: 910e524b426859dae7e5996bf25148aedf2eebea36decbc602965b483d8cf010ff224e998707c6d35bac7f80036ff0f609ef6dfd88410bf562700d967ad30b34
+  metadata.gz: 2ba24a57fee86c3e979c7acf571f73a1d8f12dfc0810f7925d34dd3b7c5d63ddee4d65bc8c2299e204224b93d3a0ab3cfd013dbab3425048e7d1c232328c7658
+  data.tar.gz: 0cd58ad653a31e062deb6801bd682cde6940ea2d1c8e8758f72304bd527dbd9549ceaf39ec594f3893f86ca2fa49683b18928ee13bde50b6a2929391df92c6ad

data/VERSION

@@ -1 +1 @@
-0.5.13
+0.5.14

data/ext/flydata/parser/mysql/parser.txt

@@ -6,7 +6,7 @@ value:
   " ", "\t", "\n", "," => :value
   "'" => :first_char
   "0" => (mark), :first_zero
-  [1-9-] => (mark), :number
+  [1-9-+\.] => (mark), :number
   "N" => :null_n
   ")" => (row_end_callback), :start
   else => :error
@@ -29,7 +29,7 @@ leading_zero:
   else => :error
 
 number:
-  [0-9.] => :number
+  [0-9.-eE] => :number
   "," => (value_callback, :end_value)(mark reset), :value
   ")" => (value_callback, :end_value)(mark reset)(row_end_callback), :start
   else => :error

data/ext/flydata/parser/mysql/sql_parser.cpp

@@ -62,7 +62,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         } else if (*current == '0') {
           mark = current; value_len = 1;//mark
           status = first_zero;
-        } else if ( (*current >= '1' && *current <= '9') || *current == '-' ) {
+        } else if ( (*current >= '1' && *current <= '9') || *current == '-' || *current == '+' || *current == '.') {
           mark = current; value_len = 1;//mark
           status = number;
         } else if (*current == ',') {
@@ -127,7 +127,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
 
       case number:
            if ($DEBUG) std::cout << "number[" << *current << "]: " << std::endl << std::flush;
-        if ((*current >= '0' && *current <= '9') || *current == '.') {
+        if ((*current >= '0' && *current <= '9') || *current == '.' || *current == 'e' || *current == 'E' || *current == '-') {
           status = number;
         } else if (*current == ',') {
           ch.value_callback(mark, value_len - 1, true);//value_callback :end_value
@@ -389,10 +389,12 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
           ch.value_callback("\x1a", 1, false);//value_callback '\Z'
           mark = 0; value_len = 0;
           status = first_char;
-        } else if (*current == '0') {
-          ch.value_callback("\0", 1, false);//value_callback '\0'
-          mark = 0; value_len = 0;
-          status = first_char;
+// Disabled handling of \0 for now in order to keep it compatible with the old
+// implementation.
+//        } else if (*current == '0') {
+//          ch.value_callback("\0", 1, false);//value_callback '\0'
+//          mark = 0; value_len = 0;
+//          status = first_char;
         } else {
           status = following_char;
         }

data/flydata-core/lib/flydata-core/mysql/command_generator.rb

@@ -51,9 +51,10 @@ module FlydataCore
         default_option = nil if default_option == ''
 
         custom_option = option[:custom_option]
+        custom_option_end = option[:custom_option_end] # put in the end of the command
 
         [command, host, port, username, password, default_option,
-         custom_option, database, tables].compact.join(' ')
+         custom_option, database, tables, custom_option_end].compact.join(' ')
       end
 
       # DDL_DUMP_CMD_TEMPLATE = "MYSQL_PWD=\"%s\" mysqldump --protocol=tcp -d -h %s -P %s -u %s %s %s"

data/flydata-core/spec/mysql/command_generator_spec.rb

@@ -114,6 +114,13 @@ module FlydataCore
             'mysql -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-auto-rehash -e "SHOW GRANTS;" testdb'
           ) }
         end
+
+        context 'when custom_option_end is set' do
+          before { option[:custom_option_end] = 'test_table' }
+          it { is_expected.to eq(
+            'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp testdb test_table'
+          ) }
+        end
       end
 
       describe '.generate_mysql_ddl_dump_cmd' do

data/lib/flydata/command/mysql.rb

@@ -1,38 +1,10 @@
-require 'open3'
 require 'flydata/command/sync'
-require 'flydata-core/mysql/command_generator'
+require 'flydata/command/mysql_command_base'
 
 module Flydata
   module Command
-
     class Mysql < Sync
-      def run
-        de = retrieve_sync_data_entry
-        de['mysql_data_entry_preference'].delete('tables')
-        cmd = FlydataCore::Mysql::CommandGenerator::generate_mysql_cmd(de['mysql_data_entry_preference'])
-        if $stdin.tty?
-          # interactive shell
-          system cmd
-        else
-          # execute queries given to $stdin
-          Open3.popen2e(cmd) do |i, o, wt|
-            $stdin.each_line do |line|
-              i.print line
-            end
-            i.close
-            while line = o.gets
-              print line
-            end
-          end
-        end
-      end
-
-      def flush; end
-      def reset; end
-      def skip; end
-      def generate_table_ddl; end
-      def fix_binlogpos; end
+      include MysqlCommandBase
     end
-
   end
 end

data/lib/flydata/command/mysql_command_base.rb

@@ -0,0 +1,46 @@
+require 'open3'
+require 'flydata-core/mysql/command_generator'
+
+module Flydata
+  module Command
+
+    module MysqlCommandBase
+      def run(*args)
+        de = retrieve_sync_data_entry
+        cmd = generate_command(de['mysql_data_entry_preference'], args)
+        return if cmd.to_s.empty?
+        $stderr.puts "command:#{cmd}" if FLYDATA_DEBUG
+        if $stdin.tty?
+          # interactive shell
+          system cmd
+        else
+          # execute queries given to $stdin
+          Open3.popen2e(cmd) do |i, o, wt|
+            $stdin.each_line do |line|
+              i.print line
+            end
+            i.close
+            while line = o.gets
+              print line
+            end
+          end
+        end
+      end
+
+      def generate_command(dbconf, args)
+        dbconf.delete('tables')
+        dbconf[:custom_option_end] = args.join(' ')
+        FlydataCore::Mysql::CommandGenerator::generate_mysql_cmd(dbconf)
+      end
+
+      def flush; end
+      def reset; end
+      def skip; end
+      def generate_table_ddl; end
+      def fix_binlogpos; end
+    end
+
+  end
+end
+
+

data/lib/flydata/command/mysqlbinlog.rb

@@ -0,0 +1,33 @@
+require 'flydata/command/sync'
+require 'flydata/command/mysql_command_base'
+
+module Flydata
+  module Command
+    class Mysqlbinlog < Sync
+      include MysqlCommandBase
+
+      def generate_command(dbconf, args)
+        dbconf.delete('tables')
+        dbconf[:custom_option_end] = args.join(' ')
+        dbconf[:command] = 'mysqlbinlog'
+        # --default-character-set=utf8 is not supported
+        dbconf[:no_default_option] = true
+        # ssl is not supported for mysql5.6 or older mysqlbinlog command
+        dbconf.delete('ssl_ca')
+        dbconf.delete('ssl_cipher')
+        dbconf.delete('database')
+        if args.empty?
+          puts <<EOT
+example:
+1. Dump raw binlog
+  flydata mysqlbinlog --raw --read-from-remote-server --result-file "binlog-" mysql-bin-changelog.xxxx
+2. Dump decoded binlog data
+  flydata mysqlbinlog  -vv --base64-output=decode-row --hexdump --read-from-remote-server --start-position=4 --stop-position=yyyyy mysql-bin-changelog.xxxxx > mysql-bin-changelog.xxxxx
+EOT
+          return
+        end
+        FlydataCore::Mysql::CommandGenerator::generate_mysql_cmd(dbconf)
+      end
+    end
+  end
+end

data/lib/flydata/command/mysqldump.rb

@@ -0,0 +1,17 @@
+require 'flydata/command/sync'
+require 'flydata/command/mysql_command_base'
+
+module Flydata
+  module Command
+    class Mysqldump < Sync
+      include MysqlCommandBase
+
+      def generate_command(dbconf, args)
+        dbconf.delete('tables')
+        dbconf[:custom_option_end] = args.join(' ')
+        dbconf[:command] = 'mysqldump'
+        FlydataCore::Mysql::CommandGenerator::generate_mysql_cmd(dbconf)
+      end
+    end
+  end
+end

data/lib/flydata/command/sync.rb

@@ -18,7 +18,7 @@ require 'flydata-core/table_def'
 require 'flydata-core/mysql/binlog_pos'
 require 'flydata/mysql/table_ddl'
 require 'flydata-core/mysql/command_generator'
-#require 'ruby-prof'
+#require 'ruby-prof' # to enable profiling, also set the class' RUN_PROFILE
 
 module Flydata
   module Command
@@ -26,6 +26,8 @@ module Flydata
     class Sync < Base
       include Helpers
 
+      RUN_PROFILE = false
+
       INSERT_PROGRESS_INTERVAL = 1000
       SERVER_DATA_PROCESSING_TIMEOUT = 3600 # seconds
 
@@ -784,6 +786,7 @@ EOM
 
         begin
           threads = []
+          RubyProf.start if RUN_PROFILE and defined?(RubyProf) and not RubyProf.running?
           Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
             mysqldump_io,
             # create table
@@ -792,12 +795,13 @@ EOM
               # dump mysql_table for resume
               #TODO: make it option
               sync_fm.save_mysql_table_marshal_dump(mysql_table)
+              mysql_table.set_value_converters(FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS)
             },
             # insert record
             Proc.new { |mysql_table, values_set|
               mysql_table_name = mysql_table.table_name
               records = values_set.collect do |values|
-                values = convert_to_flydata_values(mysql_table, values)
+                convert_to_flydata_values(mysql_table, values)
                 json = JSON.generate_kv_pairs(mysql_table.column_names, values)
                 {table_name: mysql_table_name, log: json}
               end
@@ -839,6 +843,14 @@ EOM
           ee.description = "  Please contact support@flydata.com to report the issue."
           ee.set_backtrace e.backtrace
           raise ee
+        ensure
+          if RUN_PROFILE and defined?(RubyProf) and RubyProf.running?
+            result = RubyProf.stop
+            #printer = RubyProf::GraphPrinter.new(result)
+            printer = RubyProf::GraphHtmlPrinter.new(result)
+            #printer.print(STDOUT)
+            printer.print(File.new("ruby-prof-out-#{Time.now.to_i}.html", "w"), :min_percent => 3)
+          end
         end
         forwarder.close
         log_info_stdout("  -> Done")
@@ -985,15 +997,12 @@ EOM
       end
 
       def convert_to_flydata_values(mysql_table, values)
-        types = mysql_table.format_types
-        sz = types.size
-        result = []
-        i = 0
-        while i < sz
-          result << FlydataCore::TableDef::MysqlTableDef.convert_to_flydata_value(values[i], types[i])
-          i += 1
+        vc = mysql_table.value_converters
+        return if vc.empty?
+
+        vc.each_pair do |index, converter|
+          values[index] = converter.call(values[index])
         end
-        result
       end
 
       def generate_json(column_names, values)

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -3,6 +3,7 @@ require 'io/wait'
 require 'mysql2'
 require 'flydata-core/mysql/config'
 require 'flydata-core/mysql/command_generator'
+#require 'ruby-prof' # to enable profiling, also set the class' RUN_PROFILE
 
 module Flydata
   module Parser
@@ -33,18 +34,26 @@ module Flydata
           @table_name = table_name
           @columns = columns
           @column_names = columns.collect{|k,v| v[:column_name]}
-          @format_types = columns.collect{|k,v| v[:format_type]}
           @primary_keys = primary_keys
+          @value_converters = {}
         end
 
-        attr_accessor :table_name, :columns, :column_names, :format_types,
-                      :primary_keys
+        attr_accessor :table_name, :columns, :column_names, :primary_keys
+        attr_reader :value_converters
 
         def add_column(column)
           cn = column[:column_name]
           @columns[cn] = column
           @column_names << cn
-          @format_types << column[:format_type]
+        end
+
+        def set_value_converters(converter_hash)
+          @columns.each_with_index do |(k, v), i|
+            type = v[:format_type]
+            if converter_hash.has_key?(type)
+              @value_converters[i] = converter_hash[type]
+            end
+          end
         end
       end
 
@@ -475,12 +484,14 @@ EOS
         class InsertParser
           #INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
 
+          RUN_PROFILE = false
+
           def start_ruby_prof
-            RubyProf.start if defined?(RubyProf) and not RubyProf.running?
+            RubyProf.start if RUN_PROFILE and defined?(RubyProf) and not RubyProf.running?
           end
 
           def stop_ruby_prof
-            if defined?(RubyProf) and RubyProf.running?
+            if RUN_PROFILE and defined?(RubyProf) and RubyProf.running?
               result = RubyProf.stop
               #printer = RubyProf::GraphPrinter.new(result)
               printer = RubyProf::GraphHtmlPrinter.new(result)

data/spec/flydata/command/sync_spec.rb

@@ -254,6 +254,10 @@ module Flydata
         subject { subject_object.send(:convert_to_flydata_values, mysql_table, values) }
         let(:values) { [4, 'John', nil, col4_value, nil, nil] }
 
+        before do
+          mysql_table.set_value_converters(FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS)
+        end
+
         context 'with binary column' do
           let(:col4_type) { "binary" }
           let(:col4_width) { 5 }
@@ -263,7 +267,8 @@ module Flydata
           it 'truncates trailing "0" if the type is binary' do
             expected_values = values.dup
             expected_values[3] = truncated_binary
-            is_expected.to eq expected_values
+            subject
+            expect(values).to eq expected_values
           end
         end
       end

data/spec/flydata/parser/mysql/dump_parser_spec.rb

@@ -818,11 +818,9 @@ EOT
             context 'with a single escape char' do
               let(:value) { %q|\\0| }
               #let(:expected_value) { %Q|\0| }
-              #it 'returns correct chars' do
-              #  expect(subject).to eq([[expected_value]])
-              #end
-              it 'returns an error (for now)' do
-                expect{subject}.to raise_error(ArgumentError)
+              let(:expected_value) { value }  # this is compatible with the previous implementation but it's actually wrong.  For now, keeping it as is because allowing the null character in a string could open a can of worms.
+              it 'returns correct chars' do
+                expect(subject).to eq([[expected_value]])
               end
             end
             context 'with an escape char followed by a regular char' do
@@ -974,6 +972,49 @@ EOT
             end
           end
 
+          context 'when a scientific number is given' do
+            let(:target_line) {"INSERT INTO `test_table` VALUES (1,#{num_str});"}
+            shared_examples "accepting the value as is" do
+              it 'should accept the value as is' do
+                expect(subject).to eq([['1', num_str]])
+              end
+            end
+            context 'capital E' do
+              let(:num_str) { '1.2E3' }
+              it_behaves_like "accepting the value as is"
+            end
+            context 'lowercase e' do
+              let(:num_str) { '1.2e3' }
+              it_behaves_like "accepting the value as is"
+            end
+            context 'negative integer' do
+              let(:num_str) { '-1.2e3' }
+              it_behaves_like "accepting the value as is"
+            end
+            context 'negative fractional' do
+              let(:num_str) { '1.2e-3' }
+              it_behaves_like "accepting the value as is"
+            end
+            context 'negative integer & fractional' do
+              let(:num_str) { '-1.2e-3' }
+              it_behaves_like "accepting the value as is"
+            end
+          end
+
+          context 'when a number with positive sign is given' do
+            let(:target_line) {"INSERT INTO `test_table` VALUES (1,+9.10);"}
+            it 'should accept the value as is' do
+              expect(subject).to eq([['1', '+9.10']])
+            end
+          end
+
+          context 'when a number with no leading zero is given' do
+            let(:target_line) {"INSERT INTO `test_table` VALUES (1,.10);"}
+            it 'should accept the value as is' do
+              expect(subject).to eq([['1', '.10']])
+            end
+          end
+
           context 'when 0.00 is given' do
             let(:target_line) {"INSERT INTO `test_table` VALUES (1,0.00);"}
             it 'should not remove the leading zeros' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.13
+  version: 0.5.14
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-26 00:00:00.000000000 Z
+date: 2015-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -582,6 +582,9 @@ files:
 - lib/flydata/command/kill_all.rb
 - lib/flydata/command/login.rb
 - lib/flydata/command/mysql.rb
+- lib/flydata/command/mysql_command_base.rb
+- lib/flydata/command/mysqlbinlog.rb
+- lib/flydata/command/mysqldump.rb
 - lib/flydata/command/restart.rb
 - lib/flydata/command/routine.rb
 - lib/flydata/command/sender.rb