flydata 0.5.12 → 0.5.13

data/spec/flydata/command/sync_spec.rb

@@ -40,13 +40,14 @@ module Flydata
             "data_servers"=>"localhost:9905" }
          }
       end
+      let(:col4_type) { "binary" }
+      let(:col4_width) { 34 }
+      let(:col4_type_str) { "#{col4_type}(#{col4_width})" }
       let(:mysql_table_columns) {
-        {"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>"binary(34)", :format_type=>"binary", :format_size=>34, :default=>nil}, "bin2"=>{:column_name=>"bin2", :format_type_str=>"blob", :format_type=>"blob"}, "varbin"=>{:column_name=>"varbin", :format_type_str=>"varchar(34)", :format_type=>"varchar", :format_size=>34, :default=>nil}}
+        {"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>col4_type_str, :format_type=>col4_type, :format_size=>col4_width, :default=>nil}, "bin2"=>{:column_name=>"bin2", :format_type_str=>"blob", :format_type=>"blob"}, "varbin"=>{:column_name=>"varbin", :format_type_str=>"varchar(34)", :format_type=>"varchar", :format_size=>34, :default=>nil}}
       }
       let(:mysql_table) do
-        mt = double('mysql_table')
-        allow(mt).to receive(:columns).and_return(mysql_table_columns)
-        mt
+        Flydata::Parser::Mysql::MysqlTable.new("test_table", mysql_table_columns, ['id'])
       end
 
       after :each do
@@ -154,7 +155,7 @@ module Flydata
         context 'with full options' do
           it 'issues mysqldump command with expected parameters' do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 3306 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
+             'mysqldump -h localhost -P 3306 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -176,7 +177,7 @@ module Flydata
           end
           it "uses the default port" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
+             'mysqldump -h localhost -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -186,7 +187,7 @@ module Flydata
           end
           it "uses the specified port" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 1234 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
+             'mysqldump -h localhost -P 1234 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -219,7 +220,7 @@ module Flydata
           end
           it "call mysqldump with MYSQL_PW set with correct symbols" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump -h localhost -P 3306 -umasashi -p"welcome&!@^@#^" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
+             'mysqldump -h localhost -P 3306 -umasashi -pwelcome\\&\\!@\\^@\\#\\^ --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -251,13 +252,19 @@ module Flydata
       end
       describe '#convert_to_flydata_values' do
         subject { subject_object.send(:convert_to_flydata_values, mysql_table, values) }
-        let(:values) { [4, 'John', nil, '0xC0448200', nil, nil] }
+        let(:values) { [4, 'John', nil, col4_value, nil, nil] }
 
-        it 'calls convert_to_valydata_value for each value' do
-          mysql_table.columns.values.each_with_index do |col_info, i|
-            expect(FlydataCore::TableDef::MysqlTableDef).to receive(:convert_to_flydata_value).with(values[i], col_info[:format_type]).and_return(values[i])
+        context 'with binary column' do
+          let(:col4_type) { "binary" }
+          let(:col4_width) { 5 }
+          let(:truncated_binary) { "0xC04482" }
+          let(:col4_value) { "#{truncated_binary}0000" }
+
+          it 'truncates trailing "0" if the type is binary' do
+            expected_values = values.dup
+            expected_values[3] = truncated_binary
+            is_expected.to eq expected_values
           end
-          is_expected.to eq values
         end
       end
     end

data/spec/flydata/parser/mysql/dump_parser_spec.rb

@@ -812,6 +812,56 @@ EOT
             end
           end
 
+          context 'with escape chars' do
+            let(:target_line) { "INSERT INTO `test_table` VALUES ('#{value}');" }
+
+            context 'with a single escape char' do
+              let(:value) { %q|\\0| }
+              #let(:expected_value) { %Q|\0| }
+              #it 'returns correct chars' do
+              #  expect(subject).to eq([[expected_value]])
+              #end
+              it 'returns an error (for now)' do
+                expect{subject}.to raise_error(ArgumentError)
+              end
+            end
+            context 'with an escape char followed by a regular char' do
+              let(:value) { %q|\\'m| }
+              let(:expected_value) { %Q|'m| }
+              it 'returns correct chars' do
+                expect(subject).to eq([[expected_value]])
+              end
+            end
+            context 'with two consecutive escape chars' do
+              let(:value) { %Q|\\"\b| }
+              let(:expected_value) { %Q|"\b| }
+              it 'returns correct chars' do
+                expect(subject).to eq([[expected_value]])
+              end
+            end
+            context 'with a escape char following a regular char' do
+              let(:value) { %q|a\\n| }
+              let(:expected_value) { %Q|a\n| }
+              it 'returns correct chars' do
+                expect(subject).to eq([[expected_value]])
+              end
+            end
+            context 'with various escape chars 1' do
+              let(:value) { %Q|\\r\t\\Z| }
+              let(:expected_value) { %Q|\r\t\x1a| }
+              it 'returns correct chars' do
+                expect(subject).to eq([[expected_value]])
+              end
+            end
+            context 'with various escape chars 2' do
+              let(:value) { %q|c\\\\\\\\%\\\\_| }
+              let(:expected_value) { %q|c\\\\%\\_| }
+              it 'returns correct chars' do
+                expect(subject).to eq([[expected_value]])
+              end
+            end
+          end
+
           context 'when data end with back slash' do
             let(:target_line) { "INSERT INTO `test_table` VALUES (1,'D:\\\\download\\\\','2014-04-15 13:49:14');" }
             it 'should escape back slash' do
@@ -819,6 +869,13 @@ EOT
             end
           end
 
+          context 'when a regular character is escaped' do
+            let(:target_line) { "INSERT INTO `test_table` VALUES (1,'\\a','2014-04-15 13:49:14');" }
+            it 'should leave the back slash as is' do
+              expect(subject).to eq([['1',"\\a",'2014-04-15 13:49:14']])
+            end
+          end
+
           context 'when comma is the first character of a string' do
             let(:target_line) { "INSERT INTO `test_table` VALUES (1,',9','2014-04-15 13:49:14');" }
             it 'should parse the string correctly' do
@@ -923,6 +980,53 @@ EOT
               expect(subject).to eq([['1', '0.00']])
             end
           end
+
+          context 'when a hex blob is given' do
+            let(:target_line) {"INSERT INTO `test_table` VALUES (#{hex_blob});"}
+            context 'uppercase hex' do
+              let(:hex_blob) { '0x5CFA' }
+              it 'extracts the same string' do
+                expect(subject).to eq([[hex_blob]])
+              end
+            end
+            context 'lowercase hex' do
+              let(:hex_blob) { '0x5cfa' }
+              it 'extracts the same string' do
+                expect(subject).to eq([[hex_blob]])
+              end
+            end
+          end
+
+          context 'when a negative number is given' do
+            let(:target_line) {"INSERT INTO `test_table` VALUES (1,-0.004,-293);"}
+            it 'should preserve the minus sign' do
+              expect(subject).to eq([['1', '-0.004', '-293']])
+            end
+          end
+
+          context 'when NULL is given' do
+            context 'in the end' do
+              let(:target_line) {"INSERT INTO `test_table` VALUES (1,NULL);"}
+              it 'should preserve the minus sign' do
+                expect(subject).to eq([['1', nil]])
+              end
+            end
+            context 'in the middle' do
+              let(:target_line) {"INSERT INTO `test_table` VALUES (1,NULL,'hi');"}
+              it 'should preserve the minus sign' do
+                expect(subject).to eq([['1', nil, 'hi']])
+              end
+            end
+          end
+          context 'with many columns' do
+            let(:target_line) {<<EOS.chomp}
+INSERT INTO `type_without_width_info` VALUES (1098238,0x5C,0x01,0x6974206C6F6F6B73206C696B65206120746578742062757420697427732062696E617279,0,1,'c','2015-10-18','2015-10-18 23:12:04',4,192,292.22,92,293.83,222,292,NULL,'I\\'m a \\\\ very long text',NULL,29838,'sdaklfj;',2983,12,'\\',\\\\','04:58:21','2015-10-18 12:34:56',NULL,-1,'28',1),(-2938,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,-0.49,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,2);
+EOS
+            it 'should preserve the minus sign' do
+              expect(subject).to eq([['1098238','0x5C','0x01','0x6974206C6F6F6B73206C696B65206120746578742062757420697427732062696E617279','0','1','c','2015-10-18','2015-10-18 23:12:04','4','192','292.22','92','293.83','222','292',nil,%q|I'm a \\ very long text|,nil,'29838','sdaklfj;','2983','12','\',\\','04:58:21','2015-10-18 12:34:56',nil,'-1','28','1'],['-2938',nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,'-0.49',nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,'2']])
+            end
+          end
+
         end
       end
     end

data/spec/flydata/sync_file_manager_spec.rb

@@ -120,7 +120,7 @@ module Flydata
     describe '#load_dump_pos' do
       let (:mysql_table) do
         Flydata::Parser::Mysql::MysqlTable.new(
-          table_name, { 'id' => { format_type: 'int' }, 'value' => { format_type: 'text' } }
+          table_name, { 'id' => { column_name: 'id', format_type: 'int' }, 'value' => { column_name: 'value', format_type: 'text' } }
         )
       end
 
@@ -138,8 +138,8 @@ module Flydata
           })
           expect(mt.table_name).to eq(table_name)
           expect(mt.columns).to eq({
-            'id' => { format_type: 'int' },
-            'value' => { format_type: 'text' },
+            'id' => { column_name: 'id', format_type: 'int' },
+            'value' => { column_name: 'value', format_type: 'text' },
           })
         end
       end

data/test-suite.sh

@@ -34,6 +34,7 @@ fi
 # Test flydata-agent library (flydata-agent/lib, flydata-agent/spec)
 test_main() {
     cd $BASE_DIR
+    bundle exec rake compile
     bundle install
     bundle exec rspec
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.12
+  version: 0.5.13
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-08 00:00:00.000000000 Z
+date: 2015-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -444,13 +444,35 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.5
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.5
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
 description: FlyData Agent
 email: sysadmin@flydata.com
 executables:
 - fdmysqldump
 - flydata
 - serverinfo
-extensions: []
+extensions:
+- ext/flydata/json/extconf.rb
+- ext/flydata/parser/mysql/extconf.rb
 extra_rdoc_files: []
 files:
 - .rspec
@@ -467,6 +489,15 @@ files:
 - bin/flydata
 - bin/serverinfo
 - circle.yml
+- ext/flydata/flydata.h
+- ext/flydata/json/extconf.rb
+- ext/flydata/json/json_ext.cpp
+- ext/flydata/parser/mysql/.gitignore
+- ext/flydata/parser/mysql/dump_parser_ext.cpp
+- ext/flydata/parser/mysql/extconf.rb
+- ext/flydata/parser/mysql/parser.txt
+- ext/flydata/parser/mysql/sql_parser.cpp
+- ext/flydata/parser/mysql/sql_parser.h
 - flydata-core/.gitignore
 - flydata-core/.rspec
 - flydata-core/.travis.yml
@@ -600,10 +631,13 @@ files:
 - lib/flydata/heroku.rb
 - lib/flydata/heroku/configuration_methods.rb
 - lib/flydata/heroku/instance_methods.rb
+- lib/flydata/json.rb
+- lib/flydata/json/.gitignore
 - lib/flydata/log_monitor.rb
 - lib/flydata/mysql/binlog_position.rb
 - lib/flydata/mysql/table_ddl.rb
 - lib/flydata/output/forwarder.rb
+- lib/flydata/parser/mysql/.gitignore
 - lib/flydata/parser/mysql/dump_parser.rb
 - lib/flydata/parser/mysql/mysql_alter_table.treetop
 - lib/flydata/parser_provider.rb
@@ -692,7 +726,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent