flydata 0.5.12 → 0.5.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -40,13 +40,14 @@ module Flydata
40
40
  "data_servers"=>"localhost:9905" }
41
41
  }
42
42
  end
43
+ let(:col4_type) { "binary" }
44
+ let(:col4_width) { 34 }
45
+ let(:col4_type_str) { "#{col4_type}(#{col4_width})" }
43
46
  let(:mysql_table_columns) {
44
- {"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>"binary(34)", :format_type=>"binary", :format_size=>34, :default=>nil}, "bin2"=>{:column_name=>"bin2", :format_type_str=>"blob", :format_type=>"blob"}, "varbin"=>{:column_name=>"varbin", :format_type_str=>"varchar(34)", :format_type=>"varchar", :format_size=>34, :default=>nil}}
47
+ {"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>col4_type_str, :format_type=>col4_type, :format_size=>col4_width, :default=>nil}, "bin2"=>{:column_name=>"bin2", :format_type_str=>"blob", :format_type=>"blob"}, "varbin"=>{:column_name=>"varbin", :format_type_str=>"varchar(34)", :format_type=>"varchar", :format_size=>34, :default=>nil}}
45
48
  }
46
49
  let(:mysql_table) do
47
- mt = double('mysql_table')
48
- allow(mt).to receive(:columns).and_return(mysql_table_columns)
49
- mt
50
+ Flydata::Parser::Mysql::MysqlTable.new("test_table", mysql_table_columns, ['id'])
50
51
  end
51
52
 
52
53
  after :each do
@@ -154,7 +155,7 @@ module Flydata
154
155
  context 'with full options' do
155
156
  it 'issues mysqldump command with expected parameters' do
156
157
  expect(Open3).to receive(:popen3).with(
157
- 'mysqldump -h localhost -P 3306 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
158
+ 'mysqldump -h localhost -P 3306 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
158
159
  subject.send(:do_generate_table_ddl, default_data_entry)
159
160
  end
160
161
  end
@@ -176,7 +177,7 @@ module Flydata
176
177
  end
177
178
  it "uses the default port" do
178
179
  expect(Open3).to receive(:popen3).with(
179
- 'mysqldump -h localhost -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
180
+ 'mysqldump -h localhost -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
180
181
  subject.send(:do_generate_table_ddl, default_data_entry)
181
182
  end
182
183
  end
@@ -186,7 +187,7 @@ module Flydata
186
187
  end
187
188
  it "uses the specified port" do
188
189
  expect(Open3).to receive(:popen3).with(
189
- 'mysqldump -h localhost -P 1234 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
190
+ 'mysqldump -h localhost -P 1234 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
190
191
  subject.send(:do_generate_table_ddl, default_data_entry)
191
192
  end
192
193
  end
@@ -219,7 +220,7 @@ module Flydata
219
220
  end
220
221
  it "call mysqldump with MYSQL_PW set with correct symbols" do
221
222
  expect(Open3).to receive(:popen3).with(
222
- 'mysqldump -h localhost -P 3306 -umasashi -p"welcome&!@^@#^" --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
223
+ 'mysqldump -h localhost -P 3306 -umasashi -pwelcome\\&\\!@\\^@\\#\\^ --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
223
224
  subject.send(:do_generate_table_ddl, default_data_entry)
224
225
  end
225
226
  end
@@ -251,13 +252,19 @@ module Flydata
251
252
  end
252
253
  describe '#convert_to_flydata_values' do
253
254
  subject { subject_object.send(:convert_to_flydata_values, mysql_table, values) }
254
- let(:values) { [4, 'John', nil, '0xC0448200', nil, nil] }
255
+ let(:values) { [4, 'John', nil, col4_value, nil, nil] }
255
256
 
256
- it 'calls convert_to_valydata_value for each value' do
257
- mysql_table.columns.values.each_with_index do |col_info, i|
258
- expect(FlydataCore::TableDef::MysqlTableDef).to receive(:convert_to_flydata_value).with(values[i], col_info[:format_type]).and_return(values[i])
257
+ context 'with binary column' do
258
+ let(:col4_type) { "binary" }
259
+ let(:col4_width) { 5 }
260
+ let(:truncated_binary) { "0xC04482" }
261
+ let(:col4_value) { "#{truncated_binary}0000" }
262
+
263
+ it 'truncates trailing "0" if the type is binary' do
264
+ expected_values = values.dup
265
+ expected_values[3] = truncated_binary
266
+ is_expected.to eq expected_values
259
267
  end
260
- is_expected.to eq values
261
268
  end
262
269
  end
263
270
  end
@@ -812,6 +812,56 @@ EOT
812
812
  end
813
813
  end
814
814
 
815
+ context 'with escape chars' do
816
+ let(:target_line) { "INSERT INTO `test_table` VALUES ('#{value}');" }
817
+
818
+ context 'with a single escape char' do
819
+ let(:value) { %q|\\0| }
820
+ #let(:expected_value) { %Q|\0| }
821
+ #it 'returns correct chars' do
822
+ # expect(subject).to eq([[expected_value]])
823
+ #end
824
+ it 'returns an error (for now)' do
825
+ expect{subject}.to raise_error(ArgumentError)
826
+ end
827
+ end
828
+ context 'with an escape char followed by a regular char' do
829
+ let(:value) { %q|\\'m| }
830
+ let(:expected_value) { %Q|'m| }
831
+ it 'returns correct chars' do
832
+ expect(subject).to eq([[expected_value]])
833
+ end
834
+ end
835
+ context 'with two consecutive escape chars' do
836
+ let(:value) { %Q|\\"\b| }
837
+ let(:expected_value) { %Q|"\b| }
838
+ it 'returns correct chars' do
839
+ expect(subject).to eq([[expected_value]])
840
+ end
841
+ end
842
+ context 'with a escape char following a regular char' do
843
+ let(:value) { %q|a\\n| }
844
+ let(:expected_value) { %Q|a\n| }
845
+ it 'returns correct chars' do
846
+ expect(subject).to eq([[expected_value]])
847
+ end
848
+ end
849
+ context 'with various escape chars 1' do
850
+ let(:value) { %Q|\\r\t\\Z| }
851
+ let(:expected_value) { %Q|\r\t\x1a| }
852
+ it 'returns correct chars' do
853
+ expect(subject).to eq([[expected_value]])
854
+ end
855
+ end
856
+ context 'with various escape chars 2' do
857
+ let(:value) { %q|c\\\\\\\\%\\\\_| }
858
+ let(:expected_value) { %q|c\\\\%\\_| }
859
+ it 'returns correct chars' do
860
+ expect(subject).to eq([[expected_value]])
861
+ end
862
+ end
863
+ end
864
+
815
865
  context 'when data end with back slash' do
816
866
  let(:target_line) { "INSERT INTO `test_table` VALUES (1,'D:\\\\download\\\\','2014-04-15 13:49:14');" }
817
867
  it 'should escape back slash' do
@@ -819,6 +869,13 @@ EOT
819
869
  end
820
870
  end
821
871
 
872
+ context 'when a regular character is escaped' do
873
+ let(:target_line) { "INSERT INTO `test_table` VALUES (1,'\\a','2014-04-15 13:49:14');" }
874
+ it 'should leave the back slash as is' do
875
+ expect(subject).to eq([['1',"\\a",'2014-04-15 13:49:14']])
876
+ end
877
+ end
878
+
822
879
  context 'when comma is the first character of a string' do
823
880
  let(:target_line) { "INSERT INTO `test_table` VALUES (1,',9','2014-04-15 13:49:14');" }
824
881
  it 'should parse the string correctly' do
@@ -923,6 +980,53 @@ EOT
923
980
  expect(subject).to eq([['1', '0.00']])
924
981
  end
925
982
  end
983
+
984
+ context 'when a hex blob is given' do
985
+ let(:target_line) {"INSERT INTO `test_table` VALUES (#{hex_blob});"}
986
+ context 'uppercase hex' do
987
+ let(:hex_blob) { '0x5CFA' }
988
+ it 'extracts the same string' do
989
+ expect(subject).to eq([[hex_blob]])
990
+ end
991
+ end
992
+ context 'lowercase hex' do
993
+ let(:hex_blob) { '0x5cfa' }
994
+ it 'extracts the same string' do
995
+ expect(subject).to eq([[hex_blob]])
996
+ end
997
+ end
998
+ end
999
+
1000
+ context 'when a negative number is given' do
1001
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,-0.004,-293);"}
1002
+ it 'should preserve the minus sign' do
1003
+ expect(subject).to eq([['1', '-0.004', '-293']])
1004
+ end
1005
+ end
1006
+
1007
+ context 'when NULL is given' do
1008
+ context 'in the end' do
1009
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,NULL);"}
1010
+ it 'should preserve the minus sign' do
1011
+ expect(subject).to eq([['1', nil]])
1012
+ end
1013
+ end
1014
+ context 'in the middle' do
1015
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,NULL,'hi');"}
1016
+ it 'should preserve the minus sign' do
1017
+ expect(subject).to eq([['1', nil, 'hi']])
1018
+ end
1019
+ end
1020
+ end
1021
+ context 'with many columns' do
1022
+ let(:target_line) {<<EOS.chomp}
1023
+ INSERT INTO `type_without_width_info` VALUES (1098238,0x5C,0x01,0x6974206C6F6F6B73206C696B65206120746578742062757420697427732062696E617279,0,1,'c','2015-10-18','2015-10-18 23:12:04',4,192,292.22,92,293.83,222,292,NULL,'I\\'m a \\\\ very long text',NULL,29838,'sdaklfj;',2983,12,'\\',\\\\','04:58:21','2015-10-18 12:34:56',NULL,-1,'28',1),(-2938,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,-0.49,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,2);
1024
+ EOS
1025
+ it 'should preserve the minus sign' do
1026
+ expect(subject).to eq([['1098238','0x5C','0x01','0x6974206C6F6F6B73206C696B65206120746578742062757420697427732062696E617279','0','1','c','2015-10-18','2015-10-18 23:12:04','4','192','292.22','92','293.83','222','292',nil,%q|I'm a \\ very long text|,nil,'29838','sdaklfj;','2983','12','\',\\','04:58:21','2015-10-18 12:34:56',nil,'-1','28','1'],['-2938',nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,'-0.49',nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,'2']])
1027
+ end
1028
+ end
1029
+
926
1030
  end
927
1031
  end
928
1032
  end
@@ -120,7 +120,7 @@ module Flydata
120
120
  describe '#load_dump_pos' do
121
121
  let (:mysql_table) do
122
122
  Flydata::Parser::Mysql::MysqlTable.new(
123
- table_name, { 'id' => { format_type: 'int' }, 'value' => { format_type: 'text' } }
123
+ table_name, { 'id' => { column_name: 'id', format_type: 'int' }, 'value' => { column_name: 'value', format_type: 'text' } }
124
124
  )
125
125
  end
126
126
 
@@ -138,8 +138,8 @@ module Flydata
138
138
  })
139
139
  expect(mt.table_name).to eq(table_name)
140
140
  expect(mt.columns).to eq({
141
- 'id' => { format_type: 'int' },
142
- 'value' => { format_type: 'text' },
141
+ 'id' => { column_name: 'id', format_type: 'int' },
142
+ 'value' => { column_name: 'value', format_type: 'text' },
143
143
  })
144
144
  end
145
145
  end
data/test-suite.sh CHANGED
@@ -34,6 +34,7 @@ fi
34
34
  # Test flydata-agent library (flydata-agent/lib, flydata-agent/spec)
35
35
  test_main() {
36
36
  cd $BASE_DIR
37
+ bundle exec rake compile
37
38
  bundle install
38
39
  bundle exec rspec
39
40
  }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.12
4
+ version: 0.5.13
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2015-10-08 00:00:00.000000000 Z
15
+ date: 2015-10-26 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -444,13 +444,35 @@ dependencies:
444
444
  - - '>='
445
445
  - !ruby/object:Gem::Version
446
446
  version: '0'
447
+ - !ruby/object:Gem::Dependency
448
+ name: rake-compiler
449
+ requirement: !ruby/object:Gem::Requirement
450
+ requirements:
451
+ - - '>='
452
+ - !ruby/object:Gem::Version
453
+ version: 0.9.5
454
+ - - ~>
455
+ - !ruby/object:Gem::Version
456
+ version: '0.9'
457
+ type: :development
458
+ prerelease: false
459
+ version_requirements: !ruby/object:Gem::Requirement
460
+ requirements:
461
+ - - '>='
462
+ - !ruby/object:Gem::Version
463
+ version: 0.9.5
464
+ - - ~>
465
+ - !ruby/object:Gem::Version
466
+ version: '0.9'
447
467
  description: FlyData Agent
448
468
  email: sysadmin@flydata.com
449
469
  executables:
450
470
  - fdmysqldump
451
471
  - flydata
452
472
  - serverinfo
453
- extensions: []
473
+ extensions:
474
+ - ext/flydata/json/extconf.rb
475
+ - ext/flydata/parser/mysql/extconf.rb
454
476
  extra_rdoc_files: []
455
477
  files:
456
478
  - .rspec
@@ -467,6 +489,15 @@ files:
467
489
  - bin/flydata
468
490
  - bin/serverinfo
469
491
  - circle.yml
492
+ - ext/flydata/flydata.h
493
+ - ext/flydata/json/extconf.rb
494
+ - ext/flydata/json/json_ext.cpp
495
+ - ext/flydata/parser/mysql/.gitignore
496
+ - ext/flydata/parser/mysql/dump_parser_ext.cpp
497
+ - ext/flydata/parser/mysql/extconf.rb
498
+ - ext/flydata/parser/mysql/parser.txt
499
+ - ext/flydata/parser/mysql/sql_parser.cpp
500
+ - ext/flydata/parser/mysql/sql_parser.h
470
501
  - flydata-core/.gitignore
471
502
  - flydata-core/.rspec
472
503
  - flydata-core/.travis.yml
@@ -600,10 +631,13 @@ files:
600
631
  - lib/flydata/heroku.rb
601
632
  - lib/flydata/heroku/configuration_methods.rb
602
633
  - lib/flydata/heroku/instance_methods.rb
634
+ - lib/flydata/json.rb
635
+ - lib/flydata/json/.gitignore
603
636
  - lib/flydata/log_monitor.rb
604
637
  - lib/flydata/mysql/binlog_position.rb
605
638
  - lib/flydata/mysql/table_ddl.rb
606
639
  - lib/flydata/output/forwarder.rb
640
+ - lib/flydata/parser/mysql/.gitignore
607
641
  - lib/flydata/parser/mysql/dump_parser.rb
608
642
  - lib/flydata/parser/mysql/mysql_alter_table.treetop
609
643
  - lib/flydata/parser_provider.rb
@@ -692,7 +726,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
692
726
  version: '0'
693
727
  requirements: []
694
728
  rubyforge_project:
695
- rubygems_version: 2.4.6
729
+ rubygems_version: 2.4.3
696
730
  signing_key:
697
731
  specification_version: 4
698
732
  summary: FlyData Agent