flydata 0.5.12 → 0.5.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -0
- data/Gemfile.lock +3 -0
- data/Rakefile +12 -0
- data/VERSION +1 -1
- data/ext/flydata/flydata.h +13 -0
- data/ext/flydata/json/extconf.rb +3 -0
- data/ext/flydata/json/json_ext.cpp +137 -0
- data/ext/flydata/parser/mysql/.gitignore +1 -0
- data/ext/flydata/parser/mysql/dump_parser_ext.cpp +106 -0
- data/ext/flydata/parser/mysql/extconf.rb +3 -0
- data/ext/flydata/parser/mysql/parser.txt +121 -0
- data/ext/flydata/parser/mysql/sql_parser.cpp +414 -0
- data/ext/flydata/parser/mysql/sql_parser.h +15 -0
- data/flydata-core/lib/flydata-core/mysql/command_generator.rb +20 -9
- data/flydata-core/spec/mysql/command_generator_spec.rb +17 -17
- data/flydata.gemspec +0 -0
- data/lib/flydata/command/sync.rb +78 -90
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +4 -1
- data/lib/flydata/json.rb +1 -0
- data/lib/flydata/json/.gitignore +1 -0
- data/lib/flydata/parser/mysql/.gitignore +1 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +21 -5
- data/spec/flydata/command/sync_spec.rb +20 -13
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +104 -0
- data/spec/flydata/sync_file_manager_spec.rb +3 -3
- data/test-suite.sh +1 -0
- metadata +38 -4
@@ -40,13 +40,14 @@ module Flydata
|
|
40
40
|
"data_servers"=>"localhost:9905" }
|
41
41
|
}
|
42
42
|
end
|
43
|
+
let(:col4_type) { "binary" }
|
44
|
+
let(:col4_width) { 34 }
|
45
|
+
let(:col4_type_str) { "#{col4_type}(#{col4_width})" }
|
43
46
|
let(:mysql_table_columns) {
|
44
|
-
{"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>
|
47
|
+
{"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>col4_type_str, :format_type=>col4_type, :format_size=>col4_width, :default=>nil}, "bin2"=>{:column_name=>"bin2", :format_type_str=>"blob", :format_type=>"blob"}, "varbin"=>{:column_name=>"varbin", :format_type_str=>"varchar(34)", :format_type=>"varchar", :format_size=>34, :default=>nil}}
|
45
48
|
}
|
46
49
|
let(:mysql_table) do
|
47
|
-
|
48
|
-
allow(mt).to receive(:columns).and_return(mysql_table_columns)
|
49
|
-
mt
|
50
|
+
Flydata::Parser::Mysql::MysqlTable.new("test_table", mysql_table_columns, ['id'])
|
50
51
|
end
|
51
52
|
|
52
53
|
after :each do
|
@@ -154,7 +155,7 @@ module Flydata
|
|
154
155
|
context 'with full options' do
|
155
156
|
it 'issues mysqldump command with expected parameters' do
|
156
157
|
expect(Open3).to receive(:popen3).with(
|
157
|
-
'mysqldump -h localhost -P 3306 -umasashi -
|
158
|
+
'mysqldump -h localhost -P 3306 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
|
158
159
|
subject.send(:do_generate_table_ddl, default_data_entry)
|
159
160
|
end
|
160
161
|
end
|
@@ -176,7 +177,7 @@ module Flydata
|
|
176
177
|
end
|
177
178
|
it "uses the default port" do
|
178
179
|
expect(Open3).to receive(:popen3).with(
|
179
|
-
'mysqldump -h localhost -umasashi -
|
180
|
+
'mysqldump -h localhost -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
|
180
181
|
subject.send(:do_generate_table_ddl, default_data_entry)
|
181
182
|
end
|
182
183
|
end
|
@@ -186,7 +187,7 @@ module Flydata
|
|
186
187
|
end
|
187
188
|
it "uses the specified port" do
|
188
189
|
expect(Open3).to receive(:popen3).with(
|
189
|
-
'mysqldump -h localhost -P 1234 -umasashi -
|
190
|
+
'mysqldump -h localhost -P 1234 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
|
190
191
|
subject.send(:do_generate_table_ddl, default_data_entry)
|
191
192
|
end
|
192
193
|
end
|
@@ -219,7 +220,7 @@ module Flydata
|
|
219
220
|
end
|
220
221
|
it "call mysqldump with MYSQL_PW set with correct symbols" do
|
221
222
|
expect(Open3).to receive(:popen3).with(
|
222
|
-
'mysqldump -h localhost -P 3306 -umasashi -
|
223
|
+
'mysqldump -h localhost -P 3306 -umasashi -pwelcome\\&\\!@\\^@\\#\\^ --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
|
223
224
|
subject.send(:do_generate_table_ddl, default_data_entry)
|
224
225
|
end
|
225
226
|
end
|
@@ -251,13 +252,19 @@ module Flydata
|
|
251
252
|
end
|
252
253
|
describe '#convert_to_flydata_values' do
|
253
254
|
subject { subject_object.send(:convert_to_flydata_values, mysql_table, values) }
|
254
|
-
let(:values) { [4, 'John', nil,
|
255
|
+
let(:values) { [4, 'John', nil, col4_value, nil, nil] }
|
255
256
|
|
256
|
-
|
257
|
-
|
258
|
-
|
257
|
+
context 'with binary column' do
|
258
|
+
let(:col4_type) { "binary" }
|
259
|
+
let(:col4_width) { 5 }
|
260
|
+
let(:truncated_binary) { "0xC04482" }
|
261
|
+
let(:col4_value) { "#{truncated_binary}0000" }
|
262
|
+
|
263
|
+
it 'truncates trailing "0" if the type is binary' do
|
264
|
+
expected_values = values.dup
|
265
|
+
expected_values[3] = truncated_binary
|
266
|
+
is_expected.to eq expected_values
|
259
267
|
end
|
260
|
-
is_expected.to eq values
|
261
268
|
end
|
262
269
|
end
|
263
270
|
end
|
@@ -812,6 +812,56 @@ EOT
|
|
812
812
|
end
|
813
813
|
end
|
814
814
|
|
815
|
+
context 'with escape chars' do
|
816
|
+
let(:target_line) { "INSERT INTO `test_table` VALUES ('#{value}');" }
|
817
|
+
|
818
|
+
context 'with a single escape char' do
|
819
|
+
let(:value) { %q|\\0| }
|
820
|
+
#let(:expected_value) { %Q|\0| }
|
821
|
+
#it 'returns correct chars' do
|
822
|
+
# expect(subject).to eq([[expected_value]])
|
823
|
+
#end
|
824
|
+
it 'returns an error (for now)' do
|
825
|
+
expect{subject}.to raise_error(ArgumentError)
|
826
|
+
end
|
827
|
+
end
|
828
|
+
context 'with an escape char followed by a regular char' do
|
829
|
+
let(:value) { %q|\\'m| }
|
830
|
+
let(:expected_value) { %Q|'m| }
|
831
|
+
it 'returns correct chars' do
|
832
|
+
expect(subject).to eq([[expected_value]])
|
833
|
+
end
|
834
|
+
end
|
835
|
+
context 'with two consecutive escape chars' do
|
836
|
+
let(:value) { %Q|\\"\b| }
|
837
|
+
let(:expected_value) { %Q|"\b| }
|
838
|
+
it 'returns correct chars' do
|
839
|
+
expect(subject).to eq([[expected_value]])
|
840
|
+
end
|
841
|
+
end
|
842
|
+
context 'with a escape char following a regular char' do
|
843
|
+
let(:value) { %q|a\\n| }
|
844
|
+
let(:expected_value) { %Q|a\n| }
|
845
|
+
it 'returns correct chars' do
|
846
|
+
expect(subject).to eq([[expected_value]])
|
847
|
+
end
|
848
|
+
end
|
849
|
+
context 'with various escape chars 1' do
|
850
|
+
let(:value) { %Q|\\r\t\\Z| }
|
851
|
+
let(:expected_value) { %Q|\r\t\x1a| }
|
852
|
+
it 'returns correct chars' do
|
853
|
+
expect(subject).to eq([[expected_value]])
|
854
|
+
end
|
855
|
+
end
|
856
|
+
context 'with various escape chars 2' do
|
857
|
+
let(:value) { %q|c\\\\\\\\%\\\\_| }
|
858
|
+
let(:expected_value) { %q|c\\\\%\\_| }
|
859
|
+
it 'returns correct chars' do
|
860
|
+
expect(subject).to eq([[expected_value]])
|
861
|
+
end
|
862
|
+
end
|
863
|
+
end
|
864
|
+
|
815
865
|
context 'when data end with back slash' do
|
816
866
|
let(:target_line) { "INSERT INTO `test_table` VALUES (1,'D:\\\\download\\\\','2014-04-15 13:49:14');" }
|
817
867
|
it 'should escape back slash' do
|
@@ -819,6 +869,13 @@ EOT
|
|
819
869
|
end
|
820
870
|
end
|
821
871
|
|
872
|
+
context 'when a regular character is escaped' do
|
873
|
+
let(:target_line) { "INSERT INTO `test_table` VALUES (1,'\\a','2014-04-15 13:49:14');" }
|
874
|
+
it 'should leave the back slash as is' do
|
875
|
+
expect(subject).to eq([['1',"\\a",'2014-04-15 13:49:14']])
|
876
|
+
end
|
877
|
+
end
|
878
|
+
|
822
879
|
context 'when comma is the first character of a string' do
|
823
880
|
let(:target_line) { "INSERT INTO `test_table` VALUES (1,',9','2014-04-15 13:49:14');" }
|
824
881
|
it 'should parse the string correctly' do
|
@@ -923,6 +980,53 @@ EOT
|
|
923
980
|
expect(subject).to eq([['1', '0.00']])
|
924
981
|
end
|
925
982
|
end
|
983
|
+
|
984
|
+
context 'when a hex blob is given' do
|
985
|
+
let(:target_line) {"INSERT INTO `test_table` VALUES (#{hex_blob});"}
|
986
|
+
context 'uppercase hex' do
|
987
|
+
let(:hex_blob) { '0x5CFA' }
|
988
|
+
it 'extracts the same string' do
|
989
|
+
expect(subject).to eq([[hex_blob]])
|
990
|
+
end
|
991
|
+
end
|
992
|
+
context 'lowercase hex' do
|
993
|
+
let(:hex_blob) { '0x5cfa' }
|
994
|
+
it 'extracts the same string' do
|
995
|
+
expect(subject).to eq([[hex_blob]])
|
996
|
+
end
|
997
|
+
end
|
998
|
+
end
|
999
|
+
|
1000
|
+
context 'when a negative number is given' do
|
1001
|
+
let(:target_line) {"INSERT INTO `test_table` VALUES (1,-0.004,-293);"}
|
1002
|
+
it 'should preserve the minus sign' do
|
1003
|
+
expect(subject).to eq([['1', '-0.004', '-293']])
|
1004
|
+
end
|
1005
|
+
end
|
1006
|
+
|
1007
|
+
context 'when NULL is given' do
|
1008
|
+
context 'in the end' do
|
1009
|
+
let(:target_line) {"INSERT INTO `test_table` VALUES (1,NULL);"}
|
1010
|
+
it 'should preserve the minus sign' do
|
1011
|
+
expect(subject).to eq([['1', nil]])
|
1012
|
+
end
|
1013
|
+
end
|
1014
|
+
context 'in the middle' do
|
1015
|
+
let(:target_line) {"INSERT INTO `test_table` VALUES (1,NULL,'hi');"}
|
1016
|
+
it 'should preserve the minus sign' do
|
1017
|
+
expect(subject).to eq([['1', nil, 'hi']])
|
1018
|
+
end
|
1019
|
+
end
|
1020
|
+
end
|
1021
|
+
context 'with many columns' do
|
1022
|
+
let(:target_line) {<<EOS.chomp}
|
1023
|
+
INSERT INTO `type_without_width_info` VALUES (1098238,0x5C,0x01,0x6974206C6F6F6B73206C696B65206120746578742062757420697427732062696E617279,0,1,'c','2015-10-18','2015-10-18 23:12:04',4,192,292.22,92,293.83,222,292,NULL,'I\\'m a \\\\ very long text',NULL,29838,'sdaklfj;',2983,12,'\\',\\\\','04:58:21','2015-10-18 12:34:56',NULL,-1,'28',1),(-2938,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,-0.49,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,2);
|
1024
|
+
EOS
|
1025
|
+
it 'should preserve the minus sign' do
|
1026
|
+
expect(subject).to eq([['1098238','0x5C','0x01','0x6974206C6F6F6B73206C696B65206120746578742062757420697427732062696E617279','0','1','c','2015-10-18','2015-10-18 23:12:04','4','192','292.22','92','293.83','222','292',nil,%q|I'm a \\ very long text|,nil,'29838','sdaklfj;','2983','12','\',\\','04:58:21','2015-10-18 12:34:56',nil,'-1','28','1'],['-2938',nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,'-0.49',nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,nil,'2']])
|
1027
|
+
end
|
1028
|
+
end
|
1029
|
+
|
926
1030
|
end
|
927
1031
|
end
|
928
1032
|
end
|
@@ -120,7 +120,7 @@ module Flydata
|
|
120
120
|
describe '#load_dump_pos' do
|
121
121
|
let (:mysql_table) do
|
122
122
|
Flydata::Parser::Mysql::MysqlTable.new(
|
123
|
-
table_name, { 'id' => { format_type: 'int' }, 'value' => { format_type: 'text' } }
|
123
|
+
table_name, { 'id' => { column_name: 'id', format_type: 'int' }, 'value' => { column_name: 'value', format_type: 'text' } }
|
124
124
|
)
|
125
125
|
end
|
126
126
|
|
@@ -138,8 +138,8 @@ module Flydata
|
|
138
138
|
})
|
139
139
|
expect(mt.table_name).to eq(table_name)
|
140
140
|
expect(mt.columns).to eq({
|
141
|
-
'id' => { format_type: 'int' },
|
142
|
-
'value' => { format_type: 'text' },
|
141
|
+
'id' => { column_name: 'id', format_type: 'int' },
|
142
|
+
'value' => { column_name: 'value', format_type: 'text' },
|
143
143
|
})
|
144
144
|
end
|
145
145
|
end
|
data/test-suite.sh
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2015-10-
|
15
|
+
date: 2015-10-26 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -444,13 +444,35 @@ dependencies:
|
|
444
444
|
- - '>='
|
445
445
|
- !ruby/object:Gem::Version
|
446
446
|
version: '0'
|
447
|
+
- !ruby/object:Gem::Dependency
|
448
|
+
name: rake-compiler
|
449
|
+
requirement: !ruby/object:Gem::Requirement
|
450
|
+
requirements:
|
451
|
+
- - '>='
|
452
|
+
- !ruby/object:Gem::Version
|
453
|
+
version: 0.9.5
|
454
|
+
- - ~>
|
455
|
+
- !ruby/object:Gem::Version
|
456
|
+
version: '0.9'
|
457
|
+
type: :development
|
458
|
+
prerelease: false
|
459
|
+
version_requirements: !ruby/object:Gem::Requirement
|
460
|
+
requirements:
|
461
|
+
- - '>='
|
462
|
+
- !ruby/object:Gem::Version
|
463
|
+
version: 0.9.5
|
464
|
+
- - ~>
|
465
|
+
- !ruby/object:Gem::Version
|
466
|
+
version: '0.9'
|
447
467
|
description: FlyData Agent
|
448
468
|
email: sysadmin@flydata.com
|
449
469
|
executables:
|
450
470
|
- fdmysqldump
|
451
471
|
- flydata
|
452
472
|
- serverinfo
|
453
|
-
extensions:
|
473
|
+
extensions:
|
474
|
+
- ext/flydata/json/extconf.rb
|
475
|
+
- ext/flydata/parser/mysql/extconf.rb
|
454
476
|
extra_rdoc_files: []
|
455
477
|
files:
|
456
478
|
- .rspec
|
@@ -467,6 +489,15 @@ files:
|
|
467
489
|
- bin/flydata
|
468
490
|
- bin/serverinfo
|
469
491
|
- circle.yml
|
492
|
+
- ext/flydata/flydata.h
|
493
|
+
- ext/flydata/json/extconf.rb
|
494
|
+
- ext/flydata/json/json_ext.cpp
|
495
|
+
- ext/flydata/parser/mysql/.gitignore
|
496
|
+
- ext/flydata/parser/mysql/dump_parser_ext.cpp
|
497
|
+
- ext/flydata/parser/mysql/extconf.rb
|
498
|
+
- ext/flydata/parser/mysql/parser.txt
|
499
|
+
- ext/flydata/parser/mysql/sql_parser.cpp
|
500
|
+
- ext/flydata/parser/mysql/sql_parser.h
|
470
501
|
- flydata-core/.gitignore
|
471
502
|
- flydata-core/.rspec
|
472
503
|
- flydata-core/.travis.yml
|
@@ -600,10 +631,13 @@ files:
|
|
600
631
|
- lib/flydata/heroku.rb
|
601
632
|
- lib/flydata/heroku/configuration_methods.rb
|
602
633
|
- lib/flydata/heroku/instance_methods.rb
|
634
|
+
- lib/flydata/json.rb
|
635
|
+
- lib/flydata/json/.gitignore
|
603
636
|
- lib/flydata/log_monitor.rb
|
604
637
|
- lib/flydata/mysql/binlog_position.rb
|
605
638
|
- lib/flydata/mysql/table_ddl.rb
|
606
639
|
- lib/flydata/output/forwarder.rb
|
640
|
+
- lib/flydata/parser/mysql/.gitignore
|
607
641
|
- lib/flydata/parser/mysql/dump_parser.rb
|
608
642
|
- lib/flydata/parser/mysql/mysql_alter_table.treetop
|
609
643
|
- lib/flydata/parser_provider.rb
|
@@ -692,7 +726,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
692
726
|
version: '0'
|
693
727
|
requirements: []
|
694
728
|
rubyforge_project:
|
695
|
-
rubygems_version: 2.4.
|
729
|
+
rubygems_version: 2.4.3
|
696
730
|
signing_key:
|
697
731
|
specification_version: 4
|
698
732
|
summary: FlyData Agent
|