flydata 0.1.8 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,11 +22,11 @@ class RedshiftTableDef
22
22
  'int4 unsigned' => {type: 'int8', unsigned: true},
23
23
  'int8' => {type: 'int8'},
24
24
  'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true},
25
- 'numeric' => {type: 'numeric', use_params: true},
26
- 'numeric unsigned' => {type: 'numeric', use_params: true},
25
+ 'numeric' => {type: 'numeric', use_params: true, max_size: [38,37]},
26
+ 'numeric unsigned' => {type: 'numeric', use_params: true, max_size: [38,37]},
27
27
  'text' => {type: 'varchar(max)'},
28
28
  'time' => {type: 'timestamp'},
29
- 'varbinary' => {type: 'varchar', use_params: true},
29
+ 'varbinary' => {type: 'varchar', use_params: true, max_size: 65535},
30
30
  'varchar' => {type: 'varchar', use_params: true, max_size: 65535},
31
31
  }
32
32
  def self.from_flydata_tabledef(flydata_tabledef, options = {})
@@ -85,9 +85,7 @@ EOS
85
85
  raise "Unsupported type '#{column[:type]}'" if type_info.nil?
86
86
 
87
87
  rs_type = if type_info[:use_params] && params && !params.nil?
88
- if type_info[:max_size] && /\d+/.match(params) && params.to_i > type_info[:max_size]
89
- params = type_info[:max_size]
90
- end
88
+ params = check_and_replace_max(params, Array(type_info[:max_size])) if type_info[:max_size]
91
89
  type_info[:type] + "(#{params})"
92
90
  else
93
91
  type_info[:type]
@@ -150,6 +148,15 @@ EOS
150
148
  def self.escape(text)
151
149
  text.gsub("'", "\\\\'")
152
150
  end
151
+
152
+ def self.check_and_replace_max(params, max_size_a)
153
+ final_params = []
154
+ params.split(",").each_with_index do |param, i|
155
+ final_params << (/\d+/.match(param) && max_size_a[i] && param.to_i > max_size_a[i].to_i ?
156
+ max_size_a[i] : param)
157
+ end
158
+ final_params.join(",")
159
+ end
153
160
  end
154
161
 
155
162
  end
@@ -0,0 +1,172 @@
1
+ require 'spec_helper'
2
+
3
+ module Flydata
4
+ describe Cli do
5
+ subject { Cli.new(args) }
6
+ describe '#run' do
7
+ let(:command_obj) { double('command_obj') }
8
+ let(:args) { options.unshift command }
9
+ let(:empty_slop) { double('empty_slop') }
10
+ let (:sub_command) { "subcomm" }
11
+ context 'target commands supporting no options' do
12
+ let(:main_command) { "maincomm" }
13
+ let(:target_class) { module Command; class Maincomm; end; end; Command::Maincomm }
14
+ context 'without subcommand' do
15
+ let(:command) { main_command }
16
+ context 'without options' do
17
+ let(:options) { [] }
18
+ it 'calls #run of the target class object' do
19
+ expect(target_class).to receive(:new){|a|
20
+ expect(a.to_hash).to be_empty
21
+ }.and_return(command_obj)
22
+ expect(command_obj).to receive(:run)
23
+
24
+ subject.run
25
+ end
26
+ end
27
+ context 'with options' do
28
+ let(:options) { %w(-a --host-name localhost) }
29
+ it 'returns with an error' do
30
+ expect(subject).to receive(:puts).with('! Unknown options -a, --host-name').once
31
+ allow(subject).to receive(:puts)
32
+
33
+ subject.run
34
+ end
35
+ end
36
+ context 'with arguments' do
37
+ let(:options) { %w(a b c ) }
38
+ it 'calls #run of the target class instance with arguments' do
39
+ expect(target_class).to receive(:new){|a|
40
+ expect(a.to_hash).to be_empty
41
+ }.and_return(command_obj)
42
+ expect(command_obj).to receive(:run).with(*options)
43
+
44
+ subject.run
45
+ end
46
+ end
47
+ end
48
+ context 'with subcommand' do
49
+ let (:command) { "#{main_command}:#{sub_command}" }
50
+ context 'without options' do
51
+ let(:options) { [] }
52
+ it 'calls subcommand method of the target class object' do
53
+ expect(target_class).to receive(:new){|a|
54
+ expect(a.to_hash).to be_empty
55
+ }.and_return(command_obj)
56
+
57
+ expect(command_obj).to receive(sub_command.to_sym)
58
+
59
+ subject.run
60
+ end
61
+ end
62
+ context 'with optionss' do
63
+ let(:options) { %w(-a --host-name localhost) }
64
+ it 'returns an error' do
65
+ expect(subject).to receive(:puts).with('! Unknown options -a, --host-name').once
66
+ allow(subject).to receive(:puts)
67
+
68
+ subject.run
69
+ end
70
+ end
71
+ end
72
+ end
73
+ context 'target commands supporting options' do
74
+ let(:main_command) { "maincommop" }
75
+ let(:target_class) do
76
+ module Command
77
+ class Maincommop < Base
78
+ # Options for the main command
79
+ def self.slop
80
+ Slop.new(strict: true) do
81
+ on 'a', 'short option'
82
+ on 'host-name=', 'long option taking an argument'
83
+ end
84
+ end
85
+
86
+ # Options for the sub command 'subcomm'
87
+ def self.slop_subcomm
88
+ Slop.new(strict: true) do
89
+ on 'b', 'short option'
90
+ on 'port=', 'Port', as: Integer
91
+ end
92
+ end
93
+ end
94
+ end
95
+ Command::Maincommop
96
+ end
97
+ context 'without subcommand' do
98
+ let(:command) { main_command }
99
+ context 'without options' do
100
+ let(:options) { [] }
101
+ it 'calls #run of the target class object' do
102
+ expect(target_class).to receive(:new) { |a|
103
+ expect(a.a?).to be false
104
+ expect(a[:"host-name"]).to be_nil
105
+ }.and_return(command_obj)
106
+
107
+ expect(command_obj).to receive(:run)
108
+
109
+ subject.run
110
+ end
111
+ end
112
+ context 'with options' do
113
+ let(:options) { %w(-a --host-name localhost) }
114
+ it 'calls #run of the target class object' do
115
+ expect(target_class).to receive(:new) { |a|
116
+ expect(a.a?).to be true
117
+ expect(a[:'host-name']).to eq "localhost"
118
+ }.and_return(command_obj)
119
+
120
+ expect(command_obj).to receive(:run)
121
+
122
+ subject.run
123
+ end
124
+ end
125
+ context 'with options and arguments' do
126
+ let(:arguments) { %w(foo bar bal) }
127
+ let(:options) { %w(foo -a bar --host-name localhost bal) }
128
+ it 'calls #run of the target class object' do
129
+ expect(target_class).to receive(:new) { |a|
130
+ expect(a.a?).to be true
131
+ expect(a[:'host-name']).to eq "localhost"
132
+ }.and_return(command_obj)
133
+
134
+ expect(command_obj).to receive(:run).with(*arguments)
135
+
136
+ subject.run
137
+ end
138
+ end
139
+ end
140
+ context 'with subcommand' do
141
+ let (:command) { "#{main_command}:#{sub_command}" }
142
+ context 'without options' do
143
+ let(:options) { [] }
144
+ it 'calls the subcommand of the target class object' do
145
+ expect(target_class).to receive(:new) { |a|
146
+ expect(a.b?).to be false
147
+ expect(a[:port]).to be_nil
148
+ }.and_return(command_obj)
149
+
150
+ expect(command_obj).to receive(sub_command.to_sym)
151
+
152
+ subject.run
153
+ end
154
+ end
155
+ context 'with options' do
156
+ let(:options) { %w(-b --port 5439) }
157
+ it 'calls the subcommand of the target class object' do
158
+ expect(target_class).to receive(:new) { |a|
159
+ expect(a.b?).to be true
160
+ expect(a[:port]).to eq 5439
161
+ }.and_return(command_obj)
162
+
163
+ expect(command_obj).to receive(sub_command.to_sym)
164
+
165
+ subject.run
166
+ end
167
+ end
168
+ end
169
+ end
170
+ end
171
+ end
172
+ end
@@ -1,9 +1,47 @@
1
1
  require 'spec_helper'
2
+ require 'slop'
2
3
 
3
4
  module Flydata
4
5
  module Command
5
6
  describe Sender do
6
- pending '#start'
7
+ let(:opts) do
8
+ slop = Sender.slop_start
9
+ slop.parse(args)
10
+ slop
11
+ end
12
+ subject { Sender.new(opts) }
13
+ describe '#start' do
14
+ before(:each) do
15
+ allow(subject).to receive(:process_exist?).and_return(false)
16
+ expect(subject).to receive(:wait_until_server_ready)
17
+ expect(subject).to receive(:wait_until_client_ready)
18
+ allow(Kernel).to receive(:sleep)
19
+ end
20
+ context "as daemon" do
21
+ let(:args) { [] }
22
+ it "starts fluend with daemon option" do
23
+ expect(Kernel).to receive(:system).with( Regexp.new(
24
+ "fluentd -d .+/flydata.pid -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
25
+ subject.start(false)
26
+ end
27
+ end
28
+ context "as regular process" do
29
+ let(:args) { ["-n"] }
30
+ it "starts fluentd with no daemon option" do
31
+ expect(Kernel).to receive(:system).with(Regexp.new(
32
+ "fluentd -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
33
+ subject.start(false)
34
+ end
35
+ end
36
+ context "as regular process with long option" do
37
+ let(:args) { ["--no-daemon"] }
38
+ it "starts fluentd with no daemon option" do
39
+ expect(Kernel).to receive(:system).with(Regexp.new(
40
+ "fluentd -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
41
+ subject.start(false)
42
+ end
43
+ end
44
+ end
7
45
  pending '#stop'
8
46
  pending '#restart'
9
47
  end
@@ -1,6 +1,5 @@
1
1
  # coding: utf-8
2
2
  require 'spec_helper'
3
- require 'socket'
4
3
 
5
4
  module Flydata
6
5
  module Command
@@ -49,8 +48,8 @@ module Flydata
49
48
  subject { Sync.new }
50
49
  context 'with full options' do
51
50
  it 'issues mysqldump command with expected parameters' do
52
- expect(IO).to receive(:popen).with(
53
- 'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1 table2', 'r').and_call_original
51
+ expect(Open3).to receive(:popen3).with(
52
+ 'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1 table2').and_call_original
54
53
  subject.send(:do_generate_table_ddl, default_data_entry)
55
54
  end
56
55
  end
@@ -69,8 +68,8 @@ module Flydata
69
68
  default_data_entry['mysql_data_entry_preference'].delete('port')
70
69
  end
71
70
  it "uses the default port" do
72
- expect(IO).to receive(:popen).with(
73
- 'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1 table2', 'r').and_call_original
71
+ expect(Open3).to receive(:popen3).with(
72
+ 'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1 table2').and_call_original
74
73
  subject.send(:do_generate_table_ddl, default_data_entry)
75
74
  end
76
75
  end
@@ -79,8 +78,8 @@ module Flydata
79
78
  default_data_entry['mysql_data_entry_preference']['port'] = 1234
80
79
  end
81
80
  it "uses the specified port" do
82
- expect(IO).to receive(:popen).with(
83
- 'mysqldump --protocol=tcp -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1 table2', 'r').and_call_original
81
+ expect(Open3).to receive(:popen3).with(
82
+ 'mysqldump --protocol=tcp -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1 table2').and_call_original
84
83
  subject.send(:do_generate_table_ddl, default_data_entry)
85
84
  end
86
85
  end
@@ -99,8 +98,8 @@ module Flydata
99
98
  default_data_entry['mysql_data_entry_preference'].delete('password')
100
99
  end
101
100
  it "call mysqldump without -p option" do
102
- expect(IO).to receive(:popen).with(
103
- 'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi sync_test table1 table2', 'r').and_call_original
101
+ expect(Open3).to receive(:popen3).with(
102
+ 'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi sync_test table1 table2').and_call_original
104
103
  subject.send(:do_generate_table_ddl, default_data_entry)
105
104
  end
106
105
  end
@@ -1115,35 +1114,51 @@ EOT
1115
1114
  end
1116
1115
 
1117
1116
  context 'when data includes carriage return' do
1118
- let(:target_line) { "INSERT INTO `test_table` VALUES (1,'abcd\\refgh','2014-04-15 13:49:14');" }
1117
+ let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'abcd\refgh','2014-04-15 13:49:14');| }
1119
1118
  it 'should escape return carriage' do
1120
1119
  expect(subject).to eq([['1',"abcd\refgh",'2014-04-15 13:49:14']])
1121
1120
  end
1122
1121
  end
1123
1122
 
1124
1123
  context 'when data includes new line' do
1125
- let(:target_line) { "INSERT INTO `test_table` VALUES (1,'abcd\\nefgh','2014-04-15 13:49:14');" }
1124
+ let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'abcd\nefgh','2014-04-15 13:49:14');| }
1126
1125
  it 'should escape new line' do
1127
1126
  expect(subject).to eq([['1',"abcd\nefgh",'2014-04-15 13:49:14']])
1128
1127
  end
1129
1128
  end
1130
1129
 
1131
1130
  context 'when data includes escaped single quotation' do
1132
- let(:target_line) { "INSERT INTO `test_table` VALUES (1,'abcd\\',\\'efgh','2014-04-15 13:49:14');" }
1131
+ let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'abcd\',\'efgh','2014-04-15 13:49:14');| }
1133
1132
  it 'should escape single quotation' do
1134
1133
  expect(subject).to eq([['1',"abcd','efgh",'2014-04-15 13:49:14']])
1135
1134
  end
1136
1135
  end
1137
1136
 
1138
1137
  context 'when data includes back slash' do
1139
- let(:target_line) { "INSERT INTO `test_table` VALUES (1,'abcd\\\\efgh','2014-04-15 13:49:14');" }
1138
+ let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'abcd\\efgh','2014-04-15 13:49:14');| }
1140
1139
  it 'should escape back slash' do
1141
1140
  expect(subject).to eq([['1',"abcd\\efgh",'2014-04-15 13:49:14']])
1142
1141
  end
1143
1142
  end
1144
1143
 
1144
+ context 'when data includes back slash with double quote' do
1145
+ # \"\'\' value on insert query shoulde be "''
1146
+ let(:target_line) { %q|INSERT INTO `tast_table` VALUES (1,'\"\'\'','2014-04-15 13:49:14');| }
1147
+ it 'should escape back slash' do
1148
+ expect(subject).to eq([['1',%q|"''| ,'2014-04-15 13:49:14']])
1149
+ end
1150
+ end
1151
+
1152
+ context 'when data includes back slash with double quote another case' do
1153
+ # \"\"\"\"\"''\"'' value on insert query shoulde be """""''"''
1154
+ let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'\"\"\"\"\"\'\'\"\'\'','2014-04-15 13:49:14');| }
1155
+ it 'should escape back slash' do
1156
+ expect(subject).to eq([['1',%q|"""""''"''|,'2014-04-15 13:49:14']])
1157
+ end
1158
+ end
1159
+
1145
1160
  context 'when data includes mixed escaped characters' do
1146
- let(:target_line) { "INSERT INTO `test_table` VALUES (1,'ab\\rcd\\\\e\\nf\\',\\'gh','2014-04-15 13:49:14');" }
1161
+ let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'ab\rcd\\e\nf\',\'gh','2014-04-15 13:49:14');| }
1147
1162
  it 'should escape all' do
1148
1163
  expect(subject).to eq([['1',"ab\rcd\\e\nf','gh",'2014-04-15 13:49:14']])
1149
1164
  end
@@ -1211,6 +1226,62 @@ EOT
1211
1226
  expect{subject}.to raise_error
1212
1227
  end
1213
1228
  end
1229
+
1230
+ context 'when an integer that has leading zeros is given' do
1231
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,00013);"}
1232
+ it 'should remove the leading zeros' do
1233
+ expect(subject).to eq([['1', '13']])
1234
+ end
1235
+ end
1236
+
1237
+ context 'when a decimal that has leading zeros is given' do
1238
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,00013.40);"}
1239
+ it 'should remove the leading zeros' do
1240
+ expect(subject).to eq([['1', '13.40']])
1241
+ end
1242
+ end
1243
+
1244
+ context 'when a timestamp that has leading zeros is given' do
1245
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,'0004-04-15 13:49:14');"}
1246
+ it 'should not remove the leading zeros' do
1247
+ expect(subject).to eq([['1', '0004-04-15 13:49:14']])
1248
+ end
1249
+ end
1250
+
1251
+ context 'when a string that has leading zeros is given' do
1252
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,'00000aa');"}
1253
+ it 'should not remove the leading zeros' do
1254
+ expect(subject).to eq([['1', '00000aa']])
1255
+ end
1256
+ end
1257
+
1258
+ context 'when a string that has leading zeros, numbers and a comma in between is given' do
1259
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,'0003,00033');"}
1260
+ it 'should not remove the leading zeros' do
1261
+ expect(subject).to eq([['1', '0003,00033']])
1262
+ end
1263
+ end
1264
+
1265
+ context 'when a string that has leading zeros, has only numbers is given' do
1266
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,'00033');"}
1267
+ it 'should not remove the leading zeros' do
1268
+ expect(subject).to eq([['1', '00033']])
1269
+ end
1270
+ end
1271
+
1272
+ context 'when 0 is given' do
1273
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,0);"}
1274
+ it 'should not remove the leading zeros' do
1275
+ expect(subject).to eq([['1', '0']])
1276
+ end
1277
+ end
1278
+
1279
+ context 'when 0.00 is given' do
1280
+ let(:target_line) {"INSERT INTO `test_table` VALUES (1,0.00);"}
1281
+ it 'should not remove the leading zeros' do
1282
+ expect(subject).to eq([['1', '0.00']])
1283
+ end
1284
+ end
1214
1285
  end
1215
1286
  end
1216
1287
  end
@@ -32,7 +32,7 @@ describe MysqlTableDef do
32
32
  [
33
33
  {:name=>"id", :type=>"int8(20)", :auto_increment=>true, :not_null=>true, :primary_key=>true},
34
34
  {:name=>"col_binary", :type=>"binary(100)", :default=>nil},
35
- {:name=>"col_blob", :type=>"varbinary"},
35
+ {:name=>"col_blob", :type=>"varbinary(65535)"},
36
36
  {:name=>"col_bool", :type=>"int1(1)", :default=>"0"},
37
37
  {:name=>"col_char", :type=>"varchar(18)", :default=>nil},
38
38
  {:name=>"col_date", :type=>"date", :default=>nil},
@@ -43,16 +43,16 @@ describe MysqlTableDef do
43
43
  {:name=>"col_float_4_2", :type=>"float4(4,2)", :default=>nil},
44
44
  {:name=>"col_int", :type=>"int4(11)", :default=>nil},
45
45
  {:name=>"col_int_6", :type=>"int4(6)", :default=>nil},
46
- {:name=>"col_longblob", :type=>"varbinary"},
46
+ {:name=>"col_longblob", :type=>"varbinary(4294967295)"},
47
47
  {:name=>"col_longtext", :type=>"text"},
48
- {:name=>"col_mediumblob", :type=>"varbinary"},
48
+ {:name=>"col_mediumblob", :type=>"varbinary(16777215)"},
49
49
  {:name=>"col_mediumint", :type=>"int3(9)", :default=>nil},
50
50
  {:name=>"col_mediumtext", :type=>"text"},
51
51
  {:name=>"col_smallint", :type=>"int2(6)", :default=>nil},
52
52
  {:name=>"col_text", :type=>"text"},
53
53
  {:name=>"col_time", :type=>"time", :default=>nil},
54
54
  {:name=>"col_timestamp", :type=>"datetime", :not_null=>true, :default=>"CURRENT_TIMESTAMP"},
55
- {:name=>"col_tinyblob", :type=>"varbinary"},
55
+ {:name=>"col_tinyblob", :type=>"varbinary(255)"},
56
56
  {:name=>"col_tinyint", :type=>"int1(4)", :default=>nil},
57
57
  {:name=>"col_tinytext", :type=>"text"},
58
58
  {:name=>"col_varbinary", :type=>"varbinary(255)", :default=>nil},
@@ -130,6 +130,23 @@ describe MysqlTableDef do
130
130
  )
131
131
  end
132
132
  end
133
+
134
+ context 'when table has unsigned column' do
135
+ let(:dump_file_io) { file_io('mysqldump_test_unsigned.dump')}
136
+ it 'should include unsigned in column type' do
137
+ expect(subject[:columns]).to eq(
138
+ [
139
+ {:name=>"id", :type=>"int4(11)", :auto_increment=>true, :not_null=>true, :primary_key=>true},
140
+ {:name=>"value_int", :type=>"int4(10) unsigned", :default=>nil},
141
+ {:name=>"value_float", :type=>"float4 unsigned", :default=>nil},
142
+ {:name=>"value_dec", :type=>"numeric(10,2) unsigned", :default=>nil},
143
+ {:name=>"value_double", :type=>"float8 unsigned", :default=>nil},
144
+ {:name=>"name", :type=>"varchar(768)", :default=>nil},
145
+ {:name=>"value_small_int", :type=>"int2(5) unsigned", :default=>nil}
146
+ ]
147
+ )
148
+ end
149
+ end
133
150
  end
134
151
  end
135
152