flydata 0.5.2 → 0.5.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ebe9adcd4a61f2b5867fa924d77c19d363a13d60
4
- data.tar.gz: 5b08edfb063b96ce328457e164a8ac2ba0613642
3
+ metadata.gz: 550d6891d0b0315d762ded08f079e915888c9b52
4
+ data.tar.gz: 477cbc3a4977c15b17b9dd3a7ecedfa3fe35595d
5
5
  SHA512:
6
- metadata.gz: 7ab6b64aecf376f7f0bb3a8a6e58ab5be27aab47e357f52f1904f420b9c85a7ad4808488e89a4983694b94708789dc316627a9b79ef2b92b7d28a17d7a50f3cc
7
- data.tar.gz: 9291d7b54299bef2fc32ff4b725f4982b57265a0bb2656dde863b7fb413d4c18fc074125a4e86c3d60deef31a3ac23dd9706d822f0025d5d8bcd747fd50c27a6
6
+ metadata.gz: d9eaccb595b37a199a92244e2e27d6b8c8cf79740a8dd69a9add01cce9a68bcd1f94428c2749fdf28d35784d03ff5c0f02c13ccd6d71b324d8b129d93a9f46c5
7
+ data.tar.gz: bcfa91ee81945016d24f1f686cb329f7c940ab5c36db7baad3886de2fbd34f12d5be6eae80811fccc74aa0f4a2dd31477be98f96045717e486a399d33c86e8e4
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.5.2
1
+ 0.5.3
data/circle.yml CHANGED
@@ -6,6 +6,7 @@ dependencies:
6
6
  pre:
7
7
  - sudo dpkg -l |grep libboost|grep 1.48|awk '{print $2}'|xargs sudo apt-get purge -y
8
8
  - sudo apt-get autoremove
9
+ - sudo apt-get update
9
10
  - sudo apt-get install -y libboost-system1.54.0
10
11
  - sudo apt-get install -y libboost-system1.54-dev
11
12
  - sudo apt-get install -y libboost-thread1.54.0
@@ -21,4 +22,4 @@ database:
21
22
 
22
23
  test:
23
24
  override:
24
- - bundle exec rspec
25
+ - ./test-suite.sh
@@ -5,6 +5,10 @@ module FlydataCore
5
5
  module Mysql
6
6
  class CompatibilityChecker
7
7
  def initialize(option = {})
8
+ if option && option.has_key?(:host) && option[:host] == 'localhost'
9
+ option = option.dup
10
+ option[:host] = '127.0.0.1' # avoid socket access which Agent never uses
11
+ end
8
12
  @option = option || {}
9
13
  end
10
14
 
@@ -113,62 +117,113 @@ module FlydataCore
113
117
  class BinlogParameterChecker < MysqlCompatibilityChecker
114
118
  include MysqlVariablesHandling
115
119
 
116
- SYS_VAR_TO_CHECK = {
120
+
121
+ BASE_SYS_VAR_TO_CHECK = {
117
122
  # parameter => expected value
118
- 'binlog_format'=>'ROW',
119
- 'binlog_checksum'=>'NONE',
120
- 'log_bin_use_v1_row_events'=>'ON',
121
- 'log_slave_updates'=>'ON',
123
+ 'binlog_format' => 'ROW',
124
+ #'binlog_checksum' => 'NONE', # no longer necessary with mysql-replication-listener 0.2.0
125
+ 'log_bin_use_v1_row_events' => 'ON',
126
+ 'log_slave_updates' => 'ON',
127
+ }
128
+
129
+ ADDITIONAL_SYS_VAR_TO_CHECK = {
122
130
  # parameter => {"comparison sign"=>minimum value}
123
- # TODO: Handled the error message of following items as warning
124
- #'net_read_timeout'=>{'>'=>600},
125
- #'net_write_timeout'=>{'>'=>600},
126
- #'wait_timeout'=>{'>'=>60}
131
+ 'net_read_timeout'=>{'>='=>600},
132
+ 'net_write_timeout'=>{'>='=>600},
133
+ 'wait_timeout'=>{'>='=>60},
127
134
  }
128
- OPTIONAL_SYS_VAR = %w(binlog_checksum log_bin_use_v1_row_events)
135
+
136
+ SYS_VAR_TO_CHECK = BASE_SYS_VAR_TO_CHECK.merge(ADDITIONAL_SYS_VAR_TO_CHECK)
129
137
 
130
138
  def check_result(result, option = @option)
131
- errors = {}
132
139
  param_hash = convert_result_to_hash(result)
133
-
134
- compare_sys_var_values(param_hash, SYS_VAR_TO_CHECK, errors, OPTIONAL_SYS_VAR)
140
+ errors = compare_sys_var_values(param_hash, self.class::SYS_VAR_TO_CHECK)
135
141
 
136
142
  unless errors.empty?
137
- error_explanation = ""
138
- errors.each_key do |err_key|
139
- if SYS_VAR_TO_CHECK[err_key].is_a?(Hash)
140
- error_explanation << "\n * #{err_key} is #{errors[err_key]} but should be at least #{SYS_VAR_TO_CHECK[err_key].values.first}"
141
- else
142
- error_explanation << "\n * #{err_key} is #{errors[err_key]} but should be #{SYS_VAR_TO_CHECK[err_key]}"
143
- end
144
- end
145
- raise FlydataCore::MysqlCompatibilityError,
146
- "These system variable(s) are not the correct value: #{error_explanation}\n" +
147
- " Please change these system variables for FlyData Sync to run"
143
+ raise FlydataCore::MysqlCompatibilityError, build_error_message(errors)
148
144
  end
149
145
  end
150
146
 
147
+ def build_error_message(errors)
148
+ err_msg = errors.inject('') {|m, (k, v)| m << "\n * #{v[:err_reason]}"}
149
+ "These system variable(s) are not the correct value: #{err_msg}\n" +
150
+ " Please change these system variables for FlyData Sync to run"
151
+ end
152
+
151
153
  private
152
- def compare_sys_var_values(input_value_hash, expected_value_hash, errors, exception_arr)
153
- expected_value_hash.each_key do |exp_val|
154
- if input_value_hash.has_key?(exp_val)
155
- actual_val = input_value_hash[exp_val]
156
- compare_sym = expected_value_hash[exp_val].is_a?(Hash) ? expected_value_hash[exp_val].keys.first : "="
157
- case compare_sym
158
- when "="
159
- expected_val = expected_value_hash[exp_val]
160
- unless actual_val == expected_val
161
- errors[exp_val] = actual_val
162
- end
163
- when ">"
164
- expected_val = expected_value_hash[exp_val].values.first
165
- unless actual_val.to_i >= expected_val.to_i
166
- errors[exp_val] = actual_val
154
+
155
+ def compare_sys_var_values(input_value_hash, expected_value_hash)
156
+ # { '<param-name>' => {actual_val: '<actual-val>', err_reason: '<err-reason>'} }
157
+ errors = {}
158
+
159
+ expected_value_hash.each do |param_name, exp_val_info|
160
+ # Ignore parameters if not returned
161
+ # binlog_checksum and log_bin_use_v1_row_events are supported only for Mysql v5.6
162
+ next unless input_value_hash.has_key?(param_name)
163
+ actual_val = input_value_hash[param_name]
164
+ err_reason = nil
165
+
166
+ if exp_val_info.kind_of?(Hash)
167
+ exp_val_info.each do |compare_sym, exp_val|
168
+ case compare_sym
169
+ when '='
170
+ unless actual_val.to_s != exp_val.to_s
171
+ err_reason = "#{param_name} is '#{actual_val}' but should be '#{exp_val}'"
172
+ end
173
+ when '>='
174
+ unless actual_val.to_i >= exp_val.to_i
175
+ err_reason = "#{param_name} is '#{actual_val}' but should be at least '#{exp_val}'"
176
+ end
177
+ else
178
+ raise "Unsupported compare symbol - #{compare_sym}"
167
179
  end
180
+ break # support just one key-value for now
168
181
  end
169
- elsif not exception_arr.include?(exp_val)
170
- errors[exp_val] = false
182
+ elsif exp_val_info.to_s != actual_val.to_s
183
+ err_reason = "#{param_name} is '#{actual_val}' but should be '#{exp_val_info}'"
171
184
  end
185
+ errors[param_name] = {actual_val: actual_val, err_reason: err_reason} if err_reason
186
+ end
187
+ errors
188
+ end
189
+ end
190
+
191
+ class RequiredBinlogParameterChecker < BinlogParameterChecker
192
+ SYS_VAR_TO_CHECK = BASE_SYS_VAR_TO_CHECK
193
+ end
194
+
195
+ class OptionalBinlogParameterChecker < BinlogParameterChecker
196
+ SYS_VAR_TO_CHECK = ADDITIONAL_SYS_VAR_TO_CHECK
197
+
198
+ def build_error_message(errors)
199
+ err_msg = errors.inject('') {|m, (k, v)| m << "\n * #{v[:err_reason]}"}
200
+ "[WARNING] These system variable(s) are not the recommended values: #{err_msg}\n" +
201
+ " These settings are not required but highly recommended for the FlyData Agent to run as efficiently as possible"
202
+ end
203
+ end
204
+
205
+ class TableExistenceChecker < MysqlCompatibilityChecker
206
+ TABLE_EXISTENCE_CHECK_QUERY_TMPLT = <<EOT
207
+ SELECT
208
+ t.table_name as table_name
209
+ FROM
210
+ information_schema.tables t
211
+ WHERE
212
+ t.table_schema='%{database}'
213
+ AND t.table_name in (%{table_names});
214
+ EOT
215
+ def create_query(option = @option)
216
+ TABLE_EXISTENCE_CHECK_QUERY_TMPLT % {database: option[:database], table_names: option[:tables].collect{|tn| "'#{tn}'"}.join(',')}
217
+ end
218
+
219
+ def check_result(result, option = @option)
220
+ existing_tables = []
221
+ result.each {|r| existing_tables << r['table_name']}
222
+ missing_tables = option[:tables] - existing_tables
223
+
224
+ unless missing_tables.empty?
225
+ raise FlydataCore::MysqlCompatibilityError,
226
+ "These tables are missing. Create these tables on your database or remove them from the data entry : #{missing_tables.join(", ")}"
172
227
  end
173
228
  end
174
229
  end
@@ -206,23 +261,26 @@ module FlydataCore
206
261
 
207
262
  class PrimaryKeyChecker < MysqlCompatibilityChecker
208
263
  PK_CHECK_QUERY_TMPLT = <<EOT
209
- SELECT k.table_name as table_name
210
- FROM information_schema.table_constraints t
211
- JOIN information_schema.key_column_usage k
212
- USING(constraint_name,table_schema,table_name)
213
- WHERE t.constraint_type='PRIMARY KEY'
214
- AND t.table_schema='%{database}'
215
- AND t.table_name in (%{table_names})
216
- GROUP BY k.table_name;
264
+ SELECT
265
+ c.table_name as table_name,
266
+ SUM(IF(c.column_key = 'PRI', 1, 0)) as num_pk
267
+ FROM
268
+ information_schema.columns c
269
+ WHERE
270
+ c.table_schema='%{database}'
271
+ AND c.table_name in (%{table_names})
272
+ GROUP BY
273
+ c.table_name
274
+ HAVING
275
+ num_pk = 0;
217
276
  EOT
218
277
  def create_query(option = @option)
219
278
  PK_CHECK_QUERY_TMPLT % {database: option[:database], table_names: option[:tables].collect{|tn| "'#{tn}'"}.join(',')}
220
279
  end
221
280
 
222
281
  def check_result(result, option = @option)
223
- pk_tables = []
224
- result.each {|r| pk_tables << r['table_name']}
225
- missing_pk_tables = option[:tables] - pk_tables
282
+ missing_pk_tables = []
283
+ result.each {|r| missing_pk_tables << r['table_name']}
226
284
 
227
285
  unless missing_pk_tables.empty?
228
286
  raise FlydataCore::MysqlCompatibilityError,
@@ -0,0 +1,361 @@
1
+ require 'spec_helper'
2
+ require 'flydata-core/mysql/compatibility_checker'
3
+
4
+ module FlydataCore
5
+ module Mysql
6
+ describe CompatibilityChecker do
7
+ let(:database) { 'test_db' }
8
+ let(:tables) { %w(table_1 table_2 table_3) }
9
+ let(:option) do
10
+ { database: database, tables: tables }
11
+ end
12
+
13
+ describe TableExistenceChecker do
14
+ let(:subject_object) { described_class.new(option) }
15
+
16
+ describe '#create_query' do
17
+ subject { subject_object.create_query }
18
+ it { is_expected.to eq <<EOT
19
+ SELECT
20
+ t.table_name as table_name
21
+ FROM
22
+ information_schema.tables t
23
+ WHERE
24
+ t.table_schema='test_db'
25
+ AND t.table_name in ('table_1','table_2','table_3');
26
+ EOT
27
+ }
28
+ end
29
+
30
+ describe '#check_reesult' do
31
+ let(:result) { [] }
32
+ subject { subject_object.check_result(result) }
33
+
34
+ context 'when all tables exist' do
35
+ let(:result) do
36
+ [ {'table_name' => 'table_1'},
37
+ {'table_name' => 'table_2'},
38
+ {'table_name' => 'table_3'}, ]
39
+ end
40
+ it { expect{subject}.not_to raise_error }
41
+ end
42
+
43
+ context 'when some tables does not exist' do
44
+ let(:result) do
45
+ [{"table_name" => "table_2"}]
46
+ end
47
+ it do
48
+ expect{subject}.to raise_error {|e|
49
+ expect(e).to be_kind_of(FlydataCore::MysqlCompatibilityError)
50
+ expect(e.to_s.include?("table_1")).to be_truthy
51
+ expect(e.to_s.include?("table_2")).to be_falsey
52
+ expect(e.to_s.include?("table_3")).to be_truthy
53
+ }
54
+ end
55
+ end
56
+ end
57
+ end
58
+
59
+ describe PrimaryKeyChecker do
60
+ let(:subject_object) { described_class.new(option) }
61
+
62
+ describe '#create_query' do
63
+ subject { subject_object.create_query }
64
+ it { is_expected.to eq <<EOT
65
+ SELECT
66
+ c.table_name as table_name,
67
+ SUM(IF(c.column_key = 'PRI', 1, 0)) as num_pk
68
+ FROM
69
+ information_schema.columns c
70
+ WHERE
71
+ c.table_schema='test_db'
72
+ AND c.table_name in ('table_1','table_2','table_3')
73
+ GROUP BY
74
+ c.table_name
75
+ HAVING
76
+ num_pk = 0;
77
+ EOT
78
+ }
79
+ end
80
+
81
+ describe '#check_reesult' do
82
+ let(:result) { [] }
83
+ subject { subject_object.check_result(result) }
84
+
85
+ context 'when all tables have pk' do
86
+ let(:result) do
87
+ []
88
+ end
89
+ it { expect{subject}.not_to raise_error }
90
+ end
91
+
92
+ context 'when some tables does not have pk' do
93
+ let(:result) do
94
+ [{"table_name" => "table_1"}, {"table_name" => "table_3"}]
95
+ end
96
+ it do
97
+ expect{subject}.to raise_error {|e|
98
+ expect(e).to be_kind_of(FlydataCore::MysqlCompatibilityError)
99
+ expect(e.to_s.include?("table_1")).to be_truthy
100
+ expect(e.to_s.include?("table_2")).to be_falsey
101
+ expect(e.to_s.include?("table_3")).to be_truthy
102
+ }
103
+ end
104
+ end
105
+ end
106
+ end
107
+ end
108
+
109
+ describe BinlogParameterChecker do
110
+ let(:subject_object) { described_class.new({}) }
111
+
112
+ describe "#check_result" do
113
+ let(:sys_var) do
114
+ [
115
+ {'Variable_name'=>'binlog_format','Value'=>'ROW'},
116
+ {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
117
+ {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
118
+ {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
119
+ {'Variable_name'=>'net_read_timeout','Value'=>600},
120
+ {'Variable_name'=>'net_write_timeout','Value'=>600},
121
+ {'Variable_name'=>'wait_timeout','Value'=>60}
122
+ ]
123
+ end
124
+ subject { subject_object.check_result(sys_var, {}) }
125
+
126
+ context "where parameters are valid" do
127
+ it do
128
+ expect{subject}.to_not raise_error
129
+ end
130
+ end
131
+ context "where parameters are invalid" do
132
+ let(:sys_var) do
133
+ [
134
+ {'Variable_name'=>'binlog_format','Value'=>'MIXED'},
135
+ {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
136
+ {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
137
+ {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
138
+ {'Variable_name'=>'net_read_timeout','Value'=>600},
139
+ {'Variable_name'=>'net_write_timeout','Value'=>600},
140
+ {'Variable_name'=>'wait_timeout','Value'=>30}
141
+ ]
142
+ end
143
+ it do
144
+ expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /binlog_format.*\n.*wait_timeout/)
145
+ end
146
+ end
147
+
148
+ context "where parameters are not returned" do
149
+ let(:sys_var) do
150
+ [
151
+ {'Variable_name'=>'binlog_format','Value'=>'ROW'},
152
+ {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
153
+ {'Variable_name'=>'net_read_timeout','Value'=>600},
154
+ {'Variable_name'=>'net_write_timeout','Value'=>600},
155
+ {'Variable_name'=>'wait_timeout','Value'=>60},
156
+ ]
157
+ end
158
+ it 'ignores missing parameters' do
159
+ expect{subject}.not_to raise_error
160
+ end
161
+ end
162
+ end
163
+ describe "#compare_sys_var_values" do
164
+ let(:input_val) { nil }
165
+ let(:expected_val) do
166
+ {
167
+ 'value_1' => 'foo',
168
+ 'value_2' => 50,
169
+ 'value_3' => {">="=>100}
170
+ }
171
+ end
172
+ let(:errors) { {} }
173
+
174
+ subject { subject_object.send(:compare_sys_var_values, input_val, expected_val) }
175
+
176
+ shared_examples "expect correct errors from values" do
177
+ it { is_expected.to eq(expecting_errors) }
178
+ end
179
+
180
+ context "when getting valid parameters" do
181
+ let(:input_val) do
182
+ {
183
+ 'value_1' => 'foo',
184
+ 'value_2' => 50,
185
+ 'value_3' => 200
186
+ }
187
+ end
188
+ let(:expecting_errors) { {} }
189
+ it_behaves_like "expect correct errors from values"
190
+ end
191
+ context "when getting one invalid exact parameters for int" do
192
+ let(:input_val) do
193
+ {
194
+ 'value_1' => 'foo',
195
+ 'value_2' => 20,
196
+ 'value_3' => 100
197
+ }
198
+ end
199
+ let(:expecting_errors) do
200
+ {
201
+ 'value_2' => { actual_val: 20, err_reason: "value_2 is '20' but should be '50'" }
202
+ }
203
+ end
204
+ it_behaves_like "expect correct errors from values"
205
+ end
206
+ context "when getting one invalid exact parameter for string" do
207
+ let(:input_val) do
208
+ {
209
+ 'value_1' => 'fo',
210
+ 'value_2' => 50,
211
+ 'value_3' => 100
212
+ }
213
+ end
214
+ let(:expecting_errors) do
215
+ {
216
+ 'value_1' => { actual_val: 'fo', err_reason: "value_1 is 'fo' but should be 'foo'" }
217
+ }
218
+ end
219
+ it_behaves_like "expect correct errors from values"
220
+ end
221
+ context "when getting one invalid comparitive parameter that's lower than required" do
222
+ let(:input_val) do
223
+ {
224
+ 'value_1' => 'foo',
225
+ 'value_2' => 50,
226
+ 'value_3' => 1
227
+ }
228
+ end
229
+ let(:expecting_errors) do
230
+ {
231
+ 'value_3' => { actual_val: 1, err_reason: "value_3 is '1' but should be at least '100'" }
232
+ }
233
+ end
234
+ it_behaves_like "expect correct errors from values"
235
+ end
236
+ context "when getting a valid comparitive parameter that's higher than required" do
237
+ let(:input_val) do
238
+ {
239
+ 'value_1' => 'foo',
240
+ 'value_2' => 50,
241
+ 'value_3' => 101
242
+ }
243
+ end
244
+ let(:expecting_errors) { {} }
245
+ it_behaves_like "expect correct errors from values"
246
+ end
247
+ context "when getting muliple invalid parameters" do
248
+ let(:input_val) do
249
+ {
250
+ 'value_1' => 'fo',
251
+ 'value_2' => 50,
252
+ 'value_3' => 15
253
+ }
254
+ end
255
+ let(:expecting_errors) { {value_1:'fo', value_3:15} }
256
+ let(:expecting_errors) do
257
+ {
258
+ 'value_1' => { actual_val: 'fo', err_reason: "value_1 is 'fo' but should be 'foo'" },
259
+ 'value_3' => { actual_val: 15, err_reason: "value_3 is '15' but should be at least '100'" },
260
+ }
261
+ end
262
+ it_behaves_like "expect correct errors from values"
263
+ end
264
+ end
265
+ end
266
+
267
+ describe RequiredBinlogParameterChecker do
268
+ let(:subject_object) { described_class.new({}) }
269
+
270
+ describe "#check_result" do
271
+ let(:sys_var) do
272
+ [
273
+ {'Variable_name'=>'binlog_format','Value'=>'ROW'},
274
+ {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
275
+ {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
276
+ {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
277
+ ]
278
+ end
279
+ subject { subject_object.check_result(sys_var) }
280
+
281
+ context 'where parameters are valid' do
282
+ it { expect{subject}.not_to raise_error }
283
+ end
284
+
285
+ context 'where parameters contain optional paramters' do
286
+ let(:sys_var) do
287
+ [
288
+ {'Variable_name'=>'binlog_format','Value'=>'ROW'},
289
+ {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
290
+ {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
291
+ {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
292
+ {'Variable_name'=>'net_read_timeout','Value'=>1},
293
+ {'Variable_name'=>'net_write_timeout','Value'=>1},
294
+ {'Variable_name'=>'wait_timeout','Value'=>1}
295
+ ]
296
+ end
297
+ it { expect{subject}.not_to raise_error }
298
+ end
299
+
300
+ context 'where parameters contain invalid parameters' do
301
+ let(:sys_var) do
302
+ [
303
+ {'Variable_name'=>'binlog_format','Value'=>'MIXED'},
304
+ {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
305
+ {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
306
+ {'Variable_name'=>'log_slave_updates','Value'=>'OFF'},
307
+ ]
308
+ end
309
+ it { expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError,
310
+ /binlog_format.*\n.*log_slave_updates/) }
311
+ end
312
+ end
313
+ end
314
+
315
+ describe OptionalBinlogParameterChecker do
316
+ let(:subject_object) { described_class.new({}) }
317
+
318
+ describe "#check_result" do
319
+ let(:sys_var) do
320
+ [
321
+ {'Variable_name'=>'net_read_timeout','Value'=>600},
322
+ {'Variable_name'=>'net_write_timeout','Value'=>600},
323
+ {'Variable_name'=>'wait_timeout','Value'=>60}
324
+ ]
325
+ end
326
+ subject { subject_object.check_result(sys_var) }
327
+
328
+ context 'where parameters are valid' do
329
+ it { expect{subject}.not_to raise_error }
330
+ end
331
+
332
+ context 'where parameters contain required paramters' do
333
+ let(:sys_var) do
334
+ [
335
+ {'Variable_name'=>'binlog_format','Value'=>'ROW'},
336
+ {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
337
+ {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
338
+ {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
339
+ {'Variable_name'=>'net_read_timeout','Value'=>600},
340
+ {'Variable_name'=>'net_write_timeout','Value'=>600},
341
+ {'Variable_name'=>'wait_timeout','Value'=>60},
342
+ ]
343
+ end
344
+ it { expect{subject}.not_to raise_error }
345
+ end
346
+
347
+ context 'where parameters contain invalid parameters' do
348
+ let(:sys_var) do
349
+ [
350
+ {'Variable_name'=>'net_read_timeout','Value'=>2},
351
+ {'Variable_name'=>'net_write_timeout','Value'=>600},
352
+ {'Variable_name'=>'wait_timeout','Value'=>1}
353
+ ]
354
+ end
355
+ it { expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError,
356
+ /^\[WARNING.*\n.*net_read_timeout.*\n.*wait_timeout/) }
357
+ end
358
+ end
359
+ end
360
+ end
361
+ end
data/flydata.gemspec CHANGED
@@ -2,16 +2,16 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: flydata 0.5.2 ruby lib
5
+ # stub: flydata 0.5.3 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "flydata"
9
- s.version = "0.5.2"
9
+ s.version = "0.5.3"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib"]
13
13
  s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
14
- s.date = "2015-08-14"
14
+ s.date = "2015-09-03"
15
15
  s.description = "FlyData Agent"
16
16
  s.email = "sysadmin@flydata.com"
17
17
  s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -64,7 +64,7 @@ Gem::Specification.new do |s|
64
64
  "flydata-core/spec/fluent/config_helper_spec.rb",
65
65
  "flydata-core/spec/logger_spec.rb",
66
66
  "flydata-core/spec/mysql/command_generator_spec.rb",
67
- "flydata-core/spec/mysql/compatibility_checker.rb",
67
+ "flydata-core/spec/mysql/compatibility_checker_spec.rb",
68
68
  "flydata-core/spec/option_validator_spec.rb",
69
69
  "flydata-core/spec/redshift/string_spec.rb",
70
70
  "flydata-core/spec/spec_helper.rb",
@@ -219,6 +219,7 @@ Gem::Specification.new do |s|
219
219
  "spec/flydata/util/shell_spec.rb",
220
220
  "spec/flydata_spec.rb",
221
221
  "spec/spec_helper.rb",
222
+ "test-suite.sh",
222
223
  "tmpl/redshift_mysql_data_entry.conf.tmpl"
223
224
  ]
224
225
  s.homepage = "http://flydata.com/"
@@ -111,7 +111,12 @@ module Flydata
111
111
  end
112
112
 
113
113
  def check_mysql_parameters_compat
114
- FlydataCore::Mysql::BinlogParameterChecker.new(@db_opts).do_check
114
+ begin
115
+ FlydataCore::Mysql::OptionalBinlogParameterChecker.new(@db_opts).do_check
116
+ rescue FlydataCore::MysqlCompatibilityError => e
117
+ log_warn_stderr(e.to_s)
118
+ end
119
+ FlydataCore::Mysql::RequiredBinlogParameterChecker.new(@db_opts).do_check
115
120
  end
116
121
 
117
122
  def check_mysql_binlog_retention
data/test-suite.sh ADDED
@@ -0,0 +1,56 @@
1
+ #!/bin/bash
2
+
3
+ # Usage: $0 [Options]
4
+ # Specify project
5
+ # [-m] # main project
6
+ # [-c] # core project
7
+ # Test skip option
8
+ # [--skip-main]
9
+ # [--skip-core]
10
+
11
+ set -e
12
+ BASE_DIR=$(cd $(dirname $0); pwd)
13
+
14
+ #### Parse option
15
+
16
+ TEST_MODE="skip" # skip or specify
17
+ for OPT in $*
18
+ do
19
+ case $OPT in
20
+ -m) TEST_MODE='specify'; SPECIFY_M=1; shift;;
21
+ -c) TEST_MODE='specify'; SPECIFY_C=1; shift;;
22
+ --skip-main) TEST_MODE='skip'; SKIP_M=1; shift;;
23
+ --skip-core) TEST_MODE='skip'; SKIP_C=1; shift;;
24
+ esac
25
+ done
26
+
27
+ if [ "$TEST_MODE" = 'skip' ]; then
28
+ if [ "$SKIP_M" = "" ]; then SPECIFY_M=1; fi
29
+ if [ "$SKIP_C" = "" ]; then SPECIFY_C=1; fi
30
+ fi
31
+
32
+ #### Test functions
33
+
34
+ # Test flydata-agent library (flydata-agent/lib, flydata-agent/spec)
35
+ test_main() {
36
+ cd $BASE_DIR
37
+ bundle install
38
+ bundle exec rspec
39
+ }
40
+
41
+ # Test flydata-core (flydata-core/lib, flydata-core/spec)
42
+ test_core() {
43
+ cd $BASE_DIR/flydata-core
44
+ bundle install
45
+ bundle exec rspec
46
+ }
47
+
48
+ #### Main
49
+
50
+ if [ "$SPECIFY_C" = '1' ]; then
51
+ test_core
52
+ fi
53
+
54
+ if [ "$SPECIFY_M" = '1' ]; then
55
+ test_main
56
+ fi
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.2
4
+ version: 0.5.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2015-08-14 00:00:00.000000000 Z
15
+ date: 2015-09-03 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -501,7 +501,7 @@ files:
501
501
  - flydata-core/spec/fluent/config_helper_spec.rb
502
502
  - flydata-core/spec/logger_spec.rb
503
503
  - flydata-core/spec/mysql/command_generator_spec.rb
504
- - flydata-core/spec/mysql/compatibility_checker.rb
504
+ - flydata-core/spec/mysql/compatibility_checker_spec.rb
505
505
  - flydata-core/spec/option_validator_spec.rb
506
506
  - flydata-core/spec/redshift/string_spec.rb
507
507
  - flydata-core/spec/spec_helper.rb
@@ -656,6 +656,7 @@ files:
656
656
  - spec/flydata/util/shell_spec.rb
657
657
  - spec/flydata_spec.rb
658
658
  - spec/spec_helper.rb
659
+ - test-suite.sh
659
660
  - tmpl/redshift_mysql_data_entry.conf.tmpl
660
661
  homepage: http://flydata.com/
661
662
  licenses:
@@ -1,186 +0,0 @@
1
- require 'spec_helper'
2
- require 'flydata-core/mysql/compatibility_checker'
3
-
4
- module FlydataCore
5
- module Mysql
6
- describe CompatibilityChecker do
7
- let(:database) { 'test_db' }
8
- let(:tables) { %w(table_1 table_2 table_3) }
9
- let(:option) do
10
- { database: database, tables: tables }
11
- end
12
-
13
- describe PrimaryKeyChecker do
14
- let(:subject_object) { described_class.new(option) }
15
-
16
- describe '#create_query' do
17
- subject { subject_object.create_query }
18
- it { is_expected.to eq <<EOT
19
- SELECT k.table_name as table_name
20
- FROM information_schema.table_constraints t
21
- JOIN information_schema.key_column_usage k
22
- USING(constraint_name,table_schema,table_name)
23
- WHERE t.constraint_type='PRIMARY KEY'
24
- AND t.table_schema='test_db'
25
- AND t.table_name in ('table_1','table_2','table_3')
26
- GROUP BY k.table_name;
27
- EOT
28
- }
29
- end
30
-
31
- describe '#check_reesult' do
32
- let(:result) { [] }
33
- subject { subject_object.check_result(result) }
34
-
35
- context 'when all tables have pk' do
36
- let(:result) do
37
- [ {"table_name" => "table_1"},
38
- {"table_name" => "table_2"},
39
- {"table_name" => "table_3"}, ]
40
- end
41
- it { expect{subject}.not_to raise_error }
42
- end
43
-
44
- context 'when some tables does not have pk' do
45
- let(:result) { [{"table_name" => "table_2"} ] }
46
- it do
47
- expect{subject}.to raise_error {|e|
48
- expect(e).to be_kind_of(FlydataCore::MysqlCompatibilityError)
49
- expect(e.to_s.include?("table_1")).to be_truthy
50
- expect(e.to_s.include?("table_2")).to be_falsey
51
- expect(e.to_s.include?("table_3")).to be_truthy
52
- }
53
- end
54
- end
55
- end
56
- end
57
- end
58
-
59
- describe BinlogParameterChecker do
60
- let(:subject_object) { BinlogParameterChecker.new({}) }
61
-
62
- describe "#check_result" do
63
- let(:sys_var) do
64
- [
65
- {'Variable_name'=>'binlog_format','Value'=>'ROW'},
66
- {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
67
- {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
68
- {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
69
- {'Variable_name'=>'net_read_timeout','Value'=>600},
70
- {'Variable_name'=>'net_write_timeout','Value'=>600},
71
- {'Variable_name'=>'wait_timeout','Value'=>60}
72
- ]
73
- end
74
- subject { subject_object.check_result(sys_var, {}) }
75
-
76
- context "where parameters are valid" do
77
- it do
78
- expect{subject}.to_not raise_error
79
- end
80
- end
81
- context "where parameters are invalid" do
82
- let(:sys_var) do
83
- [
84
- {'Variable_name'=>'binlog_format','Value'=>'MIXED'},
85
- {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
86
- {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
87
- {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
88
- {'Variable_name'=>'net_read_timeout','Value'=>600},
89
- {'Variable_name'=>'net_write_timeout','Value'=>600},
90
- {'Variable_name'=>'wait_timeout','Value'=>30}
91
- ]
92
- end
93
- it do
94
- expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /binlog_format.*\n.*wait_timeout/)
95
- end
96
- end
97
- end
98
- describe "#compare_sys_var_values" do
99
- let(:expected_val) do
100
- {
101
- :value_1 => 'foo',
102
- :value_2 => 50,
103
- :value_3 => {">"=>100}
104
- }
105
- end
106
- let(:errors) { {} }
107
- let(:exception_arr) { [] }
108
- subject { subject_object.send(:compare_sys_var_values, input_val, expected_val, errors, exception_arr) }
109
-
110
- shared_examples "expect correct errors from values" do
111
- it do
112
- subject
113
- expect(errors).to eq( expecting_errors )
114
- end
115
- end
116
-
117
- context "when getting valid parameters" do
118
- let(:input_val) do
119
- {
120
- :value_1 => 'foo',
121
- :value_2 => 50,
122
- :value_3 => 200
123
- }
124
- end
125
- let(:expecting_errors) { {} }
126
- it_behaves_like "expect correct errors from values"
127
- end
128
- context "when getting one invalid exact parameters for int" do
129
- let(:input_val) do
130
- {
131
- :value_1 => 'foo',
132
- :value_2 => 20,
133
- :value_3 => 100
134
- }
135
- end
136
- let(:expecting_errors) { {value_2:20} }
137
- it_behaves_like "expect correct errors from values"
138
- end
139
- context "when getting one invalid exact parameter for string" do
140
- let(:input_val) do
141
- {
142
- :value_1 => 'fo',
143
- :value_2 => 50,
144
- :value_3 => 100
145
- }
146
- end
147
- let(:expecting_errors) { {value_1:'fo'} }
148
- it_behaves_like "expect correct errors from values"
149
- end
150
- context "when getting one invalid comparitive parameter that's lower than required" do
151
- let(:input_val) do
152
- {
153
- :value_1 => 'foo',
154
- :value_2 => 50,
155
- :value_3 => 1
156
- }
157
- end
158
- let(:expecting_errors) { {value_3:1} }
159
- it_behaves_like "expect correct errors from values"
160
- end
161
- context "when getting a valid comparitive parameter that's higher than required" do
162
- let(:input_val) do
163
- {
164
- :value_1 => 'foo',
165
- :value_2 => 50,
166
- :value_3 => 101
167
- }
168
- end
169
- let(:expecting_errors) { {} }
170
- it_behaves_like "expect correct errors from values"
171
- end
172
- context "when getting muliple invalid parameters" do
173
- let(:input_val) do
174
- {
175
- :value_1 => 'fo',
176
- :value_2 => 50,
177
- :value_3 => 15
178
- }
179
- end
180
- let(:expecting_errors) { {value_1:'fo', value_3:15} }
181
- it_behaves_like "expect correct errors from values"
182
- end
183
- end
184
- end
185
- end
186
- end