flydata 0.1.9 → 0.1.10

Sign up to get free protection for your applications and to get access to all the features.
data/Gemfile CHANGED
@@ -15,7 +15,7 @@ gem "slop"
15
15
  group :development do
16
16
  gem "bundler"
17
17
  gem "jeweler"
18
- gem "rspec"
18
+ gem "rspec", '~> 3.0'
19
19
  #gem "autotest"
20
20
  #gem "autotest-standalone"
21
21
  #gem "autotest-notification"
data/Gemfile.lock CHANGED
@@ -22,7 +22,7 @@ GEM
22
22
  atomic (1.1.14)
23
23
  builder (3.1.4)
24
24
  cool.io (1.2.3)
25
- diff-lcs (1.2.4)
25
+ diff-lcs (1.2.5)
26
26
  faraday (0.8.8)
27
27
  multipart-post (~> 1.2.0)
28
28
  fluent-plugin-mysql-binlog (0.0.2)
@@ -86,14 +86,18 @@ GEM
86
86
  json (~> 1.4)
87
87
  rest-client (1.6.7)
88
88
  mime-types (>= 1.16)
89
- rspec (2.14.1)
90
- rspec-core (~> 2.14.0)
91
- rspec-expectations (~> 2.14.0)
92
- rspec-mocks (~> 2.14.0)
93
- rspec-core (2.14.5)
94
- rspec-expectations (2.14.3)
95
- diff-lcs (>= 1.1.3, < 2.0)
96
- rspec-mocks (2.14.3)
89
+ rspec (3.0.0)
90
+ rspec-core (~> 3.0.0)
91
+ rspec-expectations (~> 3.0.0)
92
+ rspec-mocks (~> 3.0.0)
93
+ rspec-core (3.0.2)
94
+ rspec-support (~> 3.0.0)
95
+ rspec-expectations (3.0.2)
96
+ diff-lcs (>= 1.2.0, < 2.0)
97
+ rspec-support (~> 3.0.0)
98
+ rspec-mocks (3.0.2)
99
+ rspec-support (~> 3.0.0)
100
+ rspec-support (3.0.2)
97
101
  ruby-binlog (1.0.1)
98
102
  ruby-prof (0.14.2)
99
103
  sigdump (0.2.2)
@@ -122,7 +126,7 @@ DEPENDENCIES
122
126
  mysql2 (~> 0.3.11)
123
127
  protected_attributes
124
128
  rest-client (~> 1.6.7)
125
- rspec
129
+ rspec (~> 3.0)
126
130
  ruby-binlog (>= 1.0.1)
127
131
  ruby-prof
128
132
  slop
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.9
1
+ 0.1.10
data/flydata.gemspec CHANGED
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = "flydata"
8
- s.version = "0.1.9"
8
+ s.version = "0.1.10"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Koichi Fujikawa"]
12
- s.date = "2014-07-01"
12
+ s.date = "2014-07-15"
13
13
  s.description = "FlyData Command Line Interface"
14
14
  s.email = "sysadmin@flydata.co"
15
15
  s.executables = ["fdmysqldump", "flydata"]
@@ -58,7 +58,10 @@ Gem::Specification.new do |s|
58
58
  "lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb",
59
59
  "lib/flydata/fluent-plugins/mysql/context.rb",
60
60
  "lib/flydata/fluent-plugins/mysql/dml_record_handler.rb",
61
- "lib/flydata/fluent-plugins/mysql/query_parser.rb",
61
+ "lib/flydata/fluent-plugins/mysql/parser.rb",
62
+ "lib/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser.rb",
63
+ "lib/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser.rb",
64
+ "lib/flydata/fluent-plugins/mysql/parser/query_parser.rb",
62
65
  "lib/flydata/fluent-plugins/out_forward_ssl.rb",
63
66
  "lib/flydata/fluent-plugins/preference.rb",
64
67
  "lib/flydata/flydata_crontab.sh",
@@ -82,7 +85,8 @@ Gem::Specification.new do |s|
82
85
  "spec/flydata/command/sync_spec.rb",
83
86
  "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
84
87
  "spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
85
- "spec/flydata/fluent-plugins/mysql/query_parser_spec.rb",
88
+ "spec/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser_spec.rb",
89
+ "spec/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser_spec.rb",
86
90
  "spec/flydata/heroku_spec.rb",
87
91
  "spec/flydata/table_def/mysql_table_def_spec.rb",
88
92
  "spec/flydata/table_def/mysqldump_test_foreign_key.dump",
@@ -120,7 +124,7 @@ Gem::Specification.new do |s|
120
124
  s.add_runtime_dependency(%q<slop>, [">= 0"])
121
125
  s.add_development_dependency(%q<bundler>, [">= 0"])
122
126
  s.add_development_dependency(%q<jeweler>, [">= 0"])
123
- s.add_development_dependency(%q<rspec>, [">= 0"])
127
+ s.add_development_dependency(%q<rspec>, ["~> 3.0"])
124
128
  s.add_development_dependency(%q<timecop>, [">= 0"])
125
129
  s.add_development_dependency(%q<sqlite3>, [">= 0"])
126
130
  s.add_development_dependency(%q<ruby-prof>, [">= 0"])
@@ -140,7 +144,7 @@ Gem::Specification.new do |s|
140
144
  s.add_dependency(%q<slop>, [">= 0"])
141
145
  s.add_dependency(%q<bundler>, [">= 0"])
142
146
  s.add_dependency(%q<jeweler>, [">= 0"])
143
- s.add_dependency(%q<rspec>, [">= 0"])
147
+ s.add_dependency(%q<rspec>, ["~> 3.0"])
144
148
  s.add_dependency(%q<timecop>, [">= 0"])
145
149
  s.add_dependency(%q<sqlite3>, [">= 0"])
146
150
  s.add_dependency(%q<ruby-prof>, [">= 0"])
@@ -161,7 +165,7 @@ Gem::Specification.new do |s|
161
165
  s.add_dependency(%q<slop>, [">= 0"])
162
166
  s.add_dependency(%q<bundler>, [">= 0"])
163
167
  s.add_dependency(%q<jeweler>, [">= 0"])
164
- s.add_dependency(%q<rspec>, [">= 0"])
168
+ s.add_dependency(%q<rspec>, ["~> 3.0"])
165
169
  s.add_dependency(%q<timecop>, [">= 0"])
166
170
  s.add_dependency(%q<sqlite3>, [">= 0"])
167
171
  s.add_dependency(%q<ruby-prof>, [">= 0"])
@@ -10,6 +10,10 @@ module Flydata
10
10
  def buffer_stat(data_entry_id, mode = nil)
11
11
  @client.get("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{mode}")
12
12
  end
13
+
14
+ def cleanup_sync(data_entry_id, tables)
15
+ @client.post("/#{@model_name.pluralize}/#{data_entry_id}/cleanup_sync", nil, {tables: tables.join(',')})
16
+ end
13
17
  end
14
18
  end
15
19
  end
@@ -25,7 +25,7 @@ module Flydata
25
25
  puts "FlyData Agent is already running. If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first."
26
26
  else
27
27
  # per-table sync
28
- puts "Flydata Agent is already running. If you'd like to Sync the table(s), run 'flydata flush' first."
28
+ puts "Flydata Agent is already running. If you'd like to Sync the table(s), run 'flydata sync:flush' first."
29
29
  end
30
30
  exit 1
31
31
  end
@@ -46,14 +46,20 @@ module Flydata
46
46
  puts "Buffers have been flushed and the sender process has been stopped."
47
47
  end
48
48
 
49
+ def self.slop_reset
50
+ Slop.new do
51
+ on 'c', 'client', 'Resets client only.'
52
+ end
53
+ end
54
+
49
55
  def reset
50
56
  return unless ask_yes_no("This resets the current Sync. Are you sure?")
51
57
  sender = Flydata::Command::Sender.new
52
58
  sender.flush_client_buffer # TODO We should rather delete buffer files
53
- # TODO Reset server when API becomes available
54
59
  sender.stop
55
60
 
56
61
  de = retrieve_data_entries.first
62
+ cleanup_sync_server(de) unless opts.client?
57
63
  sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
58
64
  [
59
65
  sync_fm.dump_file_path,
@@ -70,14 +76,14 @@ module Flydata
70
76
 
71
77
  def wait_for_server_data_processing
72
78
  state = :PROCESS
73
- puts "Processing data..."
79
+ puts "Uploading data to Redshift..."
74
80
  sleep 10
75
81
  status = nil
76
82
  while (status = check)
77
83
  if state == :PROCESS && status['state'] == 'uploading'
78
84
  puts " -> Done"
79
85
  state = :UPLOAD
80
- puts "Uploading data to Redshift..."
86
+ puts "Finishing data upload..."
81
87
  end
82
88
  print_progress(status)
83
89
  sleep 10
@@ -85,7 +91,7 @@ module Flydata
85
91
  if (state == :PROCESS)
86
92
  # :UPLOAD state was skipped due to no data
87
93
  puts " -> Done"
88
- puts "Uploading data to Redshift..."
94
+ puts "Finishing data upload..."
89
95
  end
90
96
  puts " -> Done"
91
97
  end
@@ -126,6 +132,11 @@ module Flydata
126
132
 
127
133
  private
128
134
 
135
+ def cleanup_sync_server(de, tables = [])
136
+ puts "Checking the server and cleaning up"
137
+ flydata.data_entry.cleanup_sync(de['id'], tables)
138
+ end
139
+
129
140
  def do_check(de)
130
141
  flydata.data_entry.buffer_stat(de['id'], env_mode)
131
142
  end
@@ -264,8 +275,6 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
264
275
  #...
265
276
  #CREATE TABLE ...
266
277
  def parse_mysqldump_and_send(dp, de, sync_fm)
267
- puts "Sending data to FlyData Server..."
268
-
269
278
  # Prepare forwarder
270
279
  de_tag_name = de["tag_name#{env_suffix}"]
271
280
  server_port = dp['server_port']
@@ -283,8 +292,12 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
283
292
  option = dump_pos_info || {}
284
293
  if option[:table_name]
285
294
  puts "Resuming... Last processed table: #{option[:table_name]}"
295
+ else
296
+ #If its a new sync, ensure server side resources are clean
297
+ cleanup_sync_server(de, de['mysql_data_entry_preference']['tables'].split(','))
286
298
  end
287
-
299
+ puts "Sending data to FlyData Server..."
300
+
288
301
  bench_start_time = Time.now
289
302
 
290
303
  # Start parsing dump file
@@ -1,4 +1,5 @@
1
1
  require_relative 'binlog_query_handler'
2
+ require_relative 'parser'
2
3
 
3
4
  module Mysql
4
5
  class AlterTableQueryHandler < BinlogQueryHandler
@@ -16,6 +17,8 @@ module Mysql
16
17
  case normalized_query
17
18
  when /^alter table [^\s]+ add column/i
18
19
  on_add_column(record, normalized_query)
20
+ when /^alter table [^\s]+ drop/i
21
+ on_drop_column(record, normalized_query)
19
22
  else
20
23
  $log.debug("not supported alter table query:'#{normalized_query}'")
21
24
  end
@@ -27,7 +30,15 @@ module Mysql
27
30
  $log.debug("on_add_column query:'#{record['query']}'")
28
31
  #TODO: Uncomment following lines after supporting alter table on the server side
29
32
  #emit_record(:alter_table, record, increment_table_rev: true) do
30
- # AlterTableAddColumnParser.new.parse(record['query'])
33
+ # Parser::AlterTableAddColumnParser.new.parse(record['query'])
34
+ #end
35
+ end
36
+
37
+ def on_drop_column(record, query)
38
+ $log.debug("on_drop_column query:'#{record['query']}'")
39
+ #TODO: Uncomment following lines after supporting alter table on the server side
40
+ #emit_record(:alter_table, record, increment_table_rev: true) do
41
+ # Parser::AlterTableDropColumnParser.new.parse(record['query'])
31
42
  #end
32
43
  end
33
44
  end
@@ -1,7 +1,6 @@
1
1
  require 'fluent/plugin/in_mysql_binlog'
2
2
  require 'binlog'
3
3
  require_relative 'binlog_position'
4
- require_relative 'query_parser'
5
4
 
6
5
  module Mysql
7
6
  class BinlogRecordHandler
@@ -54,6 +53,7 @@ module Mysql
54
53
  check_empty_binlog
55
54
 
56
55
  records = yield
56
+ return if records.nil? # skip
57
57
  records = [records] unless records.kind_of?(Array)
58
58
 
59
59
  table = records.first[TABLE_NAME] || record['table_name']
@@ -0,0 +1,2 @@
1
+ require_relative 'parser/alter_table_add_column_parser'
2
+ require_relative 'parser/alter_table_drop_column_parser'
@@ -1,11 +1,6 @@
1
- require_relative '../../table_def'
1
+ require_relative 'query_parser'
2
2
 
3
- module Mysql
4
- class QueryParser
5
- # Return hash object
6
- def parse(query)
7
- end
8
- end
3
+ module Mysql::Parser
9
4
 
10
5
  class AlterTableAddColumnParser < QueryParser
11
6
  # Return hash object or array
@@ -67,3 +62,4 @@ module Mysql
67
62
  end
68
63
  end
69
64
  end
65
+
@@ -0,0 +1,49 @@
1
+ require_relative 'query_parser'
2
+
3
+ module Mysql::Parser
4
+
5
+ class AlterTableDropColumnParser < QueryParser
6
+ # Return hash object or array
7
+ # {
8
+ # alter_action : "drop_column",
9
+ # table_name : table_name,
10
+ # column : {
11
+ # name: column_name,
12
+ # }
13
+ # }
14
+ def parse(query)
15
+ do_parse(query)
16
+ rescue
17
+ $log.error("Failed to parse query. query:'#{query}' error:#{$!}")
18
+ raise
19
+ end
20
+
21
+ private
22
+
23
+ def do_parse(query)
24
+ m = /^\s*alter\s+table\s+`?(?<table_name>[^\s`]+)`?\s+drop\s+/i.match(query)
25
+ table_name = m['table_name']
26
+ query = query[m[0].length..-1]
27
+
28
+ column_name = extract_column_name(query)
29
+ return nil unless column_name
30
+
31
+ { subtype: :drop_column,
32
+ table_name: table_name,
33
+ column: { name: column_name } }
34
+ end
35
+
36
+ # Parse following drop column
37
+ #
38
+ # DROP [COLUMN] col_name
39
+ # DROP PRIMARY KEY # ignore
40
+ # DROP {INDEX|KEY} index_name # ignore
41
+ # DROP FOREIGN KEY fk_symbol # ignore
42
+ def extract_column_name(query)
43
+ return nil if /primary\s+key|(foreign\s+key|key|index)\s+[^\s]+/i.match(query)
44
+ m = /(column\s+)?`?(?<column_name>[^\s`]+)`?/i.match(query)
45
+ m['column_name']
46
+ end
47
+ end
48
+ end
49
+
@@ -0,0 +1,9 @@
1
+ require_relative '../../../table_def'
2
+
3
+ module Mysql::Parser
4
+ class QueryParser
5
+ # Return hash object
6
+ def parse(query)
7
+ end
8
+ end
9
+ end
@@ -6,7 +6,7 @@ module Flydata
6
6
  describe '#create' do
7
7
  it 'creates data entry' do
8
8
  api_client = double('api_client')
9
- api_client.stub(:post)
9
+ allow(api_client).to receive(:post)
10
10
 
11
11
  api = DataEntry.new(api_client)
12
12
  api.create(data_port_id: 1, log_path: 'log-path')
@@ -42,8 +42,8 @@ module Flydata
42
42
  end
43
43
  end
44
44
  end
45
- pending '#stop'
46
- pending '#restart'
45
+ skip '#stop'
46
+ skip '#restart'
47
47
  end
48
48
  end
49
49
  end
@@ -300,21 +300,21 @@ module Flydata
300
300
  context 'with nil forwarder_key' do
301
301
  it 'should return TcpForwarder object' do
302
302
  forwarder = ForwarderFactory.create(nil, 'test_tag', ['localhost:3456', 'localhost:4567'])
303
- expect(forwarder.kind_of?(TcpForwarder)).to be_true
303
+ expect(forwarder.kind_of?(TcpForwarder)).to be_truthy
304
304
  end
305
305
  end
306
306
 
307
307
  context 'with tcpforwarder forwarder_key' do
308
308
  it 'should return TcpForwarder object' do
309
309
  forwarder = ForwarderFactory.create('tcpforwarder', 'test_tag', ['localhost:3456', 'localhost:4567'])
310
- expect(forwarder.kind_of?(TcpForwarder)).to be_true
310
+ expect(forwarder.kind_of?(TcpForwarder)).to be_truthy
311
311
  end
312
312
  end
313
313
 
314
314
  context 'with sslforwarder forwarder_key' do
315
315
  it 'should return SslForwarder object' do
316
316
  forwarder = ForwarderFactory.create('sslforwarder', 'test_tag', ['localhost:3456', 'localhost:4567'])
317
- expect(forwarder.kind_of?(SslForwarder)).to be_true
317
+ expect(forwarder.kind_of?(SslForwarder)).to be_truthy
318
318
  end
319
319
  end
320
320
  end
@@ -397,7 +397,7 @@ module Flydata
397
397
  describe '#initialize' do
398
398
  context 'with password' do
399
399
  subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
400
- it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
400
+ it { is_expected.to eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
401
401
  '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
402
402
  end
403
403
  context 'without password' do
@@ -405,7 +405,7 @@ module Flydata
405
405
  MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
406
406
  end
407
407
  subject { dump_generator.instance_variable_get(:@dump_cmd) }
408
- it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
408
+ it { is_expected.to eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
409
409
  '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
410
410
  end
411
411
  end
@@ -421,7 +421,7 @@ module Flydata
421
421
  end
422
422
  it do
423
423
  expect{ default_dump_generator.dump(file_path) }.to raise_error
424
- expect(File.exists?(file_path)).to be_false
424
+ expect(File.exists?(file_path)).to be_falsey
425
425
  end
426
426
  end
427
427
  context 'when exit status is 0 but no file' do
@@ -433,7 +433,7 @@ module Flydata
433
433
  end
434
434
  it do
435
435
  expect{ default_dump_generator.dump(file_path) }.to raise_error
436
- expect(File.exists?(file_path)).to be_false
436
+ expect(File.exists?(file_path)).to be_falsey
437
437
  end
438
438
  end
439
439
  context 'when exit status is 0 but file size is 0' do
@@ -446,7 +446,7 @@ module Flydata
446
446
  end
447
447
  it do
448
448
  expect{ default_dump_generator.dump(file_path) }.to raise_error
449
- expect(File.exists?(file_path)).to be_true
449
+ expect(File.exists?(file_path)).to be_truthy
450
450
  end
451
451
  end
452
452
  context 'when exit status is 0' do
@@ -458,8 +458,8 @@ module Flydata
458
458
  )
459
459
  end
460
460
  it do
461
- expect(default_dump_generator.dump(file_path)).to be_true
462
- expect(File.exists?(file_path)).to be_true
461
+ expect(default_dump_generator.dump(file_path)).to be_truthy
462
+ expect(File.exists?(file_path)).to be_truthy
463
463
  end
464
464
  end
465
465
  after :each do
@@ -155,6 +155,7 @@ EOT
155
155
  let(:update_two_byte_event) { create_event(TEST_EVENT_TWO_BYTE_UPDATE) }
156
156
  let(:update_three_byte_event) { create_event(TEST_EVENT_THREE_BYTE_UPDATE) }
157
157
  let(:alter_table_add_column_event) { create_event(TEST_EVENT_QUERY_ALTER_TABLE_ADD_COLUMN) }
158
+ let(:alter_table_drop_column_event) { create_event(TEST_EVENT_QUERY_ALTER_TABLE_DROP_COLUMN) }
158
159
 
159
160
  let(:query_event) { create_event(TEST_EVENT_QUERY_CREATE_DATABSE) }
160
161
  let(:table_map_event) { create_event(TEST_EVENT_TABLE_MAP) }
@@ -166,9 +167,9 @@ EOT
166
167
 
167
168
  let(:table_seq_file) {
168
169
  f = double('table_seq_file')
169
- f.stub(:read).and_return('1')
170
- f.stub(:rewind)
171
- f.stub(:truncate)
170
+ allow(f).to receive(:read).and_return('1')
171
+ allow(f).to receive(:rewind)
172
+ allow(f).to receive(:truncate)
172
173
  f
173
174
  }
174
175
 
@@ -197,7 +198,7 @@ EOT
197
198
 
198
199
  context 'when received insert event' do
199
200
  it do
200
- table_seq_file.should_receive(:write).exactly(3).with(2)
201
+ expect(table_seq_file).to receive(:write).exactly(3).with(2)
201
202
  expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "mysql-bin.000048",
202
203
  [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
203
204
  end
@@ -205,7 +206,7 @@ EOT
205
206
 
206
207
  context 'when received insert event containing two byte UTF8 chars' do
207
208
  it do
208
- table_seq_file.should_receive(:write).exactly(3).with(2)
209
+ expect(table_seq_file).to receive(:write).exactly(3).with(2)
209
210
  expect_emitted_records_with_rows(insert_two_byte_event, :insert, TEST_TABLE, 628, "mysql-bin.000048",
210
211
  [{"1"=>"0SL00000001", "2"=>"føø"}, {"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
211
212
  end
@@ -213,7 +214,7 @@ EOT
213
214
 
214
215
  context 'when received insert event containing three byte UTF8 chars' do
215
216
  it do
216
- table_seq_file.should_receive(:write).exactly(3).with(2)
217
+ expect(table_seq_file).to receive(:write).exactly(3).with(2)
217
218
  expect_emitted_records_with_rows(insert_three_byte_event, :insert, TEST_TABLE, 628, "mysql-bin.000048",
218
219
  [{"1"=>"0SL00000001", "2"=>"富无无"}, {"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
219
220
  end
@@ -221,7 +222,7 @@ EOT
221
222
 
222
223
  context 'when received delete event' do
223
224
  it do
224
- table_seq_file.should_receive(:write).twice.with(2)
225
+ expect(table_seq_file).to receive(:write).twice.with(2)
225
226
  expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048",
226
227
  [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
227
228
  end
@@ -229,7 +230,7 @@ EOT
229
230
 
230
231
  context 'when received delete event containing two byte UTF8 chars' do
231
232
  it do
232
- table_seq_file.should_receive(:write).twice.with(2)
233
+ expect(table_seq_file).to receive(:write).twice.with(2)
233
234
  expect_emitted_records_with_rows(delete_two_byte_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048",
234
235
  [{"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
235
236
  end
@@ -237,7 +238,7 @@ EOT
237
238
 
238
239
  context 'when received delete event with containing byte UTF8 chars' do
239
240
  it do
240
- table_seq_file.should_receive(:write).twice.with(2)
241
+ expect(table_seq_file).to receive(:write).twice.with(2)
241
242
  expect_emitted_records_with_rows(delete_three_byte_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048",
242
243
  [{"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
243
244
  end
@@ -245,7 +246,7 @@ EOT
245
246
 
246
247
  context 'when received update event' do
247
248
  it do
248
- table_seq_file.should_receive(:write).twice.with(2)
249
+ expect(table_seq_file).to receive(:write).twice.with(2)
249
250
  expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
250
251
  [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
251
252
  end
@@ -253,7 +254,7 @@ EOT
253
254
 
254
255
  context 'when received update event with two byte utf8 chars' do
255
256
  it do
256
- table_seq_file.should_receive(:write).twice.with(2)
257
+ expect(table_seq_file).to receive(:write).twice.with(2)
257
258
  expect_emitted_records_with_rows(update_two_byte_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
258
259
  [{"1"=>"0SL00000001", "2"=>"∑ø∑"}, {"1"=>"0SL00000003", "2"=>"fügå"}])
259
260
  end
@@ -261,7 +262,7 @@ EOT
261
262
 
262
263
  context 'when received update event with three byte utf8 chars' do
263
264
  it do
264
- table_seq_file.should_receive(:write).twice.with(2)
265
+ expect(table_seq_file).to receive(:write).twice.with(2)
265
266
  expect_emitted_records_with_rows(update_three_byte_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
266
267
  [{"1"=>"0SL00000001", "2"=>"很兴奋"}, {"1"=>"0SL00000003", "2"=>"興奮虎"}])
267
268
  end
@@ -278,12 +279,12 @@ EOT
278
279
  end
279
280
 
280
281
  #TODO: Uncomment the following test and delete the above one after supporting alter table
281
- =begin
282
- context 'when received alter table event' do
282
+ context 'when received alter table add column event' do
283
283
  before do
284
284
  allow(File).to receive(:open).with(/test_table\.rev/, 'w')
285
285
  end
286
286
  it do
287
+ skip "Now alter table add column is not supported"
287
288
  expect(table_seq_file).to receive(:write).with(2).once
288
289
  expect_emitted_records(alter_table_add_column_event, {
289
290
  subtype: :add_column,
@@ -297,7 +298,26 @@ EOT
297
298
  })
298
299
  end
299
300
  end
300
- =end
301
+
302
+ context 'when received alter table drop column event' do
303
+ before do
304
+ allow(File).to receive(:open).with(/test_table\.rev/, 'w')
305
+ end
306
+ it do
307
+ skip "Now alter table drop column is not supported"
308
+ expect(table_seq_file).to receive(:write).with(2).once
309
+ expect_emitted_records(alter_table_drop_column_event, {
310
+ subtype: :drop_column,
311
+ table_name: "test_table",
312
+ type: :alter_table,
313
+ respect_order: true,
314
+ src_pos: "mysql-bin.000048\t800",
315
+ table_rev: 2, # increment revision
316
+ seq: 2,
317
+ column: {:name=>"sum"},
318
+ })
319
+ end
320
+ end
301
321
 
302
322
  context 'when received event with another database name' do
303
323
  it do
@@ -331,28 +351,28 @@ EOT
331
351
  end
332
352
 
333
353
  it 'logs a warning and emits FET with a blank binlog file name, when it receives an insert event' do
334
- table_seq_file.should_receive(:write).exactly(3).with(2)
354
+ expect(table_seq_file).to receive(:write).exactly(3).with(2)
335
355
  expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
336
356
  expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "",
337
357
  [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
338
358
  end
339
359
 
340
360
  it 'logs a warning and emits FET with a blank binlog file name, when it receives an update event' do
341
- table_seq_file.should_receive(:write).twice.with(2)
361
+ expect(table_seq_file).to receive(:write).twice.with(2)
342
362
  expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
343
363
  expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "",
344
364
  [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
345
365
  end
346
366
 
347
367
  it 'logs a warning emits FET with a blank binlog file name, when it receives a delete event' do
348
- table_seq_file.should_receive(:write).twice.with(2)
368
+ expect(table_seq_file).to receive(:write).twice.with(2)
349
369
  expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
350
370
  expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "",
351
371
  [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
352
372
  end
353
373
 
354
374
  it 'logs warning once when it receives consecutive events' do
355
- table_seq_file.should_receive(:write).exactly(10).with(2)
375
+ expect(table_seq_file).to receive(:write).exactly(10).with(2)
356
376
  expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
357
377
  expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "",
358
378
  [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
@@ -373,7 +393,7 @@ EOT
373
393
  end
374
394
 
375
395
  it 'logs a warning and emits FET with a blank binlog file name, when it receives an insert event' do
376
- table_seq_file.should_receive(:write).exactly(3).with(2)
396
+ expect(table_seq_file).to receive(:write).exactly(3).with(2)
377
397
  expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
378
398
  expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "",
379
399
  [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
@@ -398,7 +418,7 @@ EOT
398
418
  it 'deletes the per-table-binlog file (if there is one) as soon as a record with bigger binlog position is received' do
399
419
  event = insert_event
400
420
  event['next_position'] = 1250
401
- table_seq_file.should_receive(:write).exactly(3).with(2)
421
+ expect(table_seq_file).to receive(:write).exactly(3).with(2)
402
422
  expect(FileUtils).to receive(:rm).with(/test_table.binlog.pos/, :force => true)
403
423
  expect_emitted_records_with_rows(event, :insert, TEST_TABLE, 1211, "mysql-bin.000048",
404
424
  [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
@@ -9,24 +9,24 @@ module Mysql
9
9
  let(:pos5) { BinLogPosition.new('mysql-bin.000064 1107') }
10
10
 
11
11
  it 'should respond to greater than or equal to operator' do
12
- pos1.respond_to?('>=').should be_true
12
+ expect(pos1.respond_to?('>=')).to be_truthy
13
13
  end
14
14
 
15
15
  context 'when testing greater than or equal to operator' do
16
16
  it 'should return true when it is compared with another object with smaller position value' do
17
- (pos2 >= pos1).should be_true
17
+ expect(pos2 >= pos1).to be_truthy
18
18
  end
19
19
 
20
20
  it 'should return false when it is compared with another object with bigger binlogfile value' do
21
- (pos2 >= pos3).should be_false
21
+ expect(pos2 >= pos3).to be_falsey
22
22
  end
23
23
 
24
24
  it 'should compare the position as an integer (and not string)' do
25
- (pos2 >= pos4).should be_false
25
+ expect(pos2 >= pos4).to be_falsey
26
26
  end
27
27
 
28
28
  it 'should return true when it is compared with another object with equal value' do
29
- (pos4 >= pos5).should be_true
29
+ expect(pos4 >= pos5).to be_truthy
30
30
  end
31
31
  end
32
32
  end
@@ -0,0 +1,52 @@
1
+ require_relative '../../../../spec_helper'
2
+
3
+ describe Mysql::Parser::AlterTableAddColumnParser do
4
+ describe '#parse' do
5
+ let(:query) { "" }
6
+ subject { described_class.new.parse(query) }
7
+
8
+ context 'with normal alter table query' do
9
+ let(:query) { "alter table test_table add column value varchar(26)" }
10
+ it do
11
+ expect(subject).to eq([{
12
+ :subtype => :add_column,
13
+ :table_name => "test_table",
14
+ :column => {
15
+ :name => "value",
16
+ :type => "varchar(78)"
17
+ }
18
+ }])
19
+ end
20
+ end
21
+
22
+ context 'with after option' do
23
+ let(:query) { "alter table test_table add column value varchar(26) after id" }
24
+ it do
25
+ expect(subject).to eq([{
26
+ :subtype => :add_column,
27
+ :table_name => "test_table",
28
+ :column => {
29
+ :name => "value",
30
+ :type => "varchar(78)",
31
+ :after => "id"
32
+ }
33
+ }])
34
+ end
35
+ end
36
+
37
+ context 'with first option' do
38
+ let(:query) { "alter table test_table add column value varchar(26) first" }
39
+ it do
40
+ expect(subject).to eq([{
41
+ :subtype => :add_column,
42
+ :table_name => "test_table",
43
+ :column => {
44
+ :name => "value",
45
+ :type => "varchar(78)",
46
+ :position => :first
47
+ }
48
+ }])
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,68 @@
1
+ require_relative '../../../../spec_helper'
2
+
3
+ describe Mysql::Parser::AlterTableDropColumnParser do
4
+ describe '#parse' do
5
+ let(:query) { "" }
6
+ subject { described_class.new.parse(query) }
7
+
8
+ context 'with normal alter table query' do
9
+ let(:query) { "alter table test_table drop column value" }
10
+ it do
11
+ expect(subject).to eq(
12
+ :subtype => :drop_column,
13
+ :table_name => "test_table",
14
+ :column => { :name => "value" })
15
+ end
16
+ end
17
+
18
+ context 'without column keyword' do
19
+ let(:query) { "alter table test_table drop value" }
20
+ it do
21
+ expect(subject).to eq(
22
+ :subtype => :drop_column,
23
+ :table_name => "test_table",
24
+ :column => { :name => "value" })
25
+ end
26
+ end
27
+
28
+ context 'when column_name is wrapped with backquote' do
29
+ let(:query) { "alter table test_table drop `value`" }
30
+ it do
31
+ expect(subject).to eq(
32
+ :subtype => :drop_column,
33
+ :table_name => "test_table",
34
+ :column => { :name => "value" })
35
+ end
36
+ end
37
+
38
+ context 'when table_name is wrapped with backquote' do
39
+ let(:query) { "alter table `test_table` drop value" }
40
+ it do
41
+ expect(subject).to eq(
42
+ :subtype => :drop_column,
43
+ :table_name => "test_table",
44
+ :column => { :name => "value" })
45
+ end
46
+ end
47
+
48
+ context 'with primary key drop' do
49
+ let(:query) { "alter table test_table drop PRIMARY KEY" }
50
+ it { expect(subject).to be_nil }
51
+ end
52
+
53
+ context 'with index drop' do
54
+ let(:query) { "alter table test_table drop Index index_name" }
55
+ it { expect(subject).to be_nil }
56
+ end
57
+
58
+ context 'with key drop' do
59
+ let(:query) { "alter table test_table drop key index_name" }
60
+ it { expect(subject).to be_nil }
61
+ end
62
+
63
+ context 'with foreign key drop' do
64
+ let(:query) { "alter table test_table drop foreign key foreign_key_name" }
65
+ it { expect(subject).to be_nil }
66
+ end
67
+ end
68
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.9
4
+ version: 0.1.10
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-07-01 00:00:00.000000000 Z
12
+ date: 2014-07-15 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rest-client
@@ -208,17 +208,17 @@ dependencies:
208
208
  requirement: !ruby/object:Gem::Requirement
209
209
  none: false
210
210
  requirements:
211
- - - ! '>='
211
+ - - ~>
212
212
  - !ruby/object:Gem::Version
213
- version: '0'
213
+ version: '3.0'
214
214
  type: :development
215
215
  prerelease: false
216
216
  version_requirements: !ruby/object:Gem::Requirement
217
217
  none: false
218
218
  requirements:
219
- - - ! '>='
219
+ - - ~>
220
220
  - !ruby/object:Gem::Version
221
- version: '0'
221
+ version: '3.0'
222
222
  - !ruby/object:Gem::Dependency
223
223
  name: timecop
224
224
  requirement: !ruby/object:Gem::Requirement
@@ -367,7 +367,10 @@ files:
367
367
  - lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb
368
368
  - lib/flydata/fluent-plugins/mysql/context.rb
369
369
  - lib/flydata/fluent-plugins/mysql/dml_record_handler.rb
370
- - lib/flydata/fluent-plugins/mysql/query_parser.rb
370
+ - lib/flydata/fluent-plugins/mysql/parser.rb
371
+ - lib/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser.rb
372
+ - lib/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser.rb
373
+ - lib/flydata/fluent-plugins/mysql/parser/query_parser.rb
371
374
  - lib/flydata/fluent-plugins/out_forward_ssl.rb
372
375
  - lib/flydata/fluent-plugins/preference.rb
373
376
  - lib/flydata/flydata_crontab.sh
@@ -391,7 +394,8 @@ files:
391
394
  - spec/flydata/command/sync_spec.rb
392
395
  - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
393
396
  - spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb
394
- - spec/flydata/fluent-plugins/mysql/query_parser_spec.rb
397
+ - spec/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser_spec.rb
398
+ - spec/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser_spec.rb
395
399
  - spec/flydata/heroku_spec.rb
396
400
  - spec/flydata/table_def/mysql_table_def_spec.rb
397
401
  - spec/flydata/table_def/mysqldump_test_foreign_key.dump
@@ -421,7 +425,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
421
425
  version: '0'
422
426
  segments:
423
427
  - 0
424
- hash: 4366584009775605873
428
+ hash: 1050886461710559963
425
429
  required_rubygems_version: !ruby/object:Gem::Requirement
426
430
  none: false
427
431
  requirements:
@@ -1,54 +0,0 @@
1
- require_relative '../../../spec_helper'
2
-
3
- module Mysql
4
- describe AlterTableAddColumnParser do
5
- describe '#parse' do
6
- let(:query) { "" }
7
- subject { AlterTableAddColumnParser.new.parse(query) }
8
-
9
- context 'with normal alter table query' do
10
- let(:query) { "alter table test_table add column value varchar(26)" }
11
- it do
12
- expect(subject).to eq([{
13
- :subtype => :add_column,
14
- :table_name => "test_table",
15
- :column => {
16
- :name => "value",
17
- :type => "varchar(78)"
18
- }
19
- }])
20
- end
21
- end
22
-
23
- context 'with after option' do
24
- let(:query) { "alter table test_table add column value varchar(26) after id" }
25
- it do
26
- expect(subject).to eq([{
27
- :subtype => :add_column,
28
- :table_name => "test_table",
29
- :column => {
30
- :name => "value",
31
- :type => "varchar(78)",
32
- :after => "id"
33
- }
34
- }])
35
- end
36
- end
37
-
38
- context 'with first option' do
39
- let(:query) { "alter table test_table add column value varchar(26) first" }
40
- it do
41
- expect(subject).to eq([{
42
- :subtype => :add_column,
43
- :table_name => "test_table",
44
- :column => {
45
- :name => "value",
46
- :type => "varchar(78)",
47
- :position => :first
48
- }
49
- }])
50
- end
51
- end
52
- end
53
- end
54
- end