flydata 0.6.11 → 0.6.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -1
  3. data/Gemfile.lock +4 -4
  4. data/VERSION +1 -1
  5. data/flydata-core/lib/flydata-core/postgresql/source_pos.rb +34 -0
  6. data/flydata-core/lib/flydata-core/table_def/base.rb +10 -0
  7. data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb +20 -4
  8. data/flydata-core/spec/postgresql/source_pos_spec.rb +43 -0
  9. data/flydata-core/spec/table_def/base_spec.rb +51 -0
  10. data/flydata.gemspec +0 -0
  11. data/lib/flydata/command/sender.rb +9 -6
  12. data/lib/flydata/command/setup.rb +6 -12
  13. data/lib/flydata/command/sync.rb +31 -17
  14. data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb +2 -3
  15. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +15 -14
  16. data/lib/flydata/parser/source_table.rb +4 -3
  17. data/lib/flydata/plugin_support/context.rb +46 -0
  18. data/lib/flydata/plugin_support/sync_record_emittable.rb +69 -0
  19. data/lib/flydata/source/component.rb +1 -1
  20. data/lib/flydata/source/generate_source_dump.rb +3 -2
  21. data/lib/flydata/source_mysql/mysql_compatibility_check.rb +12 -11
  22. data/lib/flydata/source_mysql/parser/dump_parser.rb +0 -4
  23. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/alter_table_query_handler.rb +8 -2
  24. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_position_file.rb +7 -1
  25. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_dispatcher.rb +10 -4
  26. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_handler.rb +8 -2
  27. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_record_dispatcher.rb +9 -3
  28. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_record_handler.rb +16 -34
  29. data/lib/flydata/source_mysql/plugin_support/context.rb +7 -0
  30. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/ddl_query_handler.rb +11 -19
  31. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/dml_record_handler.rb +8 -2
  32. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/drop_database_query_handler.rb +8 -2
  33. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/table_meta.rb +5 -1
  34. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/truncate_table_query_handler.rb +8 -2
  35. data/lib/flydata/source_postgresql/generate_source_dump.rb +175 -0
  36. data/lib/flydata/source_postgresql/parse_dump_and_send.rb +126 -0
  37. data/lib/flydata/source_postgresql/pg_client.rb +43 -0
  38. data/lib/flydata/source_postgresql/postgresql_component.rb +12 -0
  39. data/lib/flydata/source_postgresql/setup.rb +24 -0
  40. data/lib/flydata/source_postgresql/source_pos.rb +18 -0
  41. data/lib/flydata/source_postgresql/sync_generate_table_ddl.rb +7 -15
  42. data/lib/flydata/sync_file_manager.rb +39 -28
  43. data/spec/flydata/command/setup_spec.rb +0 -1
  44. data/spec/flydata/command/sync_spec.rb +2 -2
  45. data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +5 -6
  46. data/spec/flydata/plugin_support/context_spec.rb +27 -0
  47. data/spec/flydata/source_mysql/parser/dump_parser_spec.rb +4 -4
  48. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/alter_table_query_handler_spec.rb +3 -3
  49. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_dispatcher_spec.rb +5 -5
  50. data/spec/flydata/source_mysql/plugin_support/context_spec.rb +26 -0
  51. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/ddl_query_handler_spec.rb +3 -3
  52. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/dml_record_handler_spec.rb +2 -2
  53. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/drop_database_query_handler_spec.rb +3 -3
  54. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/shared_query_handler_context.rb +3 -1
  55. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/table_meta_spec.rb +3 -3
  56. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/truncate_query_handler_spec.rb +7 -4
  57. data/spec/flydata/source_postgresql/generate_source_dump_spec.rb +144 -0
  58. data/spec/flydata/sync_file_manager_spec.rb +1 -1
  59. metadata +38 -24
  60. data/lib/flydata/fluent-plugins/mysql/context.rb +0 -25
@@ -0,0 +1,18 @@
1
+ require 'flydata/source/source_pos'
2
+ require 'flydata-core/postgresql/source_pos'
3
+
4
+ module Flydata
5
+ module SourcePostgresql
6
+
7
+ class SourcePos < Source::SourcePos
8
+ def create_source_pos(source_pos_str)
9
+ FlydataCore::Postgresql::SourcePos.load(source_pos_str)
10
+ end
11
+
12
+ def resume_pos(source_pos)
13
+ source_pos
14
+ end
15
+ end
16
+
17
+ end
18
+ end
@@ -1,6 +1,6 @@
1
1
  require 'flydata/source/sync_generate_table_ddl'
2
2
  require 'flydata-core/table_def/postgresql_table_def'
3
- require 'pg'
3
+ require 'flydata/source_postgresql/pg_client'
4
4
 
5
5
  module Flydata
6
6
  module SourcePostgresql
@@ -19,14 +19,6 @@ class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
19
19
  end
20
20
 
21
21
  def each_source_tabledef(tables, options, &block)
22
- pg_opts = {
23
- host: options['host'],
24
- port: options['port'],
25
- dbname: options['database'],
26
- user: options['username'],
27
- password: options['password'],
28
- sslmode: :prefer,
29
- }
30
22
  # PostgreSQL options.
31
23
  tables = tables.clone
32
24
  missing_tables = []
@@ -34,7 +26,7 @@ class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
34
26
  if tables.to_s == '' || tables.to_s == '[]'
35
27
  raise ArgumentError, "tables is nil or empty"
36
28
  end
37
- _each_tabledef(tables, options, pg_opts, &block)
29
+ _each_tabledef(tables, options, &block)
38
30
  rescue TableMissingError => e
39
31
  tables.delete e.table
40
32
  missing_tables << e.table
@@ -60,13 +52,13 @@ ORDER BY c.table_name, c.ordinal_position;
60
52
  EOS
61
53
  TABLE_PLACEHOLDER_START_NUM = 2 # because $1 is used by table_schema
62
54
 
63
- def _each_tabledef(tables, options, pg_opts, &block)
64
- cli = PG::Connection.new(pg_opts)
55
+ def _each_tabledef(tables, options, &block)
56
+ cli = PGClient.new(options)
65
57
 
66
58
  # TODO call the query for every 50 tables
67
- placeholders = placeholder_string(tables.size, TABLE_PLACEHOLDER_START_NUM)
68
- query = COLUMNS_QUERY % [placeholders]
69
- res = cli.query(query, [options['schema']] + tables)
59
+ res = cli.query(COLUMNS_QUERY, [options['schema']] + tables,
60
+ placeholder_start_num: TABLE_PLACEHOLDER_START_NUM,
61
+ placeholder_size: tables.size)
70
62
 
71
63
  create_opt = {}
72
64
  if options.has_key?(:skip_primary_key_check)
@@ -10,13 +10,11 @@ module Flydata
10
10
  DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
11
11
  BACKUP_DIR = ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
12
12
  TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
13
- SYNC_TABLE_POSITIONS = 0
14
13
 
15
14
  def initialize(data_entry, source = nil)
16
15
  @data_entry = data_entry
17
16
  @source = source #for Source dependent objects
18
17
  @table_position_files = {} # File objects keyed by table name
19
- @sync_table_positions_count = SYNC_TABLE_POSITIONS
20
18
  end
21
19
 
22
20
  def source
@@ -38,6 +36,7 @@ module Flydata
38
36
  end
39
37
 
40
38
  def save_dump_pos(status, table_name, last_pos, source_pos, state = nil, substate = nil)
39
+ raise "Cannot create dump pos file because source position is unavailable." unless source_pos
41
40
  File.open(dump_pos_path, 'w') do |f|
42
41
  f.write(dump_pos_content(status, table_name, last_pos, source_pos, state, substate))
43
42
  end
@@ -205,11 +204,28 @@ module Flydata
205
204
  tables.map{|table| File.join(table_positions_dir_path, table + '.binlog.pos.init')}
206
205
  end
207
206
 
207
+ def increment_table_position(seq)
208
+ seq = seq.to_i + 1
209
+ seq = FlydataCore::QueryJob::SYNC_FIRST_SEQ if seq == 1
210
+ seq
211
+ end
212
+
208
213
  # Read a sequence number from the table's position file,
209
214
  # increment the number and pass the number to a block.
210
215
  # After executing the block, saves the value to the position
211
216
  # file.
212
217
  def increment_and_save_table_position(table_name)
218
+ seq = get_table_position(table_name)
219
+
220
+ seq = increment_table_position(seq)
221
+
222
+ # logical transaction starts
223
+ yield(seq)
224
+ save_table_position(table_name, seq)
225
+ # logical transaction ends
226
+ end
227
+
228
+ def open_table_position_file(table_name)
213
229
  file = File.join(table_positions_dir_path, table_name + ".pos")
214
230
  retry_count = 0
215
231
  begin
@@ -221,43 +237,34 @@ module Flydata
221
237
  retry_count += 1
222
238
  retry
223
239
  end
224
- f = @table_position_files[table_name]
225
- seq = f.read
226
- prev_seq_len = seq.size
227
- f.rewind
228
- seq = seq.to_i + 1
229
- seq = FlydataCore::QueryJob::SYNC_FIRST_SEQ if seq == 1
230
- new_seq_len = seq.to_s.size
240
+ @table_position_files[table_name]
241
+ end
242
+
243
+ def save_table_position(table_name, seq)
244
+ f = open_table_position_file(table_name)
245
+ prev_seq_len = f.size
231
246
 
232
247
  seq_to_write = seq.to_s
248
+ new_seq_len = seq_to_write.size
249
+
233
250
  if new_seq_len < prev_seq_len
234
251
  seq_to_write += " " * (prev_seq_len - new_seq_len)
235
252
  end
236
- # logical transaction starts
237
- yield(seq)
238
253
  f.write(seq_to_write)
239
- if @sync_table_positions_count > 1
240
- @sync_table_positions_count -= 1
241
- elsif @sync_table_positions_count == 1
242
- f.fsync
243
- @sync_table_positions_count = SYNC_TABLE_POSITIONS
244
- end
245
- # logical transaction ends
246
254
  f.truncate(new_seq_len) if new_seq_len < prev_seq_len
247
255
  f.rewind
248
256
  end
249
257
 
250
- def save_table_position(table_name, seq)
251
- file = File.join(table_positions_dir_path, table_name + ".pos")
252
-
253
- File.open(file, "w") {|f| f.write(seq)}
254
- end
255
-
256
258
  def get_table_position(table_name)
257
- file = File.join(table_positions_dir_path, table_name + ".pos")
258
- return nil unless File.exists?(file)
259
+ f = open_table_position_file(table_name)
260
+ seq = f.read
261
+ f.rewind
262
+ seq
263
+ end
259
264
 
260
- File.open(file) {|f| f.read}
265
+ def get_next_table_position(table_name)
266
+ seq = get_table_position(table_name)
267
+ increment_table_position(seq)
261
268
  end
262
269
 
263
270
  def lock_pid_file
@@ -477,7 +484,11 @@ module Flydata
477
484
  status = items[0]
478
485
  table_name = items[1]
479
486
  last_pos = items[2].to_i
480
-
487
+ if status == Command::Sync::STATUS_START
488
+ #Use only source_pos information. Discard and initialize other items
489
+ table_name = nil
490
+ last_pos = nil
491
+ end
481
492
  source_pos_str = items[3..-3].join("\t")
482
493
  context = source.source_pos
483
494
  source_pos = context.create_source_pos(source_pos_str)
@@ -25,7 +25,6 @@ module Flydata
25
25
  expect(credentials).to receive(:authenticated?).and_return(false)
26
26
  allow(data_port).to receive(:get).and_return(dp)
27
27
  expect(sender).to receive(:process_exist?).and_return(true)
28
- expect(sender).to receive(:stop)
29
28
  allow(sync_fm).to receive(:source_pos_path).and_return('/tmp')
30
29
  expect(conf).to receive(:copy_templates)
31
30
  expect(login).to receive(:run)
@@ -101,7 +101,7 @@ module Flydata
101
101
  allow(File).to receive(:exists?).and_return(false)
102
102
  expect(default_sync_fm).to receive(:load_dump_pos).and_return(default_dump_pos)
103
103
  expect(default_sync_fm).to receive(:dump_file_path).and_return(default_fp)
104
- expect(default_sync_fm).to receive(:backup_dir).and_return(default_backup)
104
+ expect(default_sync_fm).to receive(:backup_dir).and_return(default_backup).at_least(:once)
105
105
  expect(subject).to receive(:target_tables).and_return(target_tables).at_least(:once)
106
106
  expect(subject).to receive(:print)
107
107
  expect(subject).to receive(:log_info)
@@ -145,7 +145,7 @@ module Flydata
145
145
  let(:values) { [4, 'John', nil, col4_value, nil, nil] }
146
146
 
147
147
  let(:source_table) do
148
- Flydata::Parser::SourceTable.new("test_table", mysql_table_columns, ['id'])
148
+ Flydata::Parser::SourceTable.new("test_table", mysql_table_columns)
149
149
  end
150
150
 
151
151
  before do
@@ -150,9 +150,8 @@ EOT
150
150
 
151
151
  def expect_emitted_records(event, records)
152
152
  records = [records] unless records.kind_of?(Array)
153
- records.each do |r|
154
- expect(Engine).to receive(:emit).with(TEST_TAG, TEST_TIMESTAMP, r).ordered
155
- end
153
+ expect(Engine).to receive(:emit_array).
154
+ with(TEST_TAG, records.collect{|r| [TEST_TIMESTAMP, r] })
156
155
  plugin.event_listener(event)
157
156
  end
158
157
 
@@ -235,10 +234,10 @@ EOT
235
234
  cleanup_flydata_files
236
235
  setup_initial_flydata_files
237
236
  allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
238
- allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
239
- allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_SENT_POSITION_FILE).
237
+ allow(Flydata::SourceMysql::PluginSupport::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
238
+ allow(Flydata::SourceMysql::PluginSupport::BinLogPositionFile).to receive(:new).with(TEST_SENT_POSITION_FILE).
240
239
  and_return(sent_position_file)
241
- allow(Flydata::Mysql::TableMeta).to receive(:update)
240
+ allow(Flydata::SourceMysql::PluginSupport::TableMeta).to receive(:update)
242
241
  allow(plugin).to receive(:`).with("mysql -V").and_return("mysql Ver 14.14 Distrib 5.5.40")
243
242
  allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
244
243
  Timecop.freeze(now)
@@ -0,0 +1,27 @@
1
+ require 'fluent_plugins_spec_helper'
2
+ require 'flydata/plugin_support/context'
3
+
4
+ describe ::Flydata::PluginSupport::Context do
5
+ let(:subject_object) { described_class.new(params) }
6
+
7
+ describe '#initialize' do
8
+ subject { subject_object }
9
+
10
+ context 'when no missing params' do
11
+ let(:params) { {
12
+ tables: %w(table_a table_b table_c),
13
+ tag: 'test_tag',
14
+ sync_fm: 'test_sync_fm',
15
+ omit_events: {'table_c' => %i(delete truncate_table)},
16
+ table_revs: {'table_a' => 1, 'table_b' => 2, 'table_c' => 3},
17
+ } }
18
+
19
+ it { expect{subject}.not_to raise_error }
20
+ it { expect(subject.tables).to eq(%w(table_a table_b table_c)) }
21
+ it { expect(subject.tag).to eq('test_tag') }
22
+ it { expect(subject.sync_fm).to eq('test_sync_fm') }
23
+ it { expect(subject.omit_events).to eq({'table_c' => %i(delete truncate_table)}) }
24
+ it { expect(subject.table_revs).to eq({'table_a' => 1, 'table_b' => 2, 'table_c' => 3}) }
25
+ end
26
+ end
27
+ end
@@ -225,7 +225,7 @@ EOT
225
225
  }
226
226
  expect(check_point_block).to receive(:call) { |source_table, last_pos, bytesize, binlog_pos, state, substate|
227
227
  expect(source_table.table_name).to eq('users_login')
228
- expect(source_table.columns.keys).to eq([col_name, 'login_count', 'create_time', 'update_time'])
228
+ expect(source_table.instance_variable_get(:@columns).keys).to eq([col_name, 'login_count', 'create_time', 'update_time'])
229
229
  expect(last_pos).to eq(index_after(dump_content, 'ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\'\';'))
230
230
  expect(binlog_pos).to eq(default_binlog_pos)
231
231
  expect(state).to eq(Flydata::Parser::State::INSERT_RECORD)
@@ -306,7 +306,7 @@ EOT
306
306
  # create_table_block
307
307
  expect(create_table_block).to receive(:call) { |source_table|
308
308
  expect(source_table.table_name).to eq('users_login')
309
- expect(source_table.columns.keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
309
+ expect(source_table.instance_variable_get(:@columns).keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
310
310
  }.once
311
311
  expect(create_table_block).to receive(:call).never
312
312
 
@@ -412,7 +412,7 @@ EOT
412
412
  # create_table_block
413
413
  expect(create_table_block).to receive(:call) { |source_table|
414
414
  expect(source_table.table_name).to eq('users_login')
415
- expect(source_table.columns.keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
415
+ expect(source_table.instance_variable_get(:@columns).keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
416
416
  }.once
417
417
  expect(create_table_block).to receive(:call).never
418
418
 
@@ -715,7 +715,7 @@ EOT
715
715
  # create_table_block
716
716
  expect(create_table_block).to receive(:call) { |source_table|
717
717
  expect(source_table.table_name).to eq('users_login')
718
- expect(source_table.columns.keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
718
+ expect(source_table.instance_variable_get(:@columns).keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
719
719
  }.once
720
720
  expect(create_table_block).to receive(:call).never
721
721
 
@@ -1,8 +1,8 @@
1
1
  require 'fluent_plugins_spec_helper'
2
- require 'flydata/fluent-plugins/mysql/alter_table_query_handler'
3
- require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
2
+ require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
3
+ require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
4
4
 
5
- module Mysql
5
+ module Flydata::SourceMysql::PluginSupport
6
6
  describe AlterTableQueryHandler do
7
7
  include_context "query handler context"
8
8
 
@@ -1,7 +1,7 @@
1
1
  require 'fluent_plugins_spec_helper'
2
- require 'flydata/fluent-plugins/mysql/binlog_query_dispatcher'
2
+ require 'flydata/source_mysql/plugin_support/binlog_query_dispatcher'
3
3
 
4
- module Mysql
4
+ module Flydata::SourceMysql::PluginSupport
5
5
 
6
6
  QUERY_WITH_COMMENTS = <<EOT
7
7
  -- ALTER TABLE QUERY comment
@@ -51,17 +51,17 @@ EOT
51
51
  end
52
52
  let(:alter_query_handler) do
53
53
  r = double('alter_query_handler')
54
- allow(r).to receive(:pattern).and_return(Mysql::AlterTableQueryHandler::PATTERN)
54
+ allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::AlterTableQueryHandler::PATTERN)
55
55
  r
56
56
  end
57
57
  let(:truncate_query_handler) do
58
58
  r = double('truncate_query_handler')
59
- allow(r).to receive(:pattern).and_return(Mysql::TruncateTableQueryHandler::PATTERN)
59
+ allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::TruncateTableQueryHandler::PATTERN)
60
60
  r
61
61
  end
62
62
  let(:drop_database_query_handler) do
63
63
  r = double('drop_database_query_handler')
64
- allow(r).to receive(:pattern).and_return(Mysql::DropDatabaseQueryHandler::PATTERN)
64
+ allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropDatabaseQueryHandler::PATTERN)
65
65
  r
66
66
  end
67
67
  let(:context) { double('context') }
@@ -0,0 +1,26 @@
1
+ require 'fluent_plugins_spec_helper'
2
+ require 'flydata/source_mysql/plugin_support/context'
3
+
4
+ describe ::Flydata::SourceMysql::PluginSupport::Context do
5
+ let(:subject_object) { described_class.new(params) }
6
+
7
+ describe '#initialize' do
8
+ subject { subject_object }
9
+
10
+ context 'when no missing params' do
11
+ let(:params) { {
12
+ tables: %w(table_a table_b table_c),
13
+ tag: 'test_tag',
14
+ sync_fm: 'test_sync_fm',
15
+ omit_events: {'table_c' => %i(delete truncate_table)},
16
+ table_revs: {'table_a' => 1, 'table_b' => 2, 'table_c' => 3},
17
+ database: 'test_db',
18
+ table_meta: 'test_table_meta',
19
+ } }
20
+
21
+ it { expect{subject}.not_to raise_error }
22
+ it { expect(subject.database).to eq('test_db') }
23
+ it { expect(subject.table_meta).to eq('test_table_meta') }
24
+ end
25
+ end
26
+ end
@@ -1,8 +1,8 @@
1
1
  require 'fluent_plugins_spec_helper'
2
- require 'flydata/fluent-plugins/mysql/ddl_query_handler'
3
- require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
2
+ require 'flydata/source_mysql/plugin_support/ddl_query_handler'
3
+ require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
4
4
 
5
- module Mysql
5
+ module Flydata::SourceMysql::PluginSupport
6
6
  describe DdlQueryHandler do
7
7
  include_context "query handler context"
8
8
 
@@ -1,9 +1,9 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  require 'fluent_plugins_spec_helper'
4
- require 'flydata/fluent-plugins/mysql/dml_record_handler'
4
+ require 'flydata/source_mysql/plugin_support/dml_record_handler'
5
5
 
6
- module Mysql
6
+ module Flydata::SourceMysql::PluginSupport
7
7
 
8
8
  describe DmlRecordHandler do
9
9
  let(:subject_object) { described_class.new(context) }
@@ -1,8 +1,8 @@
1
1
  require 'fluent_plugins_spec_helper'
2
- require 'flydata/fluent-plugins/mysql/drop_database_query_handler'
3
- require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
2
+ require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
3
+ require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
4
4
 
5
- module Mysql
5
+ module Flydata::SourceMysql::PluginSupport
6
6
  describe DropDatabaseQueryHandler do
7
7
  let(:subject_object) { described_class.new(context) }
8
8
  include_context "query handler context"
@@ -1,4 +1,4 @@
1
- module Mysql
1
+ module Flydata::SourceMysql::PluginSupport
2
2
  shared_context "query handler context" do
3
3
  let(:target_database) { "testdb" }
4
4
  let(:database) { target_database }
@@ -10,6 +10,8 @@ module Mysql
10
10
  r = double('sync_fm')
11
11
  allow(r).to receive(:get_table_source_raw_pos).and_return(binlog_pos_string)
12
12
  allow(r).to receive(:increment_and_save_table_position).with(table).and_yield(seq).and_return(nil)
13
+ allow(r).to receive(:get_table_position).and_return(seq - 1)
14
+ allow(r).to receive(:increment_table_position).and_return(seq)
13
15
  allow(r).to receive(:delete_table_source_pos).with(table)
14
16
  r
15
17
  end
@@ -1,8 +1,8 @@
1
1
  require 'fluent_plugins_spec_helper'
2
- require 'flydata/fluent-plugins/mysql/table_meta'
2
+ require 'flydata/source_mysql/plugin_support/table_meta'
3
3
 
4
- module Flydata
5
- module Mysql
4
+ module Flydata::SourceMysql
5
+ module PluginSupport
6
6
  describe TableMeta do
7
7
  let(:db_opts) { { host: 'test-host', port: 3306, username: 'test-user', password: 'test-pswd', database: 'test-db' } }
8
8
  let(:database) { 'test-db' }
@@ -1,8 +1,8 @@
1
1
  require 'fluent_plugins_spec_helper'
2
- require 'flydata/fluent-plugins/mysql/truncate_table_query_handler'
3
- require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
2
+ require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
3
+ require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
4
4
 
5
- module Mysql
5
+ module Flydata::SourceMysql::PluginSupport
6
6
  describe TruncateTableQueryHandler do
7
7
  include_context "query handler context"
8
8
 
@@ -31,7 +31,10 @@ module Mysql
31
31
 
32
32
  context "for a non append only table" do
33
33
  it "should call Fluent's emit with appropriate params" do
34
- expect(Fluent::Engine).to receive(:emit).with(tag, timestamp, expected_record)
34
+ expect(Fluent::Engine).to receive(:emit_array).
35
+ with(tag, [[timestamp, expected_record]])
36
+ expect(sync_fm).to receive(:save_table_position).with(table, seq)
37
+
35
38
  expect(subject.process(record))
36
39
  end
37
40
  end