flydata 0.6.11 → 0.6.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/Gemfile.lock +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/postgresql/source_pos.rb +34 -0
- data/flydata-core/lib/flydata-core/table_def/base.rb +10 -0
- data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb +20 -4
- data/flydata-core/spec/postgresql/source_pos_spec.rb +43 -0
- data/flydata-core/spec/table_def/base_spec.rb +51 -0
- data/flydata.gemspec +0 -0
- data/lib/flydata/command/sender.rb +9 -6
- data/lib/flydata/command/setup.rb +6 -12
- data/lib/flydata/command/sync.rb +31 -17
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb +2 -3
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +15 -14
- data/lib/flydata/parser/source_table.rb +4 -3
- data/lib/flydata/plugin_support/context.rb +46 -0
- data/lib/flydata/plugin_support/sync_record_emittable.rb +69 -0
- data/lib/flydata/source/component.rb +1 -1
- data/lib/flydata/source/generate_source_dump.rb +3 -2
- data/lib/flydata/source_mysql/mysql_compatibility_check.rb +12 -11
- data/lib/flydata/source_mysql/parser/dump_parser.rb +0 -4
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/alter_table_query_handler.rb +8 -2
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_position_file.rb +7 -1
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_dispatcher.rb +10 -4
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_handler.rb +8 -2
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_record_dispatcher.rb +9 -3
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_record_handler.rb +16 -34
- data/lib/flydata/source_mysql/plugin_support/context.rb +7 -0
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/ddl_query_handler.rb +11 -19
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/dml_record_handler.rb +8 -2
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/drop_database_query_handler.rb +8 -2
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/table_meta.rb +5 -1
- data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/truncate_table_query_handler.rb +8 -2
- data/lib/flydata/source_postgresql/generate_source_dump.rb +175 -0
- data/lib/flydata/source_postgresql/parse_dump_and_send.rb +126 -0
- data/lib/flydata/source_postgresql/pg_client.rb +43 -0
- data/lib/flydata/source_postgresql/postgresql_component.rb +12 -0
- data/lib/flydata/source_postgresql/setup.rb +24 -0
- data/lib/flydata/source_postgresql/source_pos.rb +18 -0
- data/lib/flydata/source_postgresql/sync_generate_table_ddl.rb +7 -15
- data/lib/flydata/sync_file_manager.rb +39 -28
- data/spec/flydata/command/setup_spec.rb +0 -1
- data/spec/flydata/command/sync_spec.rb +2 -2
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +5 -6
- data/spec/flydata/plugin_support/context_spec.rb +27 -0
- data/spec/flydata/source_mysql/parser/dump_parser_spec.rb +4 -4
- data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/alter_table_query_handler_spec.rb +3 -3
- data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_dispatcher_spec.rb +5 -5
- data/spec/flydata/source_mysql/plugin_support/context_spec.rb +26 -0
- data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/ddl_query_handler_spec.rb +3 -3
- data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/dml_record_handler_spec.rb +2 -2
- data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/drop_database_query_handler_spec.rb +3 -3
- data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/shared_query_handler_context.rb +3 -1
- data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/table_meta_spec.rb +3 -3
- data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/truncate_query_handler_spec.rb +7 -4
- data/spec/flydata/source_postgresql/generate_source_dump_spec.rb +144 -0
- data/spec/flydata/sync_file_manager_spec.rb +1 -1
- metadata +38 -24
- data/lib/flydata/fluent-plugins/mysql/context.rb +0 -25
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'flydata/source/source_pos'
|
2
|
+
require 'flydata-core/postgresql/source_pos'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module SourcePostgresql
|
6
|
+
|
7
|
+
class SourcePos < Source::SourcePos
|
8
|
+
def create_source_pos(source_pos_str)
|
9
|
+
FlydataCore::Postgresql::SourcePos.load(source_pos_str)
|
10
|
+
end
|
11
|
+
|
12
|
+
def resume_pos(source_pos)
|
13
|
+
source_pos
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'flydata/source/sync_generate_table_ddl'
|
2
2
|
require 'flydata-core/table_def/postgresql_table_def'
|
3
|
-
require '
|
3
|
+
require 'flydata/source_postgresql/pg_client'
|
4
4
|
|
5
5
|
module Flydata
|
6
6
|
module SourcePostgresql
|
@@ -19,14 +19,6 @@ class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def each_source_tabledef(tables, options, &block)
|
22
|
-
pg_opts = {
|
23
|
-
host: options['host'],
|
24
|
-
port: options['port'],
|
25
|
-
dbname: options['database'],
|
26
|
-
user: options['username'],
|
27
|
-
password: options['password'],
|
28
|
-
sslmode: :prefer,
|
29
|
-
}
|
30
22
|
# PostgreSQL options.
|
31
23
|
tables = tables.clone
|
32
24
|
missing_tables = []
|
@@ -34,7 +26,7 @@ class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
|
|
34
26
|
if tables.to_s == '' || tables.to_s == '[]'
|
35
27
|
raise ArgumentError, "tables is nil or empty"
|
36
28
|
end
|
37
|
-
_each_tabledef(tables, options,
|
29
|
+
_each_tabledef(tables, options, &block)
|
38
30
|
rescue TableMissingError => e
|
39
31
|
tables.delete e.table
|
40
32
|
missing_tables << e.table
|
@@ -60,13 +52,13 @@ ORDER BY c.table_name, c.ordinal_position;
|
|
60
52
|
EOS
|
61
53
|
TABLE_PLACEHOLDER_START_NUM = 2 # because $1 is used by table_schema
|
62
54
|
|
63
|
-
def _each_tabledef(tables, options,
|
64
|
-
cli =
|
55
|
+
def _each_tabledef(tables, options, &block)
|
56
|
+
cli = PGClient.new(options)
|
65
57
|
|
66
58
|
# TODO call the query for every 50 tables
|
67
|
-
|
68
|
-
|
69
|
-
|
59
|
+
res = cli.query(COLUMNS_QUERY, [options['schema']] + tables,
|
60
|
+
placeholder_start_num: TABLE_PLACEHOLDER_START_NUM,
|
61
|
+
placeholder_size: tables.size)
|
70
62
|
|
71
63
|
create_opt = {}
|
72
64
|
if options.has_key?(:skip_primary_key_check)
|
@@ -10,13 +10,11 @@ module Flydata
|
|
10
10
|
DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
|
11
11
|
BACKUP_DIR = ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
|
12
12
|
TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
|
13
|
-
SYNC_TABLE_POSITIONS = 0
|
14
13
|
|
15
14
|
def initialize(data_entry, source = nil)
|
16
15
|
@data_entry = data_entry
|
17
16
|
@source = source #for Source dependent objects
|
18
17
|
@table_position_files = {} # File objects keyed by table name
|
19
|
-
@sync_table_positions_count = SYNC_TABLE_POSITIONS
|
20
18
|
end
|
21
19
|
|
22
20
|
def source
|
@@ -38,6 +36,7 @@ module Flydata
|
|
38
36
|
end
|
39
37
|
|
40
38
|
def save_dump_pos(status, table_name, last_pos, source_pos, state = nil, substate = nil)
|
39
|
+
raise "Cannot create dump pos file because source position is unavailable." unless source_pos
|
41
40
|
File.open(dump_pos_path, 'w') do |f|
|
42
41
|
f.write(dump_pos_content(status, table_name, last_pos, source_pos, state, substate))
|
43
42
|
end
|
@@ -205,11 +204,28 @@ module Flydata
|
|
205
204
|
tables.map{|table| File.join(table_positions_dir_path, table + '.binlog.pos.init')}
|
206
205
|
end
|
207
206
|
|
207
|
+
def increment_table_position(seq)
|
208
|
+
seq = seq.to_i + 1
|
209
|
+
seq = FlydataCore::QueryJob::SYNC_FIRST_SEQ if seq == 1
|
210
|
+
seq
|
211
|
+
end
|
212
|
+
|
208
213
|
# Read a sequence number from the table's position file,
|
209
214
|
# increment the number and pass the number to a block.
|
210
215
|
# After executing the block, saves the value to the position
|
211
216
|
# file.
|
212
217
|
def increment_and_save_table_position(table_name)
|
218
|
+
seq = get_table_position(table_name)
|
219
|
+
|
220
|
+
seq = increment_table_position(seq)
|
221
|
+
|
222
|
+
# logical transaction starts
|
223
|
+
yield(seq)
|
224
|
+
save_table_position(table_name, seq)
|
225
|
+
# logical transaction ends
|
226
|
+
end
|
227
|
+
|
228
|
+
def open_table_position_file(table_name)
|
213
229
|
file = File.join(table_positions_dir_path, table_name + ".pos")
|
214
230
|
retry_count = 0
|
215
231
|
begin
|
@@ -221,43 +237,34 @@ module Flydata
|
|
221
237
|
retry_count += 1
|
222
238
|
retry
|
223
239
|
end
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
new_seq_len = seq.to_s.size
|
240
|
+
@table_position_files[table_name]
|
241
|
+
end
|
242
|
+
|
243
|
+
def save_table_position(table_name, seq)
|
244
|
+
f = open_table_position_file(table_name)
|
245
|
+
prev_seq_len = f.size
|
231
246
|
|
232
247
|
seq_to_write = seq.to_s
|
248
|
+
new_seq_len = seq_to_write.size
|
249
|
+
|
233
250
|
if new_seq_len < prev_seq_len
|
234
251
|
seq_to_write += " " * (prev_seq_len - new_seq_len)
|
235
252
|
end
|
236
|
-
# logical transaction starts
|
237
|
-
yield(seq)
|
238
253
|
f.write(seq_to_write)
|
239
|
-
if @sync_table_positions_count > 1
|
240
|
-
@sync_table_positions_count -= 1
|
241
|
-
elsif @sync_table_positions_count == 1
|
242
|
-
f.fsync
|
243
|
-
@sync_table_positions_count = SYNC_TABLE_POSITIONS
|
244
|
-
end
|
245
|
-
# logical transaction ends
|
246
254
|
f.truncate(new_seq_len) if new_seq_len < prev_seq_len
|
247
255
|
f.rewind
|
248
256
|
end
|
249
257
|
|
250
|
-
def save_table_position(table_name, seq)
|
251
|
-
file = File.join(table_positions_dir_path, table_name + ".pos")
|
252
|
-
|
253
|
-
File.open(file, "w") {|f| f.write(seq)}
|
254
|
-
end
|
255
|
-
|
256
258
|
def get_table_position(table_name)
|
257
|
-
|
258
|
-
|
259
|
+
f = open_table_position_file(table_name)
|
260
|
+
seq = f.read
|
261
|
+
f.rewind
|
262
|
+
seq
|
263
|
+
end
|
259
264
|
|
260
|
-
|
265
|
+
def get_next_table_position(table_name)
|
266
|
+
seq = get_table_position(table_name)
|
267
|
+
increment_table_position(seq)
|
261
268
|
end
|
262
269
|
|
263
270
|
def lock_pid_file
|
@@ -477,7 +484,11 @@ module Flydata
|
|
477
484
|
status = items[0]
|
478
485
|
table_name = items[1]
|
479
486
|
last_pos = items[2].to_i
|
480
|
-
|
487
|
+
if status == Command::Sync::STATUS_START
|
488
|
+
#Use only source_pos information. Discard and initialize other items
|
489
|
+
table_name = nil
|
490
|
+
last_pos = nil
|
491
|
+
end
|
481
492
|
source_pos_str = items[3..-3].join("\t")
|
482
493
|
context = source.source_pos
|
483
494
|
source_pos = context.create_source_pos(source_pos_str)
|
@@ -25,7 +25,6 @@ module Flydata
|
|
25
25
|
expect(credentials).to receive(:authenticated?).and_return(false)
|
26
26
|
allow(data_port).to receive(:get).and_return(dp)
|
27
27
|
expect(sender).to receive(:process_exist?).and_return(true)
|
28
|
-
expect(sender).to receive(:stop)
|
29
28
|
allow(sync_fm).to receive(:source_pos_path).and_return('/tmp')
|
30
29
|
expect(conf).to receive(:copy_templates)
|
31
30
|
expect(login).to receive(:run)
|
@@ -101,7 +101,7 @@ module Flydata
|
|
101
101
|
allow(File).to receive(:exists?).and_return(false)
|
102
102
|
expect(default_sync_fm).to receive(:load_dump_pos).and_return(default_dump_pos)
|
103
103
|
expect(default_sync_fm).to receive(:dump_file_path).and_return(default_fp)
|
104
|
-
expect(default_sync_fm).to receive(:backup_dir).and_return(default_backup)
|
104
|
+
expect(default_sync_fm).to receive(:backup_dir).and_return(default_backup).at_least(:once)
|
105
105
|
expect(subject).to receive(:target_tables).and_return(target_tables).at_least(:once)
|
106
106
|
expect(subject).to receive(:print)
|
107
107
|
expect(subject).to receive(:log_info)
|
@@ -145,7 +145,7 @@ module Flydata
|
|
145
145
|
let(:values) { [4, 'John', nil, col4_value, nil, nil] }
|
146
146
|
|
147
147
|
let(:source_table) do
|
148
|
-
Flydata::Parser::SourceTable.new("test_table", mysql_table_columns
|
148
|
+
Flydata::Parser::SourceTable.new("test_table", mysql_table_columns)
|
149
149
|
end
|
150
150
|
|
151
151
|
before do
|
@@ -150,9 +150,8 @@ EOT
|
|
150
150
|
|
151
151
|
def expect_emitted_records(event, records)
|
152
152
|
records = [records] unless records.kind_of?(Array)
|
153
|
-
|
154
|
-
|
155
|
-
end
|
153
|
+
expect(Engine).to receive(:emit_array).
|
154
|
+
with(TEST_TAG, records.collect{|r| [TEST_TIMESTAMP, r] })
|
156
155
|
plugin.event_listener(event)
|
157
156
|
end
|
158
157
|
|
@@ -235,10 +234,10 @@ EOT
|
|
235
234
|
cleanup_flydata_files
|
236
235
|
setup_initial_flydata_files
|
237
236
|
allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
|
238
|
-
allow(
|
239
|
-
allow(
|
237
|
+
allow(Flydata::SourceMysql::PluginSupport::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
|
238
|
+
allow(Flydata::SourceMysql::PluginSupport::BinLogPositionFile).to receive(:new).with(TEST_SENT_POSITION_FILE).
|
240
239
|
and_return(sent_position_file)
|
241
|
-
allow(Flydata::
|
240
|
+
allow(Flydata::SourceMysql::PluginSupport::TableMeta).to receive(:update)
|
242
241
|
allow(plugin).to receive(:`).with("mysql -V").and_return("mysql Ver 14.14 Distrib 5.5.40")
|
243
242
|
allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
|
244
243
|
Timecop.freeze(now)
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'fluent_plugins_spec_helper'
|
2
|
+
require 'flydata/plugin_support/context'
|
3
|
+
|
4
|
+
describe ::Flydata::PluginSupport::Context do
|
5
|
+
let(:subject_object) { described_class.new(params) }
|
6
|
+
|
7
|
+
describe '#initialize' do
|
8
|
+
subject { subject_object }
|
9
|
+
|
10
|
+
context 'when no missing params' do
|
11
|
+
let(:params) { {
|
12
|
+
tables: %w(table_a table_b table_c),
|
13
|
+
tag: 'test_tag',
|
14
|
+
sync_fm: 'test_sync_fm',
|
15
|
+
omit_events: {'table_c' => %i(delete truncate_table)},
|
16
|
+
table_revs: {'table_a' => 1, 'table_b' => 2, 'table_c' => 3},
|
17
|
+
} }
|
18
|
+
|
19
|
+
it { expect{subject}.not_to raise_error }
|
20
|
+
it { expect(subject.tables).to eq(%w(table_a table_b table_c)) }
|
21
|
+
it { expect(subject.tag).to eq('test_tag') }
|
22
|
+
it { expect(subject.sync_fm).to eq('test_sync_fm') }
|
23
|
+
it { expect(subject.omit_events).to eq({'table_c' => %i(delete truncate_table)}) }
|
24
|
+
it { expect(subject.table_revs).to eq({'table_a' => 1, 'table_b' => 2, 'table_c' => 3}) }
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -225,7 +225,7 @@ EOT
|
|
225
225
|
}
|
226
226
|
expect(check_point_block).to receive(:call) { |source_table, last_pos, bytesize, binlog_pos, state, substate|
|
227
227
|
expect(source_table.table_name).to eq('users_login')
|
228
|
-
expect(source_table.columns.keys).to eq([col_name, 'login_count', 'create_time', 'update_time'])
|
228
|
+
expect(source_table.instance_variable_get(:@columns).keys).to eq([col_name, 'login_count', 'create_time', 'update_time'])
|
229
229
|
expect(last_pos).to eq(index_after(dump_content, 'ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\'\';'))
|
230
230
|
expect(binlog_pos).to eq(default_binlog_pos)
|
231
231
|
expect(state).to eq(Flydata::Parser::State::INSERT_RECORD)
|
@@ -306,7 +306,7 @@ EOT
|
|
306
306
|
# create_table_block
|
307
307
|
expect(create_table_block).to receive(:call) { |source_table|
|
308
308
|
expect(source_table.table_name).to eq('users_login')
|
309
|
-
expect(source_table.columns.keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
|
309
|
+
expect(source_table.instance_variable_get(:@columns).keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
|
310
310
|
}.once
|
311
311
|
expect(create_table_block).to receive(:call).never
|
312
312
|
|
@@ -412,7 +412,7 @@ EOT
|
|
412
412
|
# create_table_block
|
413
413
|
expect(create_table_block).to receive(:call) { |source_table|
|
414
414
|
expect(source_table.table_name).to eq('users_login')
|
415
|
-
expect(source_table.columns.keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
|
415
|
+
expect(source_table.instance_variable_get(:@columns).keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
|
416
416
|
}.once
|
417
417
|
expect(create_table_block).to receive(:call).never
|
418
418
|
|
@@ -715,7 +715,7 @@ EOT
|
|
715
715
|
# create_table_block
|
716
716
|
expect(create_table_block).to receive(:call) { |source_table|
|
717
717
|
expect(source_table.table_name).to eq('users_login')
|
718
|
-
expect(source_table.columns.keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
|
718
|
+
expect(source_table.instance_variable_get(:@columns).keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
|
719
719
|
}.once
|
720
720
|
expect(create_table_block).to receive(:call).never
|
721
721
|
|
@@ -1,8 +1,8 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
|
-
require 'flydata/
|
3
|
-
require 'flydata/
|
2
|
+
require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
|
3
|
+
require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
|
4
4
|
|
5
|
-
module
|
5
|
+
module Flydata::SourceMysql::PluginSupport
|
6
6
|
describe AlterTableQueryHandler do
|
7
7
|
include_context "query handler context"
|
8
8
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
|
-
require 'flydata/
|
2
|
+
require 'flydata/source_mysql/plugin_support/binlog_query_dispatcher'
|
3
3
|
|
4
|
-
module
|
4
|
+
module Flydata::SourceMysql::PluginSupport
|
5
5
|
|
6
6
|
QUERY_WITH_COMMENTS = <<EOT
|
7
7
|
-- ALTER TABLE QUERY comment
|
@@ -51,17 +51,17 @@ EOT
|
|
51
51
|
end
|
52
52
|
let(:alter_query_handler) do
|
53
53
|
r = double('alter_query_handler')
|
54
|
-
allow(r).to receive(:pattern).and_return(
|
54
|
+
allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::AlterTableQueryHandler::PATTERN)
|
55
55
|
r
|
56
56
|
end
|
57
57
|
let(:truncate_query_handler) do
|
58
58
|
r = double('truncate_query_handler')
|
59
|
-
allow(r).to receive(:pattern).and_return(
|
59
|
+
allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::TruncateTableQueryHandler::PATTERN)
|
60
60
|
r
|
61
61
|
end
|
62
62
|
let(:drop_database_query_handler) do
|
63
63
|
r = double('drop_database_query_handler')
|
64
|
-
allow(r).to receive(:pattern).and_return(
|
64
|
+
allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropDatabaseQueryHandler::PATTERN)
|
65
65
|
r
|
66
66
|
end
|
67
67
|
let(:context) { double('context') }
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'fluent_plugins_spec_helper'
|
2
|
+
require 'flydata/source_mysql/plugin_support/context'
|
3
|
+
|
4
|
+
describe ::Flydata::SourceMysql::PluginSupport::Context do
|
5
|
+
let(:subject_object) { described_class.new(params) }
|
6
|
+
|
7
|
+
describe '#initialize' do
|
8
|
+
subject { subject_object }
|
9
|
+
|
10
|
+
context 'when no missing params' do
|
11
|
+
let(:params) { {
|
12
|
+
tables: %w(table_a table_b table_c),
|
13
|
+
tag: 'test_tag',
|
14
|
+
sync_fm: 'test_sync_fm',
|
15
|
+
omit_events: {'table_c' => %i(delete truncate_table)},
|
16
|
+
table_revs: {'table_a' => 1, 'table_b' => 2, 'table_c' => 3},
|
17
|
+
database: 'test_db',
|
18
|
+
table_meta: 'test_table_meta',
|
19
|
+
} }
|
20
|
+
|
21
|
+
it { expect{subject}.not_to raise_error }
|
22
|
+
it { expect(subject.database).to eq('test_db') }
|
23
|
+
it { expect(subject.table_meta).to eq('test_table_meta') }
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/ddl_query_handler_spec.rb
RENAMED
@@ -1,8 +1,8 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
|
-
require 'flydata/
|
3
|
-
require 'flydata/
|
2
|
+
require 'flydata/source_mysql/plugin_support/ddl_query_handler'
|
3
|
+
require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
|
4
4
|
|
5
|
-
module
|
5
|
+
module Flydata::SourceMysql::PluginSupport
|
6
6
|
describe DdlQueryHandler do
|
7
7
|
include_context "query handler context"
|
8
8
|
|
data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/dml_record_handler_spec.rb
RENAMED
@@ -1,9 +1,9 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
3
|
require 'fluent_plugins_spec_helper'
|
4
|
-
require 'flydata/
|
4
|
+
require 'flydata/source_mysql/plugin_support/dml_record_handler'
|
5
5
|
|
6
|
-
module
|
6
|
+
module Flydata::SourceMysql::PluginSupport
|
7
7
|
|
8
8
|
describe DmlRecordHandler do
|
9
9
|
let(:subject_object) { described_class.new(context) }
|
@@ -1,8 +1,8 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
|
-
require 'flydata/
|
3
|
-
require 'flydata/
|
2
|
+
require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
|
3
|
+
require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
|
4
4
|
|
5
|
-
module
|
5
|
+
module Flydata::SourceMysql::PluginSupport
|
6
6
|
describe DropDatabaseQueryHandler do
|
7
7
|
let(:subject_object) { described_class.new(context) }
|
8
8
|
include_context "query handler context"
|
@@ -1,4 +1,4 @@
|
|
1
|
-
module
|
1
|
+
module Flydata::SourceMysql::PluginSupport
|
2
2
|
shared_context "query handler context" do
|
3
3
|
let(:target_database) { "testdb" }
|
4
4
|
let(:database) { target_database }
|
@@ -10,6 +10,8 @@ module Mysql
|
|
10
10
|
r = double('sync_fm')
|
11
11
|
allow(r).to receive(:get_table_source_raw_pos).and_return(binlog_pos_string)
|
12
12
|
allow(r).to receive(:increment_and_save_table_position).with(table).and_yield(seq).and_return(nil)
|
13
|
+
allow(r).to receive(:get_table_position).and_return(seq - 1)
|
14
|
+
allow(r).to receive(:increment_table_position).and_return(seq)
|
13
15
|
allow(r).to receive(:delete_table_source_pos).with(table)
|
14
16
|
r
|
15
17
|
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
|
-
require 'flydata/
|
2
|
+
require 'flydata/source_mysql/plugin_support/table_meta'
|
3
3
|
|
4
|
-
module Flydata
|
5
|
-
module
|
4
|
+
module Flydata::SourceMysql
|
5
|
+
module PluginSupport
|
6
6
|
describe TableMeta do
|
7
7
|
let(:db_opts) { { host: 'test-host', port: 3306, username: 'test-user', password: 'test-pswd', database: 'test-db' } }
|
8
8
|
let(:database) { 'test-db' }
|
@@ -1,8 +1,8 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
|
-
require 'flydata/
|
3
|
-
require 'flydata/
|
2
|
+
require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
|
3
|
+
require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
|
4
4
|
|
5
|
-
module
|
5
|
+
module Flydata::SourceMysql::PluginSupport
|
6
6
|
describe TruncateTableQueryHandler do
|
7
7
|
include_context "query handler context"
|
8
8
|
|
@@ -31,7 +31,10 @@ module Mysql
|
|
31
31
|
|
32
32
|
context "for a non append only table" do
|
33
33
|
it "should call Fluent's emit with appropriate params" do
|
34
|
-
expect(Fluent::Engine).to receive(:
|
34
|
+
expect(Fluent::Engine).to receive(:emit_array).
|
35
|
+
with(tag, [[timestamp, expected_record]])
|
36
|
+
expect(sync_fm).to receive(:save_table_position).with(table, seq)
|
37
|
+
|
35
38
|
expect(subject.process(record))
|
36
39
|
end
|
37
40
|
end
|