flydata 0.6.14 → 0.7.0

data/lib/flydata/table_meta.rb

@@ -0,0 +1,19 @@
+module Flydata
+  class TableMeta
+    def initialize(opts)
+      @table_meta = {}
+    end
+
+    # Override
+    # Reload table meta info
+    #def reload(db_client)
+    #end
+
+    # Return table meta
+    #   :character_set_name
+    def [](table_name)
+      @table_meta[table_name.to_sym]
+    end
+  end
+end
+

data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_context.rb

@@ -0,0 +1,85 @@
+require 'timecop'
+require 'fileutils'
+
+module Fluent
+  FLYDATA_HOME = Flydata::FLYDATA_HOME
+  TABLE_POSITIONS_DIR = Flydata::FLYDATA_TABLE_POSITIONS_DIR
+  TEST_TAG = "test_tag"
+  TEST_DB = "test_db"
+  TEST_TABLE = "test_table"
+  TEST_SEQUENCE_FILE = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.pos")
+  TEST_SEQUENCE_NUM = 1
+  TEST_TABLE_BINLOG_POS = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.binlog.pos")
+  TEST_TABLES = "#{TEST_TABLE},test_table_1,test_table_2,test_table_3"
+  TEST_POSITION_FILE = "test_position.binlog.pos"
+  TEST_SENT_POSITION_FILE = "test_position.binlog.sent.pos"
+  TEST_REVISION_FILE = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.rev")
+  TEST_TIMESTAMP = 1389214083
+  TEST_TABLE_APPEND_ONLY = "test_table_4"
+  TEST_SEQUENCE_FILE_1 = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE_APPEND_ONLY}.pos")
+
+  shared_context 'flydata sync context' do
+    def create_file(file_path, content)
+      File.open(file_path, 'w') {|f| f.write(content)}
+    end
+
+    def delete_file(file_path)
+      FileUtils.rm(file_path) if File.exists?(file_path)
+    end
+
+    def create_binlog_pos_file(content = '')
+      create_file(TEST_POSITION_FILE, content)
+    end
+
+    def create_table_pos_file(tables, content = '')
+      create_table_file(tables, 'pos', content)
+    end
+
+    def create_table_rev_file(tables, content = '')
+      create_table_file(tables, 'rev', content)
+    end
+
+    def create_table_file(tables, ext, content = '')
+      tables = Array(tables)
+      tables.each do |table|
+        create_file(File.join(FLYDATA_HOME, 'positions', "#{table}.#{ext}"), content)
+      end
+    end
+
+    def config_value(variable_name, a_plugin = plugin)
+      a_plugin.instance_variable_get(variable_name.to_s)
+    end
+
+    def replace_conf(conf_str, conf_name, new_value)
+      if conf_str.include?(conf_name)
+        conf_str.gsub(/#{conf_name} .*$/, "#{conf_name} #{new_value}")
+      else
+        conf_str += "\n  #{conf_name} #{new_value}\n"
+      end
+    end
+
+    def setup_initial_flydata_dirs
+      %w(positions dump conf).each{|f| FileUtils.mkdir_p(File.join(FLYDATA_HOME, f))}
+    end
+
+    def cleanup_flydata_dirs
+      %w(positions dump conf).each{|f| FileUtils.rm_rf(File.join(FLYDATA_HOME, f))}
+    end
+
+    before do
+      cleanup_flydata_dirs
+      setup_initial_flydata_dirs
+
+      @current_dir_path = Dir::pwd
+      Dir.chdir(FLYDATA_HOME)
+      create_file(TEST_POSITION_FILE, '')
+    end
+
+    after do
+      delete_file(TEST_POSITION_FILE)
+      Dir.chdir(@current_dir_path)
+
+      cleanup_flydata_dirs
+    end
+  end
+end

data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based_shared_examples.rb

@@ -0,0 +1,36 @@
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_shared_examples'
+
+
+module Fluent
+
+shared_examples 'flydata sync diff based shared examples' do
+  include_examples 'flydata sync shared examples'
+
+  describe '#configure' do
+    let(:config) do
+      c = default_config
+      c = replace_conf(c, 'tables', 'table_1')
+      c = replace_conf(c, 'fetch_interval', '10m')
+      c = replace_conf(c, 'retry_interval', '5m')
+      c = replace_conf(c, 'emit_chunk_limit', '32m')
+      c
+    end
+
+    subject { Test.configure_plugin(plugin, config) }
+
+    before do
+      create_table_pos_file('table_1')
+      subject
+    end
+
+    it { expect(config_value(:@fetch_interval)).to eq(10 * 60) }
+    it { expect(config_value(:@retry_interval)).to eq(5 * 60) }
+    it { expect(config_value(:@emit_chunk_limit)).to eq(32 * 1024**2) }
+    it do
+      expect(config_value(:@table_src_pos_files).size).to eq(1)
+      expect(config_value(:@table_src_pos_files)[:table_1]).to be_kind_of(described_class::SOURCE_POSITION_FILE_CLASS)
+    end
+  end
+end
+
+end

data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based_shared_examples.rb

@@ -0,0 +1,37 @@
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based_shared_examples'
+
+
+module Fluent
+
+shared_examples 'flydata sync query based shared examples' do
+  include_examples 'flydata sync diff based shared examples'
+
+  describe '#configure' do
+    let(:config) do
+      c = default_config
+      c = replace_conf(c, 'host', 'test-host')
+      c = replace_conf(c, 'port', '5555')
+      c = replace_conf(c, 'username', 'test-user')
+      c = replace_conf(c, 'password', 'test-pass')
+      c = replace_conf(c, 'database', 'test-db')
+      c = replace_conf(c, 'schema', 'test-schema')
+      c
+    end
+
+    subject { Test.configure_plugin(plugin, config) }
+
+    before do
+      subject
+    end
+
+    it { expect(config_value(:@host)).to eq('test-host') }
+    it { expect(config_value(:@port)).to eq(5555) }
+    it { expect(config_value(:@username)).to eq('test-user') }
+    it { expect(config_value(:@password)).to eq('test-pass') }
+    it { expect(config_value(:@database)).to eq('test-db') }
+    it { expect(config_value(:@schema)).to eq('test-schema') }
+  end
+
+end
+
+end

data/spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_shared_examples.rb

@@ -0,0 +1,67 @@
+require 'timecop'
+require 'fileutils'
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_context'
+
+module Fluent
+
+# plugin and config is required
+shared_examples 'flydata sync shared examples' do
+  include_context 'flydata sync context'
+
+  describe '#configure' do
+    let(:config) do
+      c = default_config
+      c = replace_conf(c, 'tables', 'table_1,table_2')
+      c = replace_conf(c, 'tables_append_only', 'table_3,table_4')
+      c
+    end
+
+    subject { Test.configure_plugin(plugin, config) }
+
+    before do
+      create_table_pos_file(%w(table_1 table_2 table_3))
+      create_table_rev_file('table_1', '2')
+      subject
+    end
+
+    it { expect(config_value(:@data_entry_name)).to be_kind_of(String) }
+    it { expect(config_value(:@data_entry_type)).to be_kind_of(String) }
+    it { expect(config_value(:@tag)).to be_kind_of(String) }
+    it { expect(config_value(:@position_file)).to be_kind_of(String) }
+
+    it 'sets @source_position_file' do
+      expect(config_value(:@source_position_file)).
+        to be_kind_of(described_class::SOURCE_POSITION_FILE_CLASS)
+    end
+
+    it 'sets @sent_position_file' do
+      expect(config_value(:@sent_position_file)).
+        to be_kind_of(described_class::SOURCE_POSITION_FILE_CLASS)
+    end
+
+    it { expect(config_value(:@data_entry)).to have_key('name') }
+    it { expect(config_value(:@data_entry)).to have_key('type') }
+    it { expect(config_value(:@tables)).to be_kind_of(Array) }
+
+    it do
+      expect(config_value(:@tables)).to eq(%w(table_1 table_2 table_3))
+    end
+
+    it do
+      expect(config_value(:@omit_events)).to eq(
+        'table_3' => [:delete, :truncate_table],
+        'table_4' => [:delete, :truncate_table],
+      )
+    end
+
+    it do
+      expect(config_value(:@table_revs)).to eq(
+        'table_1' => 2,
+        'table_2' => 1,
+        'table_3' => 1,
+      )
+    end
+  end
+end
+
+end

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -2,40 +2,49 @@
 
 require 'fluent_plugins_spec_helper'
 require 'in_mysql_binlog_flydata'
-require 'timecop'
+
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_shared_examples'
 
 module Fluent
 
-  FLYDATA_HOME = Flydata::FLYDATA_HOME
-  TEST_TAG = "test_tag"
-  TEST_DB = "test_db"
-  TEST_TABLE = "test_table"
-  #TEST_SEQUENCE_FILE = /positions\/#{TEST_TABLE}.pos$/
-  TEST_SEQUENCE_FILE = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.pos")
-  TEST_SEQUENCE_NUM = 1
-  TEST_TABLE_BINLOG_POS = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.binlog.pos")
-  TEST_TABLES = "#{TEST_TABLE},test_table_1,test_table_2,test_table_3"
-  TEST_POSITION_FILE = "test_position.binlog.pos"
-  TEST_SENT_POSITION_FILE = "test_position.binlog.sent.pos"
-  TEST_REVISION_FILE = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.rev")
-  TEST_TIMESTAMP = 1389214083
-  TEST_TABLE_APPEND_ONLY = "test_table_4"
-  TEST_SEQUENCE_FILE_1 = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE_APPEND_ONLY}.pos")
-  TEST_CONFIG = <<EOT
+  describe MysqlBinlogFlydataInput do
+    let(:plugin) { described_class.new }
+    let(:default_config) { <<EOT
+  data_entry_name test_name
+  data_entry_type RedshiftMysqlDataEntry
+  tag #{TEST_TAG}
+  database #{TEST_DB}
+  tables test_table,test_table_1,test_table_2
+  tables_append_only test_table_3
+  position_file #{TEST_POSITION_FILE}
+  host localhost
+  port 3306
+  username test_admin
+  password test_password
+  database test_db
+  ssl_ca_content
+  ssl_cipher
+EOT
+    }
+    include_examples 'flydata sync shared examples'
+
+    TEST_CONFIG = <<EOT
+  data_entry_name test_name
+  data_entry_type RedshiftMysqlDataEntry
   tag #{TEST_TAG}
   database #{TEST_DB}
   tables #{TEST_TABLES}
   tables_append_only
   position_file #{TEST_POSITION_FILE}
 EOT
-  TEST_TABLES_APPEND_ONLY_CONFIG = <<EOT
+    TEST_TABLES_APPEND_ONLY_CONFIG = <<EOT
   tag #{TEST_TAG}
   database #{TEST_DB}
   tables #{TEST_TABLES}
   tables_append_only #{TEST_TABLE_APPEND_ONLY}
   position_file #{TEST_POSITION_FILE}
 EOT
-  TEST_TABLES_DUPLICATE_CONFIG = <<EOT
+    TEST_TABLES_DUPLICATE_CONFIG = <<EOT
   tag #{TEST_TAG}
   database #{TEST_DB}
   tables #{TEST_TABLES},#{TEST_TABLE_APPEND_ONLY}
@@ -43,105 +52,104 @@ EOT
   position_file #{TEST_POSITION_FILE}
 EOT
 
-  # mysqlbinlog event
-  #   http://dev.mysql.com/doc/refman/5.7/en/mysqlbinlog-hexdump.html
+    # mysqlbinlog event
+    #   http://dev.mysql.com/doc/refman/5.7/en/mysqlbinlog-hexdump.html
 
-  # Supported events
-  # - 17 WRITE_ROWS_EVENT
-  TEST_EVENT_INSERT=<<EOT
+    # Supported events
+    # - 17 WRITE_ROWS_EVENT
+    TEST_EVENT_INSERT=<<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>23, "server_id"=>1, "event_length"=>39, "next_position"=>667, "flags"=>1, "event_type"=>"Write_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 3, 0, 0, 0, 3, 0, 102, 111, 111], "rows"=>[["1", "foo"],["2","var"],["3","hoge"]]}
 EOT
-  TEST_EVENT_TWO_BYTE_INSERT=<<EOT
+    TEST_EVENT_TWO_BYTE_INSERT=<<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>23, "server_id"=>1, "event_length"=>39, "next_position"=>667, "flags"=>1, "event_type"=>"Write_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 3, 0, 0, 0, 3, 0, 102, 111, 111], "rows"=>[["1", "føø"],["2","vår"],["3","høgé"]]}
 EOT
-  TEST_EVENT_THREE_BYTE_INSERT=<<EOT
+    TEST_EVENT_THREE_BYTE_INSERT=<<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>23, "server_id"=>1, "event_length"=>39, "next_position"=>667, "flags"=>1, "event_type"=>"Write_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 3, 0, 0, 0, 3, 0, 102, 111, 111], "rows"=>[["1", "富无无"],["2","易变的"],["3","切实切实"]]}
 EOT
-  # - 18 UPDATE_ROWS_EVENT
-  TEST_EVENT_UPDATE = <<EOT
+    # - 18 UPDATE_ROWS_EVENT
+    TEST_EVENT_UPDATE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>24, "server_id"=>1, "event_length"=>78, "next_position"=>2606, "flags"=>1, "event_type"=>"Update_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[255], "raw_used_columns"=>[255], "raw_row"=>[252, 6, 0, 0, 0, 1, 0, 97, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 98, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 99, 252, 6, 0, 0, 0, 1, 0, 100], "rows"=>[[["1", "foo"], ["1", "wow"]],[["3", "hoge"], ["3", "fuga"]]]}
 EOT
-  TEST_EVENT_TWO_BYTE_UPDATE = <<EOT
+    TEST_EVENT_TWO_BYTE_UPDATE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>24, "server_id"=>1, "event_length"=>78, "next_position"=>2606, "flags"=>1, "event_type"=>"Update_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[255], "raw_used_columns"=>[255], "raw_row"=>[252, 6, 0, 0, 0, 1, 0, 97, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 98, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 99, 252, 6, 0, 0, 0, 1, 0, 100], "rows"=>[[["1", "føø"], ["1", "∑ø∑"]],[["3", "høgé"], ["3", "fügå"]]]}
 EOT
-  TEST_EVENT_THREE_BYTE_UPDATE = <<EOT
+    TEST_EVENT_THREE_BYTE_UPDATE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>24, "server_id"=>1, "event_length"=>78, "next_position"=>2606, "flags"=>1, "event_type"=>"Update_rows", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[255], "raw_used_columns"=>[255], "raw_row"=>[252, 6, 0, 0, 0, 1, 0, 97, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 98, 252, 6, 0, 0, 0, 1, 0, 100, 252, 6, 0, 0, 0, 1, 0, 99, 252, 6, 0, 0, 0, 1, 0, 100], "rows"=>[[["1", "富无无"], ["1", "很兴奋"]],[["3", "切实切实"], ["3", "興奮虎"]]]}
 EOT
-  # - 19 DELETE_ROWS_EVENT
-  TEST_EVENT_DELETE = <<EOT
+    # - 19 DELETE_ROWS_EVENT
+    TEST_EVENT_DELETE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>25, "server_id"=>1, "event_length"=>41, "next_position"=>5365, "flags"=>1, "event_type"=>"Delete_rows", "table_id"=>170, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 2, 0, 0, 0, 5, 0, 104, 111, 104, 111, 103], "rows"=>[["2", "var"],["3","hoge"]]}
 EOT
-  TEST_EVENT_TWO_BYTE_DELETE = <<EOT
+    TEST_EVENT_TWO_BYTE_DELETE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>25, "server_id"=>1, "event_length"=>41, "next_position"=>5365, "flags"=>1, "event_type"=>"Delete_rows", "table_id"=>170, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 2, 0, 0, 0, 5, 0, 104, 111, 104, 111, 103], "rows"=>[["2", "vår"],["3","høgé"]]}
 EOT
-  TEST_EVENT_THREE_BYTE_DELETE = <<EOT
+    TEST_EVENT_THREE_BYTE_DELETE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>25, "server_id"=>1, "event_length"=>41, "next_position"=>5365, "flags"=>1, "event_type"=>"Delete_rows", "table_id"=>170, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "columns"=>["LONG", "BLOB"], "columns_len"=>2, "null_bits_len"=>1, "raw_columns_before_image"=>[], "raw_used_columns"=>[255], "raw_row"=>[252, 2, 0, 0, 0, 5, 0, 104, 111, 104, 111, 103], "rows"=>[["2", "易变的"],["3","切实切实"]]}
 EOT
-  # Unsupported event
-  # - 02 QUERY_EVENT
-  # QUERY: BEGIN
-  TEST_EVENT_QUERY_BEGIN = <<EOT
+    # Unsupported event
+    # - 02 QUERY_EVENT
+    # QUERY: BEGIN
+    TEST_EVENT_QUERY_BEGIN = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>73, "next_position"=>2472, "flags"=>8, "event_type"=>"Query", "thread_id"=>71, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"#{TEST_DB}", "query"=>"BEGIN"}
 EOT
-  # QUERY: create database
-  TEST_EVENT_QUERY_CREATE_DATABSE = <<EOT
+    # QUERY: create database
+    TEST_EVENT_QUERY_CREATE_DATABSE = <<EOT
 {"marker"=>0, "timestamp"=>1389309478, "type_code"=>2, "server_id"=>1, "event_length"=>89, "next_position"=>196, "flags"=>8, "event_type"=>"Query", "thread_id"=>39, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"create database test_db"}
 EOT
-  # QUERY: create table
-  TEST_EVENT_QUERY_CREATE_TABLE = <<EOT
+    # QUERY: create table
+    TEST_EVENT_QUERY_CREATE_TABLE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>121, "next_position"=>317, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"create table test_table(id int primary key, value text)"}
 EOT
-  # QUERY: drop table
-  TEST_EVENT_QUERY_DROP_TABLE = <<EOT
+    # QUERY: drop table
+    TEST_EVENT_QUERY_DROP_TABLE = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>115, "next_position"=>568, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"DROP TABLE `test_table` /* generated by server */"}
 EOT
-  # QUERY: alter table add column
-  TEST_EVENT_QUERY_ALTER_TABLE_ADD_COLUMN = <<EOT
+    # QUERY: alter table add column
+    TEST_EVENT_QUERY_ALTER_TABLE_ADD_COLUMN = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>111, "next_position"=>800, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table add column sum integer"}
 EOT
-  # QUERY: alter table drop column
-  TEST_EVENT_QUERY_ALTER_TABLE_DROP_COLUMN = <<EOT
+    # QUERY: alter table drop column
+    TEST_EVENT_QUERY_ALTER_TABLE_DROP_COLUMN = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>104, "next_position"=>904, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table drop column sum"}
 EOT
-  # QUERY: alter table modify column
-  TEST_EVENT_QUERY_ALTER_TABLE_MODIFY_COLUMN = <<EOT
+    # QUERY: alter table modify column
+    TEST_EVENT_QUERY_ALTER_TABLE_MODIFY_COLUMN = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>112, "next_position"=>1352, "flags"=>0, "event_type"=>"Query", "thread_id"=>48, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table modify column sum float"}
 EOT
-  # QUERY: alter table add index
-  TEST_EVENT_QUERY_ALTER_TABLE_ADD_INDEX = <<EOT
+    # QUERY: alter table add index
+    TEST_EVENT_QUERY_ALTER_TABLE_ADD_INDEX = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>217, "next_position"=>337, "flags"=>0, "event_type"=>"Query", "thread_id"=>10097, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 64, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0, 12, 1, 109, 97, 107, 95, 100, 101, 118, 101, 108, 111, 112, 109, 101, 110, 116, 0], "db_name"=>"test_db", "query"=>"ALTER TABLE `test_db`.`test_table` \nADD INDEX `my_index` USING HASH (`extra` ASC) COMMENT 'How does this look in binlog?'"}
 EOT
-  # - 03 STOP_EVENT
-  TEST_EVENT_STOP = <<EOT
+    # - 03 STOP_EVENT
+    TEST_EVENT_STOP = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>3, "server_id"=>1, "event_length"=>19, "next_position"=>2929, "flags"=>0, "event_type"=>"Stop"}
 EOT
-  # - 04 ROTATE_EVENT
-  TEST_EVENT_ROTATE = <<EOT
+    # - 04 ROTATE_EVENT
+    TEST_EVENT_ROTATE = <<EOT
 {"marker"=>0, "timestamp"=>0, "type_code"=>4, "server_id"=>1, "event_length"=>43, "next_position"=>0, "flags"=>32, "event_type"=>"Rotate", "binlog_file"=>"mysql-bin.000048", "binlog_pos"=>2883}
 EOT
-  # - 0f FORMAT_DESCRIPTION_EVENT
-  TEST_EVENT_FORMAT_DESC = <<EOT
+    # - 0f FORMAT_DESCRIPTION_EVENT
+    TEST_EVENT_FORMAT_DESC = <<EOT
 {"marker"=>0, "timestamp"=>1389292075, "type_code"=>15, "server_id"=>1, "event_length"=>103, "next_position"=>0, "flags"=>0, "event_type"=>"Format_desc", "binlog_version"=>0, "created_ts"=>2560145104, "log_header_len"=>71}
 EOT
-  # - 10 XID_EVENT
-  TEST_EVENT_XID = <<EOT
+    # - 10 XID_EVENT
+    TEST_EVENT_XID = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>16, "server_id"=>1, "event_length"=>27, "next_position"=>2633, "flags"=>0, "event_type"=>"Xid", "xid_id"=>129345}
 EOT
 
-  # - 13 TABLE_MAP_EVENT
-  TEST_EVENT_TABLE_MAP = <<EOT
+    # - 13 TABLE_MAP_EVENT
+    TEST_EVENT_TABLE_MAP = <<EOT
 {"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>19, "server_id"=>1, "event_length"=>56, "next_position"=>2528, "flags"=>1, "event_type"=>"Table_map", "table_id"=>163, "db_name"=>"#{TEST_DB}", "table_name"=>"#{TEST_TABLE}", "raw_columns"=>[3, 252], "columns"=>["LONG", "BLOB"], "metadata"=>[2], "null_bits"=>[3]}
 EOT
-  # la
-  TEST_EVENT_INCIDENT = <<EOT
+    # la
+    TEST_EVENT_INCIDENT = <<EOT
 {"marker"=>0, "timestamp"=>0, "type_code"=>26, "server_id"=>1, "event_length"=>40, "next_position"=>2883, "flags"=>32, "event_type"=>"Incident", "incident_type"=>175, "message"=>"Operation canceled"}
 EOT
-  # - 04 ROTATE_EVENT
-  TEST_EVENT_ROTATE_MISSING_BINLOG_FILE = <<EOT
+    # - 04 ROTATE_EVENT
+    TEST_EVENT_ROTATE_MISSING_BINLOG_FILE = <<EOT
 {"marker"=>0, "timestamp"=>0, "type_code"=>4, "server_id"=>1, "event_length"=>43, "next_position"=>0, "flags"=>32, "event_type"=>"Rotate", "binlog_pos"=>2883}
 EOT
 
-  describe MysqlBinlogFlydataInput do
     def create_event(event_str)
       event = eval(event_str)
       allow(event).to receive(:event_type).and_return(event["event_type"])
@@ -173,7 +181,6 @@ EOT
       expect_emitted_records(event, records)
     end
 
-    let(:plugin) { MysqlBinlogFlydataInput.new }
     let(:insert_event) { create_event(TEST_EVENT_INSERT) }
     let(:insert_two_byte_event) { create_event(TEST_EVENT_TWO_BYTE_INSERT) }
     let(:insert_three_byte_event) { create_event(TEST_EVENT_THREE_BYTE_INSERT) }
@@ -196,58 +203,74 @@ EOT
     let(:now) { Time.now }
     let(:flydata_record_version) { 2 }
 
-    let(:binlog_position_file) do
-      f = double('binlog_position_file')
+    let(:source_position_file) do
+      f = double('source_position_file')
       allow(f).to receive(:exists?).and_return(true)
       allow(f).to receive(:read).and_return('test_position')
       f
     end
 
-    let(:sent_position_file) do
-      f = double('sent_position_file')
-      allow(f).to receive(:exists?).and_return(true)
-      allow(f).to receive(:read).and_return('test_sent_position')
-      f
-    end
-
-    def create_file(file_path, content)
-      File.open(file_path, 'w') {|f| f.write(content)}
-    end
-
-    def delete_file(file_path)
-      FileUtils.rm(file_path) if File.exists?(file_path)
-    end
-
-    def setup_initial_flydata_files
-      %w(positions dump conf).each{|f| FileUtils.mkdir_p(File.join(FLYDATA_HOME, f))}
+    def setup_pos_files
       create_file(TEST_SEQUENCE_FILE, TEST_SEQUENCE_NUM.to_s)
       create_file(TEST_SEQUENCE_FILE_1, TEST_SEQUENCE_NUM.to_s)
     end
 
-    def cleanup_flydata_files
-      %w(positions dump conf).each{|f| FileUtils.rm_rf(File.join(FLYDATA_HOME, f))}
+    def cleanup_pos_files
       delete_file(TEST_SEQUENCE_FILE)
       delete_file(TEST_SEQUENCE_FILE_1)
     end
 
     before do
-      cleanup_flydata_files
-      setup_initial_flydata_files
+      cleanup_pos_files
+      setup_pos_files
       allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
-      allow(Flydata::SourceMysql::PluginSupport::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
-      allow(Flydata::SourceMysql::PluginSupport::BinLogPositionFile).to receive(:new).with(TEST_SENT_POSITION_FILE).
-        and_return(sent_position_file)
-      allow(Flydata::SourceMysql::PluginSupport::TableMeta).to receive(:update)
+      allow(Flydata::SourceMysql::TableMeta).to receive(:update)
       allow(plugin).to receive(:`).with("mysql -V").and_return("mysql  Ver 14.14 Distrib 5.5.40")
       allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
       Timecop.freeze(now)
     end
 
     after do
-      cleanup_flydata_files
+      cleanup_pos_files
       Timecop.return
     end
 
+    describe '#configure' do
+      let(:config) { default_config }
+      before do
+        Test.configure_plugin(plugin, config)
+      end
+
+      it 'sets data entry object' do
+        expect(plugin.instance_variable_get(:@data_entry)).to eq(
+          'name' => 'test_name',
+          'type' => 'RedshiftMysqlDataEntry',
+          'mysql_data_entry_preference' => {
+            'database' => 'test_db',
+            'tables' => 'test_table,test_table_1,test_table_2',
+            'tables_append_only' => 'test_table_3',
+            'host' => 'localhost',
+            'username' => 'test_admin',
+            'password' => 'test_password',
+            'ssl_ca_content' => '',
+            'ssl_cipher' => '',
+          }
+        )
+      end
+
+      it 'sets data tables as array' do
+        expect(plugin.instance_variable_get(:@tables)).to eq(%w(test_table))
+      end
+
+      it 'sets sync_fm' do
+        expect(plugin.instance_variable_get(:@sync_fm)).to be_kind_of(Flydata::SyncFileManager)
+      end
+
+      it 'sets omit_events' do
+        expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table]})
+      end
+    end
+
     describe '#event_listener' do
 
       context 'when received rotate event' do