flydata 0.6.11 → 0.6.12

data/lib/flydata/source_postgresql/source_pos.rb

@@ -0,0 +1,18 @@
+require 'flydata/source/source_pos'
+require 'flydata-core/postgresql/source_pos'
+
+module Flydata
+module SourcePostgresql
+
+class SourcePos < Source::SourcePos
+  def create_source_pos(source_pos_str)
+    FlydataCore::Postgresql::SourcePos.load(source_pos_str)
+  end
+
+  def resume_pos(source_pos)
+    source_pos
+  end
+end
+
+end
+end

data/lib/flydata/source_postgresql/sync_generate_table_ddl.rb

@@ -1,6 +1,6 @@
 require 'flydata/source/sync_generate_table_ddl'
 require 'flydata-core/table_def/postgresql_table_def'
-require 'pg'
+require 'flydata/source_postgresql/pg_client'
 
 module Flydata
 module SourcePostgresql
@@ -19,14 +19,6 @@ class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
   end
 
   def each_source_tabledef(tables, options, &block)
-    pg_opts = {
-      host: options['host'],
-      port: options['port'],
-      dbname: options['database'],
-      user: options['username'],
-      password: options['password'],
-      sslmode: :prefer,
-    }
     # PostgreSQL options.
     tables = tables.clone
     missing_tables = []
@@ -34,7 +26,7 @@ class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
       if tables.to_s == '' || tables.to_s == '[]'
         raise ArgumentError, "tables is nil or empty"
       end
-      _each_tabledef(tables, options, pg_opts, &block)
+      _each_tabledef(tables, options, &block)
     rescue TableMissingError => e
       tables.delete e.table
       missing_tables << e.table
@@ -60,13 +52,13 @@ ORDER BY c.table_name, c.ordinal_position;
 EOS
   TABLE_PLACEHOLDER_START_NUM = 2 # because $1 is used by table_schema
 
-  def _each_tabledef(tables, options, pg_opts, &block)
-    cli = PG::Connection.new(pg_opts)
+  def _each_tabledef(tables, options, &block)
+    cli = PGClient.new(options)
 
     # TODO call the query for every 50 tables
-    placeholders = placeholder_string(tables.size, TABLE_PLACEHOLDER_START_NUM)
-    query = COLUMNS_QUERY % [placeholders]
-    res = cli.query(query, [options['schema']] + tables)
+    res = cli.query(COLUMNS_QUERY, [options['schema']] + tables,
+                    placeholder_start_num: TABLE_PLACEHOLDER_START_NUM,
+                    placeholder_size: tables.size)
 
     create_opt = {}
     if options.has_key?(:skip_primary_key_check)

data/lib/flydata/sync_file_manager.rb

@@ -10,13 +10,11 @@ module Flydata
     DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
     BACKUP_DIR = ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
     TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
-    SYNC_TABLE_POSITIONS = 0
 
     def initialize(data_entry, source = nil)
       @data_entry = data_entry
       @source = source #for Source dependent objects
       @table_position_files = {} # File objects keyed by table name
-      @sync_table_positions_count = SYNC_TABLE_POSITIONS
     end
 
     def source
@@ -38,6 +36,7 @@ module Flydata
     end
 
     def save_dump_pos(status, table_name, last_pos, source_pos, state = nil, substate = nil)
+      raise "Cannot create dump pos file because source position is unavailable." unless source_pos
       File.open(dump_pos_path, 'w') do |f|
         f.write(dump_pos_content(status, table_name, last_pos, source_pos, state, substate))
       end
@@ -205,11 +204,28 @@ module Flydata
         tables.map{|table| File.join(table_positions_dir_path, table + '.binlog.pos.init')}
     end
 
+    def increment_table_position(seq)
+      seq = seq.to_i + 1
+      seq = FlydataCore::QueryJob::SYNC_FIRST_SEQ if seq == 1
+      seq
+    end
+
     # Read a sequence number from the table's position file,
     # increment the number and pass the number to a block.
     # After executing the block, saves the value to the position
     # file.
     def increment_and_save_table_position(table_name)
+      seq = get_table_position(table_name)
+
+      seq = increment_table_position(seq)
+
+      # logical transaction starts
+      yield(seq)
+      save_table_position(table_name, seq)
+      # logical transaction ends
+    end
+
+    def open_table_position_file(table_name)
       file = File.join(table_positions_dir_path, table_name + ".pos")
       retry_count = 0
       begin
@@ -221,43 +237,34 @@ module Flydata
         retry_count += 1
         retry
       end
-      f = @table_position_files[table_name]
-      seq = f.read
-      prev_seq_len = seq.size
-      f.rewind
-      seq = seq.to_i + 1
-      seq = FlydataCore::QueryJob::SYNC_FIRST_SEQ if seq == 1
-      new_seq_len = seq.to_s.size
+      @table_position_files[table_name]
+    end
+
+    def save_table_position(table_name, seq)
+      f = open_table_position_file(table_name)
+      prev_seq_len = f.size
 
       seq_to_write =  seq.to_s
+      new_seq_len = seq_to_write.size
+
       if new_seq_len < prev_seq_len
         seq_to_write += " " * (prev_seq_len - new_seq_len)
       end
-      # logical transaction starts
-      yield(seq)
       f.write(seq_to_write)
-      if @sync_table_positions_count > 1
-        @sync_table_positions_count -= 1
-      elsif @sync_table_positions_count == 1
-        f.fsync
-        @sync_table_positions_count = SYNC_TABLE_POSITIONS
-      end
-      # logical transaction ends
       f.truncate(new_seq_len) if new_seq_len < prev_seq_len
       f.rewind
     end
 
-    def save_table_position(table_name, seq)
-      file = File.join(table_positions_dir_path, table_name + ".pos")
-
-      File.open(file, "w") {|f| f.write(seq)}
-    end
-
     def get_table_position(table_name)
-      file = File.join(table_positions_dir_path, table_name + ".pos")
-      return nil unless File.exists?(file)
+      f = open_table_position_file(table_name)
+      seq = f.read
+      f.rewind
+      seq
+    end
 
-      File.open(file) {|f| f.read}
+    def get_next_table_position(table_name)
+      seq = get_table_position(table_name)
+      increment_table_position(seq)
     end
 
     def lock_pid_file
@@ -477,7 +484,11 @@ module Flydata
       status = items[0]
       table_name = items[1]
       last_pos = items[2].to_i
-
+      if status == Command::Sync::STATUS_START
+        #Use only source_pos information. Discard and initialize other items
+        table_name = nil
+        last_pos = nil
+      end
       source_pos_str = items[3..-3].join("\t")
       context = source.source_pos
       source_pos = context.create_source_pos(source_pos_str)

data/spec/flydata/command/setup_spec.rb

@@ -25,7 +25,6 @@ module Flydata
         expect(credentials).to receive(:authenticated?).and_return(false)
         allow(data_port).to receive(:get).and_return(dp)
         expect(sender).to receive(:process_exist?).and_return(true)
-        expect(sender).to receive(:stop)
         allow(sync_fm).to receive(:source_pos_path).and_return('/tmp')
         expect(conf).to receive(:copy_templates)
         expect(login).to receive(:run)

data/spec/flydata/command/sync_spec.rb

@@ -101,7 +101,7 @@ module Flydata
           allow(File).to receive(:exists?).and_return(false)
           expect(default_sync_fm).to receive(:load_dump_pos).and_return(default_dump_pos)
           expect(default_sync_fm).to receive(:dump_file_path).and_return(default_fp)
-          expect(default_sync_fm).to receive(:backup_dir).and_return(default_backup)
+          expect(default_sync_fm).to receive(:backup_dir).and_return(default_backup).at_least(:once)
           expect(subject).to receive(:target_tables).and_return(target_tables).at_least(:once)
           expect(subject).to receive(:print)
           expect(subject).to receive(:log_info)
@@ -145,7 +145,7 @@ module Flydata
         let(:values) { [4, 'John', nil, col4_value, nil, nil] }
 
         let(:source_table) do
-          Flydata::Parser::SourceTable.new("test_table", mysql_table_columns, ['id'])
+          Flydata::Parser::SourceTable.new("test_table", mysql_table_columns)
         end
 
         before do

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -150,9 +150,8 @@ EOT
 
     def expect_emitted_records(event, records)
       records = [records] unless records.kind_of?(Array)
-      records.each do |r|
-        expect(Engine).to receive(:emit).with(TEST_TAG, TEST_TIMESTAMP, r).ordered
-      end
+      expect(Engine).to receive(:emit_array).
+       with(TEST_TAG, records.collect{|r| [TEST_TIMESTAMP, r] })
       plugin.event_listener(event)
     end
 
@@ -235,10 +234,10 @@ EOT
       cleanup_flydata_files
       setup_initial_flydata_files
       allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
-      allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
-      allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_SENT_POSITION_FILE).
+      allow(Flydata::SourceMysql::PluginSupport::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
+      allow(Flydata::SourceMysql::PluginSupport::BinLogPositionFile).to receive(:new).with(TEST_SENT_POSITION_FILE).
         and_return(sent_position_file)
-      allow(Flydata::Mysql::TableMeta).to receive(:update)
+      allow(Flydata::SourceMysql::PluginSupport::TableMeta).to receive(:update)
       allow(plugin).to receive(:`).with("mysql -V").and_return("mysql  Ver 14.14 Distrib 5.5.40")
       allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
       Timecop.freeze(now)

data/spec/flydata/plugin_support/context_spec.rb

@@ -0,0 +1,27 @@
+require 'fluent_plugins_spec_helper'
+require 'flydata/plugin_support/context'
+
+describe ::Flydata::PluginSupport::Context do
+  let(:subject_object) { described_class.new(params) }
+
+  describe '#initialize' do
+    subject { subject_object }
+
+    context 'when no missing params' do
+      let(:params) { {
+        tables: %w(table_a table_b table_c),
+        tag: 'test_tag',
+        sync_fm: 'test_sync_fm',
+        omit_events: {'table_c' => %i(delete truncate_table)},
+        table_revs: {'table_a' => 1, 'table_b' => 2, 'table_c' => 3},
+      } }
+
+      it { expect{subject}.not_to raise_error }
+      it { expect(subject.tables).to eq(%w(table_a table_b table_c)) }
+      it { expect(subject.tag).to eq('test_tag') }
+      it { expect(subject.sync_fm).to eq('test_sync_fm') }
+      it { expect(subject.omit_events).to eq({'table_c' => %i(delete truncate_table)}) }
+      it { expect(subject.table_revs).to eq({'table_a' => 1, 'table_b' => 2, 'table_c' => 3}) }
+    end
+  end
+end

data/spec/flydata/source_mysql/parser/dump_parser_spec.rb

@@ -225,7 +225,7 @@ EOT
               }
               expect(check_point_block).to receive(:call) { |source_table, last_pos, bytesize, binlog_pos, state, substate|
                 expect(source_table.table_name).to eq('users_login')
-                expect(source_table.columns.keys).to eq([col_name, 'login_count', 'create_time', 'update_time'])
+                expect(source_table.instance_variable_get(:@columns).keys).to eq([col_name, 'login_count', 'create_time', 'update_time'])
                 expect(last_pos).to eq(index_after(dump_content, 'ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\'\';'))
                 expect(binlog_pos).to eq(default_binlog_pos)
                 expect(state).to eq(Flydata::Parser::State::INSERT_RECORD)
@@ -306,7 +306,7 @@ EOT
               # create_table_block
               expect(create_table_block).to receive(:call) { |source_table|
                 expect(source_table.table_name).to eq('users_login')
-                expect(source_table.columns.keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
+                expect(source_table.instance_variable_get(:@columns).keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
               }.once
               expect(create_table_block).to receive(:call).never
 
@@ -412,7 +412,7 @@ EOT
               # create_table_block
               expect(create_table_block).to receive(:call) { |source_table|
                 expect(source_table.table_name).to eq('users_login')
-                expect(source_table.columns.keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
+                expect(source_table.instance_variable_get(:@columns).keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
               }.once
               expect(create_table_block).to receive(:call).never
 
@@ -715,7 +715,7 @@ EOT
               # create_table_block
               expect(create_table_block).to receive(:call) { |source_table|
                 expect(source_table.table_name).to eq('users_login')
-                expect(source_table.columns.keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
+                expect(source_table.instance_variable_get(:@columns).keys).to eq(['user_id', 'login_count', 'comment', 'create_time', 'update_time'])
               }.once
               expect(create_table_block).to receive(:call).never
 

data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/alter_table_query_handler_spec.rb

@@ -1,8 +1,8 @@
 require 'fluent_plugins_spec_helper'
-require 'flydata/fluent-plugins/mysql/alter_table_query_handler'
-require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
+require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
+require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
 
-module Mysql
+module Flydata::SourceMysql::PluginSupport
   describe AlterTableQueryHandler do
     include_context "query handler context"
 

data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_dispatcher_spec.rb

@@ -1,7 +1,7 @@
 require 'fluent_plugins_spec_helper'
-require 'flydata/fluent-plugins/mysql/binlog_query_dispatcher'
+require 'flydata/source_mysql/plugin_support/binlog_query_dispatcher'
 
-module Mysql
+module Flydata::SourceMysql::PluginSupport
 
   QUERY_WITH_COMMENTS = <<EOT
 -- ALTER TABLE QUERY comment
@@ -51,17 +51,17 @@ EOT
     end
     let(:alter_query_handler) do
       r = double('alter_query_handler')
-      allow(r).to receive(:pattern).and_return(Mysql::AlterTableQueryHandler::PATTERN)
+      allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::AlterTableQueryHandler::PATTERN)
       r
     end
     let(:truncate_query_handler) do
       r = double('truncate_query_handler')
-      allow(r).to receive(:pattern).and_return(Mysql::TruncateTableQueryHandler::PATTERN)
+      allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::TruncateTableQueryHandler::PATTERN)
       r
     end
     let(:drop_database_query_handler) do
       r = double('drop_database_query_handler')
-      allow(r).to receive(:pattern).and_return(Mysql::DropDatabaseQueryHandler::PATTERN)
+      allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropDatabaseQueryHandler::PATTERN)
       r
     end
     let(:context) { double('context') }

data/spec/flydata/source_mysql/plugin_support/context_spec.rb

@@ -0,0 +1,26 @@
+require 'fluent_plugins_spec_helper'
+require 'flydata/source_mysql/plugin_support/context'
+
+describe ::Flydata::SourceMysql::PluginSupport::Context do
+  let(:subject_object) { described_class.new(params) }
+
+  describe '#initialize' do
+    subject { subject_object }
+
+    context 'when no missing params' do
+      let(:params) { {
+        tables: %w(table_a table_b table_c),
+        tag: 'test_tag',
+        sync_fm: 'test_sync_fm',
+        omit_events: {'table_c' => %i(delete truncate_table)},
+        table_revs: {'table_a' => 1, 'table_b' => 2, 'table_c' => 3},
+        database: 'test_db',
+        table_meta: 'test_table_meta',
+      } }
+
+      it { expect{subject}.not_to raise_error }
+      it { expect(subject.database).to eq('test_db') }
+      it { expect(subject.table_meta).to eq('test_table_meta') }
+    end
+  end
+end

data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/ddl_query_handler_spec.rb

@@ -1,8 +1,8 @@
 require 'fluent_plugins_spec_helper'
-require 'flydata/fluent-plugins/mysql/ddl_query_handler'
-require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
+require 'flydata/source_mysql/plugin_support/ddl_query_handler'
+require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
 
-module Mysql
+module Flydata::SourceMysql::PluginSupport
   describe DdlQueryHandler do
     include_context "query handler context"
 

data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/dml_record_handler_spec.rb

@@ -1,9 +1,9 @@
 # encoding: utf-8
 
 require 'fluent_plugins_spec_helper'
-require 'flydata/fluent-plugins/mysql/dml_record_handler'
+require 'flydata/source_mysql/plugin_support/dml_record_handler'
 
-module Mysql
+module Flydata::SourceMysql::PluginSupport
 
 describe DmlRecordHandler do
   let(:subject_object) { described_class.new(context) }

data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/drop_database_query_handler_spec.rb

@@ -1,8 +1,8 @@
 require 'fluent_plugins_spec_helper'
-require 'flydata/fluent-plugins/mysql/drop_database_query_handler'
-require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
+require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
+require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
 
-module Mysql
+module Flydata::SourceMysql::PluginSupport
   describe DropDatabaseQueryHandler do
     let(:subject_object) { described_class.new(context) }
     include_context "query handler context"

data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/shared_query_handler_context.rb

@@ -1,4 +1,4 @@
-module Mysql
+module Flydata::SourceMysql::PluginSupport
   shared_context "query handler context" do
     let(:target_database) { "testdb" }
     let(:database) { target_database }
@@ -10,6 +10,8 @@ module Mysql
       r = double('sync_fm')
       allow(r).to receive(:get_table_source_raw_pos).and_return(binlog_pos_string)
       allow(r).to receive(:increment_and_save_table_position).with(table).and_yield(seq).and_return(nil)
+      allow(r).to receive(:get_table_position).and_return(seq - 1)
+      allow(r).to receive(:increment_table_position).and_return(seq)
       allow(r).to receive(:delete_table_source_pos).with(table)
       r
     end

data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/table_meta_spec.rb

@@ -1,8 +1,8 @@
 require 'fluent_plugins_spec_helper'
-require 'flydata/fluent-plugins/mysql/table_meta'
+require 'flydata/source_mysql/plugin_support/table_meta'
 
-module Flydata
-module Mysql
+module Flydata::SourceMysql
+module PluginSupport
   describe TableMeta do
     let(:db_opts) { { host: 'test-host', port: 3306, username: 'test-user', password: 'test-pswd', database: 'test-db' } }
     let(:database) { 'test-db' }

data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/truncate_query_handler_spec.rb

@@ -1,8 +1,8 @@
 require 'fluent_plugins_spec_helper'
-require 'flydata/fluent-plugins/mysql/truncate_table_query_handler'
-require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
+require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
+require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
 
-module Mysql
+module Flydata::SourceMysql::PluginSupport
   describe TruncateTableQueryHandler do
     include_context "query handler context"
 
@@ -31,7 +31,10 @@ module Mysql
 
         context "for a non append only table" do
           it "should call Fluent's emit with appropriate params" do
-            expect(Fluent::Engine).to receive(:emit).with(tag, timestamp, expected_record)
+            expect(Fluent::Engine).to receive(:emit_array).
+             with(tag, [[timestamp, expected_record]])
+            expect(sync_fm).to receive(:save_table_position).with(table, seq)
+
             expect(subject.process(record))
           end
         end