flydata 0.6.4 → 0.6.5

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -233,6 +233,15 @@ EOT
         let(:default_sql) { " DEFAULT #{default_value_sql}" }
         it_behaves_like *examples
       end
+      context "replaced nil default" do
+        before do
+          column[:default] = default_value
+          allow(subject_object).to receive(:replace_default_value).
+            and_return(nil)
+        end
+        let(:default_sql) { no_default_sql }
+        it_behaves_like *examples
+      end
     end
 
     shared_examples "not null values" do |*examples|
@@ -977,6 +986,22 @@ EOS
       end
     end
   end
+  describe '.replace_default_value' do
+    subject { subject_object.replace_default_value(flydata_type, redshift_type,
+              default_value) }
+
+    context 'with integer type' do
+      let(:flydata_type) { "int8" }
+      let(:redshift_type) { "int8" }
+
+      context "nextval() as the value" do
+        let(:default_value) { "nextval('test_id_seq'::regclass)" }
+        it 'returns nil because Redshift does not support nextval()' do
+          is_expected.to be_nil
+        end
+      end
+    end
+  end
 end
 
 end

data/lib/flydata.rb

@@ -36,10 +36,3 @@ module Flydata
   VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
   include Flydata::Heroku
 end
-
-# Require all helper files -
-# TODO : Check if this should be moved else where or can be more generic (not just helper files)
-lib_dir = File.dirname(File.absolute_path(__FILE__))
-FileUtils.cd(lib_dir) do
-  Dir["flydata/helper/**/*.rb"].each { |file| require file }
-end

data/lib/flydata/command/base.rb

@@ -27,10 +27,11 @@ module Flydata
           raise "Failed to retrieve data_entries"
         end
         data_entries.collect do |de|
+          source = Source.create(de)
           if Flydata::Preference::DataEntryPreference.conf_exists?(de)
-            Flydata::Preference::DataEntryPreference.load_conf(de)
+            Flydata::Preference::DataEntryPreference.load_conf(de, source)
           else
-            Flydata::Preference::DataEntryPreference.filter_data_entry(de)
+            Flydata::Preference::DataEntryPreference.filter_data_entry(de, source)
             de
           end
         end

data/lib/flydata/fluent-plugins/flydata_plugin_ext/base.rb

@@ -0,0 +1,5 @@
+# Load client library(flydata-agent/lib)
+lib = File.expand_path(File.join(File.dirname(__FILE__), '../../../'))
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'flydata'

data/lib/flydata/fluent-plugins/flydata_plugin_ext/flush_support.rb

@@ -0,0 +1,52 @@
+module Fluent
+
+  #Monkey-patch fluentd class (EngineClass) to support shutdown for input plugin.
+  #This will be called when USR1 signal is received
+  class EngineClass
+    #Send shutdown to all the sources
+    def shutdown_source
+      @sources.map {|s|
+        Thread.new do
+          begin
+            s.shutdown
+          rescue => e
+            $log.warn "unexpected error while shutting down", :error_class=>e.class, :error=>e
+            $log.warn_backtrace
+          end
+        end
+      }.each {|t|
+        t.join
+      }
+    end
+  end
+
+  module FlushSupport
+    def initialize
+      super
+      install_custom_signal_handler
+    end
+
+    #Hack: All that has been added here is `Fluent::Engine.shutdown_source`. This should be in
+    #fluentd's supervisor#install_main_process_signal_handlers
+    def install_custom_signal_handler
+      trap :USR1 do
+        $log.debug "fluentd main process get SIGUSR1"
+        $log.info "force flushing buffered events"
+        #@log.reopen!
+
+        # Creating new thread due to mutex can't lock
+        # in main thread during trap context
+        Thread.new {
+          begin
+            Fluent::Engine.shutdown_source
+            Fluent::Engine.flush!
+            $log.debug "flushing thread: flushed"
+          rescue Exception => e
+            $log.warn "flushing thread error: #{e}"
+          end
+        }.run
+      end
+    end
+  end
+
+end

data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb

@@ -0,0 +1,55 @@
+require 'flydata/sync_file_manager'
+require 'flydata-core/fluent/config_helper'
+require 'flydata/fluent-plugins/flydata_plugin_ext/flush_support'
+require 'flydata/fluent-plugins/flydata_plugin_ext/transaction_support'
+
+module Fluent
+
+  module FlydataSync
+    def self.included(base)
+      base.class_eval do
+        include FlushSupport
+        prepend TransactionSupport
+
+        config_param :tables, :string
+        config_param :tables_append_only, :string
+
+        # binlog plugin
+        # TODO: Need to check if not configured or not
+        #config_param :tag, :string
+        #config_param :position_file, :string, :default => 'position.log'
+      end
+    end
+
+    def configure(conf)
+      super
+
+      @binlog_position_file = self.class::BINLOG_POSITION_FILE_CLASS.new(@position_file)
+      unless @binlog_position_file.exists?
+        raise "No position file(#{@binlog_position_file.path}). Initial synchronization is required before starting."
+      end
+
+      @sync_fm = Flydata::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
+      sent_position_file_path = @sync_fm.sent_source_pos_path(@position_file)
+      @sent_position_file = self.class::BINLOG_POSITION_FILE_CLASS.new(sent_position_file_path)
+
+      # Create positions dir
+      positions_path = @sync_fm.table_positions_dir_path
+      Dir.mkdir positions_path unless File.exists? positions_path
+
+      load_custom_conf
+
+      @tables = @tables.split(/(?:\s*,\s*|\s+)/)
+      @omit_events = Hash.new
+      @tables_append_only.split(/(?:\s*,\s*|\s+)/).each do |table|
+        @tables << table unless @tables.include?(table)
+        @omit_events[table] = [:delete, :truncate_table]
+      end
+
+      # Remove tables that do not have pos files
+      new_tables = @sync_fm.get_new_table_list(@tables, "pos")
+      @tables -= new_tables
+      $log.info "Not watching these tables: #{new_tables.join(", ")}"
+    end
+  end
+end

data/lib/flydata/fluent-plugins/{preference.rb → flydata_plugin_ext/preference.rb}

@@ -1,5 +1,6 @@
 require 'yaml'
 require 'flydata/util/encryptor'
+require 'flydata/source_mysql/data_entry'
 
 module Fluent
   module DataEntryPreferenceConfigurable
@@ -68,23 +69,10 @@ module Fluent
   end
 
   module MysqlBinlogFlydataInputPreference
-    CUSTOM_CONFIG_PARAMS = {
-      mysql_data_entry_preference: {
-        database: {},
-        tables: {},
-        tables_append_only: {},
-        host: {},
-        username: {},
-        password: {encrypted: true},
-        ssl_ca_content: {},
-        ssl_cipher: {},
-      },
-    }
-
     def self.included(base)
       base.class_eval do
         include DataEntryPreferenceConfigurable
-        CUSTOM_CONFIG_PARAMS.each do |type, custom_conf|
+        Flydata::SourceMysql::DataEntry::CONFIG_PARAMS.each do |type, custom_conf|
           custom_conf.each do |key, option|
             custom_config_param key, type, option
           end

data/lib/flydata/fluent-plugins/flydata_plugin_ext/transaction_support.rb

@@ -0,0 +1,58 @@
+module Fluent
+
+  class TransactionContext
+    def set_transaction_broken
+      @transaction_broken = true
+    end
+
+    def transaction_broken?
+      @transaction_broken == true
+    end
+  end
+
+  # Transaction support
+  # This module is expected to be prepended
+  module TransactionSupport
+    def self.included(base)
+      raise "This module is expected to be prepended"
+    end
+
+    #def self.prepended(base)
+    #end
+
+    def configure(conf)
+      @lock_file = Flydata::FLYDATA_LOCK
+      super
+    end
+
+    def start
+      if File.exists?(@lock_file)
+        $log.error "Previous process was terminated abnormally.  To start, remove the lock file after checking data integrity."
+        @abort = true
+        Process.kill(:TERM, Process.ppid)
+        return
+      end
+
+      super
+    end
+
+    def process_aborted?
+      instance_variable_defined? :@abort
+    end
+
+    def do_transaction
+      transaction_context =TransactionContext.new
+      File.open(@lock_file, "w") {|f| f.write(Process.pid)}
+      begin
+        yield(transaction_context)
+      ensure
+        # leave the lock file when a transaction is broken
+        if !transaction_context.transaction_broken? && File.exists?(@lock_file) &&
+            Process.pid == File.open(@lock_file, "r") {|f| f.read}.to_i
+          File.delete(@lock_file)
+        end
+      end
+    end
+  end
+
+end

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -1,54 +1,32 @@
+require_relative 'flydata_plugin_ext/base'
+
 module Fluent
 
 require 'fluent/plugin/in_mysql_binlog'
 require 'binlog'
 require 'kodama'
 
-# Load client library(flydata-agent/lib)
-lib = File.expand_path(File.join(File.dirname(__FILE__), '../../'))
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'flydata'
 require 'flydata/sync_file_manager'
 require 'flydata-core/mysql/command_generator'
-require 'flydata/fluent-plugins/preference'
+require 'flydata/fluent-plugins/flydata_plugin_ext/preference'
 require 'flydata/fluent-plugins/mysql/binlog_position_file'
 require 'flydata/fluent-plugins/mysql/binlog_record_dispatcher'
 require 'flydata/fluent-plugins/mysql/context'
-require 'flydata/fluent-plugins/idle_event_detector'
+require 'flydata/fluent-plugins/flydata_plugin_ext/idle_event_detector'
 require 'flydata/fluent-plugins/mysql/table_meta'
 require 'flydata/source_mysql/table_ddl'
 require 'flydata-core/fluent/config_helper'
 require 'flydata-core/mysql/ssl'
-
-#Monkey-patch fluentd class (EngineClass) to support shutdown for input plugin.
-#This will be called when USR1 signal is received
-class EngineClass
-  #Send shutdown to all the sources
-  def shutdown_source
-    @sources.map {|s|
-      Thread.new do
-        begin
-          s.shutdown
-        rescue => e
-          $log.warn "unexpected error while shutting down", :error_class=>e.class, :error=>e
-          $log.warn_backtrace
-        end
-      end
-    }.each {|t|
-      t.join
-    }
-  end
-end
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync'
 
 
 class MysqlBinlogFlydataInput < MysqlBinlogInput
   include MysqlBinlogFlydataInputPreference
+  include FlydataSync
+
   Plugin.register_input('mysql_binlog_flydata', self)
 
-  def initialize
-    super
-    install_custom_signal_handler
-  end
+  BINLOG_POSITION_FILE_CLASS = Mysql::BinLogPositionFile
 
   config_param :database, :string
   config_param :tables, :string
@@ -77,19 +55,10 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
   def configure(conf)
     super
-    @binlog_position_file = Mysql::BinLogPositionFile.new(@position_file)
-    unless @binlog_position_file.exists?
-      raise "No position file(#{@position_file}). Initial synchronization is required before starting."
-    end
-
-    @sync_fm = Flydata::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
-    @sent_position_file_path = @sync_fm.sent_source_pos_path(@position_file)
-
-    load_custom_conf
 
     # SSL configuration
     unless @ssl_ca_content.to_s.strip.empty?
-      @ssl_ca_path = @sync_fm.ssl_ca_path(@position_file)
+      @ssl_ca_path = @sync_fm.ssl_ca_path(@binlog_position_file.path)
       @sync_fm.save_ssl_ca(FlydataCore::Fluent::ConfigHelper.unescape_conf(@ssl_ca_content), @ssl_ca_path)
     end
 
@@ -108,18 +77,6 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       #exit 1   # causes retry loop
     end
 
-    @tables = @tables.split(/,\s*/)
-    @omit_events = Hash.new
-    @tables_append_only.split(/,\s*/).each do |table|
-      @tables << table unless @tables.include?(table)
-      @omit_events[table] = [:delete, :truncate_table]
-    end
-
-    # Remove tables that do not have pos files
-    new_tables = @sync_fm.get_new_table_list(@tables, "pos")
-    @tables -= new_tables
-    $log.info "Not watching these tables: #{new_tables.join(", ")}"
-
     table_meta = Flydata::Mysql::TableMeta.new(@db_opts.merge(tables: @tables))
 
     table_revs = tables.inject({}) do |h, table_name|
@@ -135,18 +92,11 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     )
     @record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
     @idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval, @idle_timeout)
-    @lock_file = Flydata::FLYDATA_LOCK
   end
 
   def start
-    if File.exists?(@lock_file)
-      $log.error "Previous process was terminated abnormally.  To start, remove the lock file after checking data integrity."
-      @abort = true
-      Process.kill(:TERM, Process.ppid)
-      return
-    end
-
     super
+
     @idle_event_detector.start do |reason, timestamp|
       case reason
       when :event_not_coming
@@ -161,9 +111,6 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       end
     end
 
-    positions_path = @context.sync_fm.table_positions_dir_path
-    Dir.mkdir positions_path unless File.exists? positions_path
-
   rescue Binlog::Error
     if (/basic_string::_M_replace_aux/ === $!.to_s)
       # TODO Fix the root cause in mysql-replication-listener
@@ -181,73 +128,67 @@ EOS
   end
 
   def run
-    return if instance_variable_defined? :@abort
+    return if process_aborted?
 
     @context.table_meta.update
     Flydata::SourceMysql::TableDdl.migrate_tables(@context.tables, @db_opts,
-                                            @context.sync_fm, @position_file,
+                                            @context.sync_fm, @binlog_position_file.path,
                                             @context) do |event|
       @record_dispatcher.dispatch(event)
     end
 
     current_ssl_cipher = @ssl_cipher
     retried = false
-    transaction_broken = false
-    File.open(@lock_file, "w") {|f| f.write(Process.pid)}
-    begin
+
+    do_transaction do |transaction_context|
       begin
-        start_kodama(mysql_url) do |c|
-          c.binlog_position_file = @position_file
-          if @ssl_ca_path.to_s != '' && c.respond_to?(:ssl_ca=)
-            $log.info "SSL is enabled. (ssl_ca: #{@ssl_ca_path})"
-            c.ssl_ca = @ssl_ca_path
-            unless current_ssl_cipher.to_s.empty?
-              $log.info "SSL cipher is set. (ssl_cipher: #{current_ssl_cipher})"
-              c.ssl_cipher = current_ssl_cipher
+        begin
+          start_kodama(mysql_url) do |c|
+            c.binlog_position_file = @binlog_position_file.path
+            if @ssl_ca_path.to_s != '' && c.respond_to?(:ssl_ca=)
+              $log.info "SSL is enabled. (ssl_ca: #{@ssl_ca_path})"
+              c.ssl_ca = @ssl_ca_path
+              unless current_ssl_cipher.to_s.empty?
+                $log.info "SSL cipher is set. (ssl_cipher: #{current_ssl_cipher})"
+                c.ssl_cipher = current_ssl_cipher
+              end
             end
-          end
 
-          if c.respond_to?(:sent_binlog_position_file=)
-            $log.info "Sent position feature is enabled. sent_position_file:#{@sent_position_file_path}"
-            c.sent_binlog_position_file = @sent_position_file_path
-          end
+            if c.respond_to?(:sent_binlog_position_file=)
+              $log.info "Sent position feature is enabled. sent_position_file:#{@sent_position_file.path}"
+              c.sent_binlog_position_file = @sent_position_file.path
+            end
 
-          $log.info("Binlog position - resume_pos:'#{IO.read(@position_file) rescue IOError}' " +
-                                      "sent_pos:'#{IO.read(@sent_position_file_path) rescue IOError}'")
+            $log.info("Binlog position - resume_pos:'#{@binlog_position_file.read rescue IOError}' " +
+                      "sent_pos:'#{@sent_position_file.read rescue IOError}'")
 
-          c.connection_retry_limit = @retry_limit
-          c.connection_retry_wait = @retry_wait
-          c.log_level = @log_level.to_sym
-          @listen_events.each do |event_type|
-            $log.trace { "registered binlog event listener '#{event_type}'" }
-            c.send("on_#{event_type}", &method(:event_listener))
+            c.connection_retry_limit = @retry_limit
+            c.connection_retry_wait = @retry_wait
+            c.log_level = @log_level.to_sym
+            @listen_events.each do |event_type|
+              $log.trace { "registered binlog event listener '#{event_type}'" }
+              c.send("on_#{event_type}", &method(:event_listener))
+            end
           end
+        rescue Kodama::TransactionError
+          $log.debug "TransactionError"
+          transaction_context.set_transaction_broken
+          raise
+        end
+      rescue Binlog::Error => e
+        if /dh key too small/.match(e.to_s) && !retried && !@secondary_ssl_cipher.to_s.empty?
+          retried = true
+          current_ssl_cipher = @secondary_ssl_cipher
+          $log.warn("Retry with secondary ssl cipher list due to '#{e}' - secondary_ssl_cipher: '#{@secondary_ssl_cipher}'")
+          retry
+        elsif /binlog file.*does not exist/.match(e.to_s)
+          $log.error("#{e.to_s}.  Sync must be reset.  Terminating the agent.")
+          Process.kill(:TERM, Process.ppid)
+        else
+          raise e
         end
-      rescue Kodama::TransactionError
-        $log.debug "TransactionError"
-        transaction_broken = true
-        raise
-      end
-    rescue Binlog::Error => e
-      if /dh key too small/.match(e.to_s) && !retried && !@secondary_ssl_cipher.to_s.empty?
-        retried = true
-        current_ssl_cipher = @secondary_ssl_cipher
-        $log.warn("Retry with secondary ssl cipher list due to '#{e}' - secondary_ssl_cipher: '#{@secondary_ssl_cipher}'")
-        retry
-      elsif /binlog file.*does not exist/.match(e.to_s)
-        $log.error("#{e.to_s}.  Sync must be reset.  Terminating the agent.")
-        Process.kill(:TERM, Process.ppid)
-      else
-        raise e
-      end
-    ensure
-      if !transaction_broken && # leave the lock file when a transaction is broken
-         File.exists?(@lock_file) &&
-         Process.pid == File.open(@lock_file, "r") {|f| f.read}.to_i
-        File.delete(@lock_file)
       end
     end
-
   rescue => e
     # HACK: mysql-replication-listener has a network connection leak bug which doesn't release a connection
     # to MySQL.  Rather than fixing the bug, restarting the fluentd process for now.
@@ -279,7 +220,7 @@ EOS
   end
 
   def shutdown
-    return if instance_variable_defined? :@abort
+    return if process_aborted?
 
     @idle_event_detector.stop
     if @thread and @thread.alive?
@@ -296,6 +237,7 @@ EOS
          $log.warn "an error occurred during Kodama shutdown.  error:'#{e.to_s}'\n#{e.backtrace.join("\n")}"
       end
     end
+
     @sync_fm.close
   end
 
@@ -307,28 +249,6 @@ EOS
     end
     false
   end
-
-  #Hack: All that has been added here is `Fluent::Engine.shutdown_source`. This should be in
-  #fluentd's supervisor#install_main_process_signal_handlers
-  def install_custom_signal_handler
-      trap :USR1 do
-        $log.debug "fluentd main process get SIGUSR1"
-        $log.info "force flushing buffered events"
-        #@log.reopen!
-
-        # Creating new thread due to mutex can't lock
-        # in main thread during trap context
-        Thread.new {
-          begin
-            Fluent::Engine.shutdown_source
-            Fluent::Engine.flush!
-            $log.debug "flushing thread: flushed"
-          rescue Exception => e
-            $log.warn "flushing thread error: #{e}"
-          end
-        }.run
-      end
-  end
 end