flydata 0.1.7 → 0.1.8

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -3,10 +3,32 @@ module Fluent
 require 'fluent/plugin/in_mysql_binlog'
 require 'binlog'
 require 'kodama'
-require File.dirname(__FILE__) + '/preference'
+require_relative 'preference'
+require_relative '../../flydata'
+require_relative '../sync_file_manager'
+require_relative 'mysql/binlog_record_dispatcher'
+require_relative 'mysql/context'
+
+#Monkey-patch fluentd class (EngineClass) to support shutdown for input plugin.
+#This will be called when USR1 signal is received
+class EngineClass
+  #Send shutdown to all the sources
+  def shutdown_source
+    @sources.map {|s|
+      Thread.new do
+        begin
+          s.shutdown
+        rescue => e
+          $log.warn "unexpected error while shutting down", :error_class=>e.class, :error=>e
+          $log.warn_backtrace
+        end
+      end
+    }.each {|t|
+      t.join
+    }
+  end
+end
 
-require File.expand_path(File.join(File.dirname(__FILE__), '../../flydata'))
-require 'flydata/sync_file_manager'
 
 class MysqlBinlogFlydataInput < MysqlBinlogInput
   include MysqlBinlogFlydataInputPreference
@@ -14,6 +36,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
   def initialize
     super
+    install_custom_signal_handler
   end
 
   config_param :database, :string
@@ -27,17 +50,18 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     load_custom_conf
     $log.info "mysql host:\"#{@host}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\""
     @tables = @tables.split(/,\s*/)
-    @sync_fm = Flydata::FileUtil::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
-    @record_handler = FlydataMysqlBinlogRecordHandler.new(
-                  database: @database,
-                  tables: @tables,
-                  tag: @tag,
-                  sync_fm: @sync_fm)
+    sync_fm = Flydata::FileUtil::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
+
+    @context = Mysql::Context.new(
+      database: @database, tables: @tables,
+      tag: @tag, sync_fm: sync_fm
+    )
+    @record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
   end
 
   def start
     super
-    positions_path = @sync_fm.table_positions_dir_path
+    positions_path = @context.sync_fm.table_positions_dir_path
     Dir.mkdir positions_path unless File.exists? positions_path
   rescue Binlog::Error
     if (/basic_string::_M_replace_aux/ === $!.to_s)
@@ -56,7 +80,16 @@ EOS
   end
 
   def run
-    super
+    start_kodama(mysql_url) do |c|
+      c.binlog_position_file = @position_file
+      c.connection_retry_limit = @retry_limit
+      c.connection_retry_wait = @retry_wait
+      c.log_level = @log_level.to_sym
+      @listen_events.each do |event_type|
+        $log.trace { "registered binlog event listener '#{event_type}'" }
+        c.send("on_#{event_type}", &method(:event_listener))
+      end
+    end
   rescue
     $log.error "unexpected error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
     raise
@@ -68,229 +101,88 @@ EOS
     raise
   end
 
+  def start_kodama(options, &block)
+    @kodama_client = Kodama::Client.new(Kodama::Client.mysql_url(options))
+    @kodama_client.logger = $log
+    block.call(@kodama_client)
+    @kodama_client.start
+  end
+
   def event_listener(event)
-    @record_handler.dispatch(event)
-  rescue Exception
+    @record_dispatcher.dispatch(event)
+  rescue Exception => e
     position = File.open(@position_file) {|f| f.read }
     $log.error "error occured while processing #{event.event_type} event at #{position}\n#{e.message}\n#{$!.backtrace.join("\n")}"
     # Not reraising a StandardError because the underlying code can't handle an error well.
     raise unless e.kind_of?(StandardError)
   end
 
-end
-
-class MysqlBinlogRecordHandler
-  def dispatch(event)
-    method_name = "on_#{event.event_type.downcase}"
-    if self.respond_to?(method_name)
-      # TODO to_hash method call below can fail if event.event_type is
-      # "Update_rows".  This seems to be a bug of ruby-binlog.  The bug must
-      # be fixed when we support record update.
-      record = MysqlBinlogInput::BinlogUtil.to_hash(event)
-      self.send(method_name, record)
-    else
-      # $log.trace "Unhandled type: #{record["event_type"]}"
-    end
-  end
-end
-
-class FlydataMysqlBinlogRecordHandler < MysqlBinlogRecordHandler
-  TABLE_NAME = 'table_name' # A Flydata JSON tag to specify a table name
-  TYPE = 'type'
-  ROW = 'row'
-  SEQ = 'seq'
-  RESPECT_ORDER = 'respect_order'
-  INTEGER_TYPES = {'TINY' => 1,
-                   'SHORT' => 2,
-                   'INT24' => 3,
-                   'LONG' => 4,
-                   'LONGLONG' => 8
-                  }
-  SIGNLESS_INTEGER_PREFIX = '0SL'
-  SRC_POS = 'src_pos'
-
-  def initialize(opts)
-    mandatory_opts = [:database, :tables, :tag, :sync_fm]
-
-    missing_opts = mandatory_opts - opts.keys
-    unless (missing_opts.empty?)
-      raise "Mandatory option(s) are missing: #{missing_opts.join(', ')}"
-    end
-    @database = opts[:database]
-    @tables = opts[:tables]
-    @tag = opts[:tag]
-    @sync_fm = opts[:sync_fm]
-    @current_binlog_file = ""
-    @first_empty_binlog = true
-    @query_handler = FlydataMysqlBinlogQueryHandler.new(record_handler: self)
-  end
-
-  def on_rotate(record)
-    @current_binlog_file = record["binlog_file"]
-  end
-
-  def on_write_rows(record)
-    emit_insert(record)
-  end
-
-  def on_update_rows(record)
-    emit_update(record)
-  end
-
-  def on_delete_rows(record)
-    emit_delete(record)
-  end
-
-  def on_query(record)
-    @query_handler.dispatch(record)
-  end
-
-  def on_table_changed(table)
-    $log.trace "Table #{table} has changed.  Reloading the table column"
-  end
-
-  private
-
-  def acceptable?(record)
-    (@database == record["db_name"]) and @tables.include?(record["table_name"])
-  end
-
-  def emit_insert(record)
-    emit_record(:insert, record)
-  end
-
-  def emit_delete(record)
-    emit_record(:delete, record)
-  end
-
-  def emit_update(record)
-    emit_record(:update, record) do |row|
-      row.last # For update, row has two arrays (old and new values)  Use new values
-    end
-  end
-
-  def emit_record(type, record)
-    return unless acceptable?(record)
-
-    table = record['table_name']
-    position = record['next_position'] - record['event_length']
-    check_empty_binlog    
-    
-    records = record["rows"].collect do |row|
-                row = yield(row) if block_given?  # Give the caller a chance to generate the correct row
-                { TYPE => type, TABLE_NAME => table,
-                  RESPECT_ORDER => true, # Continuous sync needs record order to be kept
-                  SRC_POS => "#{@current_binlog_file}\t#{position}",        
-                  ROW => row.each.with_index(1).inject({}) do |h, (v, i)|
-                    if v.kind_of?(String)
-                      v = v.encode('utf-16', :undef => :replace, :invalid => :replace).encode('utf-8')
-                    end
-                    h[i.to_s] = v
-                    h
-                  end
-                }
-              end
-
-    encode_signless_integer(records, record["columns"])
-
-    # Use binlog's timestamp
-    timestamp = record["timestamp"].to_i
-    records.each do |row|
-      @sync_fm.increment_and_save_table_position(row[TABLE_NAME]) do |seq|
-        row[SEQ] = seq
-        Engine.emit(@tag, timestamp, row)
+  def shutdown
+    if @thread and @thread.alive?
+      $log.info "Requesting stop Kodama"
+      @kodama_client.stop_request
+      if wait_till_safe_to_stop
+        $log.info "Killing Kodama client"
+        Thread.kill(@thread)
+      else
+        $log.error "Unable to stop Kodama"
       end
     end
   end
 
-  private
-  def encode_signless_integer(records, column_types)
-    records.each do |record|
-      record[ROW].keys.each do |position|
-        index = position.to_i - 1
-        column_type = column_types[index]
-        if INTEGER_TYPES.keys.include?(column_type)
-          # It's a signless integer.
-          intval = record[ROW][position]
-          next unless (intval.kind_of?(Numeric) || intval =~ /^-?[\d]+$/)
-          width = INTEGER_TYPES[column_type] * 2 # * 2 because a single byte requires two characters (e.g. ff)
-          signless_val = SIGNLESS_INTEGER_PREFIX
-          signless_val += sprintf("%0#{width}x", intval).gsub(/\.\.f/, 'f' * width).slice(-width..-1)
-          record[ROW][position] = signless_val
-        end
-      end
+  def wait_till_safe_to_stop
+    retry_count = 5
+    1.upto(retry_count) do |i|
+      return true if @kodama_client.safe_to_stop?
+      sleep 3
     end
-  end
-  
-  def check_empty_binlog
-    #Log one warning per consecutive records that have empty binlog filename
-    if @current_binlog_file.to_s.empty? 
-      if @first_empty_binlog
-        $log.warn "Binlog file name is empty. Rotate event not received!"
-        @first_empty_binlog = false
-      end  
-    else
-      @first_empty_binlog = true
-    end        
+    false
+  end
+
+  #Hack: All that has been added here is `Fluent::Engine.shutdown_source`. This should be in
+  #fluentd's supervisor#install_main_process_signal_handlers
+  def install_custom_signal_handler
+      trap :USR1 do
+        $log.debug "fluentd main process get SIGUSR1"
+        $log.info "force flushing buffered events"
+        #@log.reopen!
+
+        # Creating new thread due to mutex can't lock
+        # in main thread during trap context
+        Thread.new {
+          begin 
+            Fluent::Engine.shutdown_source
+            Fluent::Engine.flush!
+            $log.debug "flushing thread: flushed"
+          rescue Exception => e
+            $log.warn "flushing thread error: #{e}"
+          end
+        }.run
+      end
   end
 end
 
-class MysqlBinlogQueryHandler
-  def initialize
-    @mapping_table = []
-  end
+class BinLogPosition
+  include Comparable
+  attr_accessor :file, :pos
 
-  def dispatch(record)
-    @mapping_table.each do |pattern, method_name|
-      query = normalize_query(record["query"])
-      if (pattern.match(query))
-        if (self.respond_to?(method_name))
-          self.send(method_name, record, query)
-        else
-          raise "method '#{method_name}' is not defined in #{self.class.name} although its matching pattern is defined"
-        end
-        break
-      end
-    end
+  def initialize(binlog_content)
+    items = binlog_content.split("\t")
+    @file = items[0]
+    @pos = items[1].to_i
   end
 
-  private
-  def normalize_query(query)
-    query = strip_comments(query)
+  def <=>(obj)
+    (self.file <=> obj.file) == 0 ? (self.pos <=> obj.pos) : (self.file <=> obj.file)
   end
 
-  def strip_comments(query)
-    query = query.gsub(/--\s.*\n/, ' ')  # -- style comments
-    query = query.gsub(/\/\*[^\*].*\*\//, ' ') # /* */ style comments
-    query = query.gsub(/\s+/, ' ') # replace multiple spaces with a space
+  def to_s
+    "#{file}\t#{pos}"
   end
 end
 
-class FlydataMysqlBinlogQueryHandler < MysqlBinlogQueryHandler
-  def initialize(opts)
-    mandatory_opts = [:record_handler]
-    missing_opts = mandatory_opts - opts.keys
-    unless missing_opts.empty?
-      raise "mandatory options are missing: #{missing_opts.join(", ")}"
-    end
-    @opts = opts
-
-    @mapping_table = [
-      [/^alter table/i, :on_alter_table],
-    ]
-  end
-
-  def on_alter_table(record, query)
-    m = /alter table\s+(?<table>[^\s]+)/i.match(query)
-    if m.nil?
-      raise "This alter table query has no table name? '#{query}'"
-    end
-
-    @opts[:record_handler].on_table_changed(m[:table])
-  end
 end
 
-end
 
 # HACK
 # Monkey patch the class to manage string's character encoding.
@@ -335,15 +227,6 @@ module Kodama
 
 Client.class_eval do
   attr_accessor :logger
-
-  # Except for replacing logger, the implementation is a copy of the original
-  # method.
-  def self.start(options = {}, &block)
-    client = self.new(mysql_url(options))
-    client.logger = $log
-    block.call(client)
-    client.start
-  end
 end
 
 end

data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb

@@ -0,0 +1,34 @@
+require_relative 'binlog_query_handler'
+
+module Mysql
+  class AlterTableQueryHandler < BinlogQueryHandler
+    PATTERN = /^alter table/
+
+    def initialize(context)
+      super
+    end
+
+    def pattern
+      PATTERN
+    end
+
+    def process(record, normalized_query)
+      case normalized_query
+      when /^alter table [^\s]+ add column/i
+        on_add_column(record, normalized_query)
+      else
+        $log.debug("not supported alter table query:'#{normalized_query}'")
+      end
+    end
+
+    private
+
+    def on_add_column(record, query)
+      $log.debug("on_add_column query:'#{record['query']}'")
+      #TODO: Uncomment following lines after supporting alter table on the server side
+      #emit_record(:alter_table, record, increment_table_rev: true) do
+      #  AlterTableAddColumnParser.new.parse(record['query'])
+      #end
+    end
+  end
+end

data/lib/flydata/fluent-plugins/mysql/binlog_position.rb

@@ -0,0 +1,20 @@
+module Mysql
+  class BinLogPosition
+    include Comparable
+    attr_accessor :file, :pos
+
+    def initialize(binlog_content)
+      items = binlog_content.split("\t")
+      @file = items[0]
+      @pos = items[1].to_i
+    end
+
+    def <=>(obj)
+      (self.file <=> obj.file) == 0 ? (self.pos <=> obj.pos) : (self.file <=> obj.file)
+    end
+
+    def to_s
+      "#{file}\t#{pos}"
+    end
+  end
+end

data/lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb

@@ -0,0 +1,39 @@
+require_relative 'alter_table_query_handler'
+
+module Mysql
+  class BinlogQueryDispatcher
+    def initialize
+      @handlers = []
+    end
+
+    def dispatch(record)
+      @handlers.each do |handler|
+        query = normalize_query(record["query"])
+        if (handler.pattern.match(query))
+          handler.process(record, query)
+          break
+        end
+      end
+    end
+
+    private
+
+    def normalize_query(query)
+      query = strip_comments(query)
+    end
+
+    def strip_comments(query)
+      query = query.gsub(/--\s.*\n/, ' ')  # -- style comments
+      query = query.gsub(/\/\*[^\*].*\*\//, ' ') # /* */ style comments
+      query = query.gsub(/\s+/, ' ') # replace multiple spaces with a space
+    end
+  end
+
+  class FlydataBinlogQueryDispatcher < BinlogQueryDispatcher
+    def initialize(context)
+      @handlers = [
+        AlterTableQueryHandler.new(context),
+      ]
+    end
+  end
+end