flydata 0.3.16 → 0.3.17

data/lib/flydata/compatibility_check.rb

@@ -1,6 +1,6 @@
 require 'mysql2'
 require 'flydata/command_loggable'
-require 'flydata/util/mysql_util'
+require 'flydata/mysql/mysql_util'
 
 module Flydata
 
@@ -123,7 +123,7 @@ module Flydata
     end
 
     def check_mysql_protocol_tcp_compat
-      query = Util::MysqlUtil.generate_mysql_show_grants_cmd(@db_opts)
+      query = Mysql::MysqlUtil.generate_mysql_show_grants_cmd(@db_opts)
 
       Open3.popen3(query) do |stdin, stdout, stderr|
         stdin.close

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -9,13 +9,14 @@ lib = File.expand_path(File.join(File.dirname(__FILE__), '../../'))
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'flydata'
 require 'flydata/sync_file_manager'
-require 'flydata/util/mysql_util'
+require 'flydata/mysql/mysql_util'
 require 'flydata/fluent-plugins/preference'
 require 'flydata/fluent-plugins/mysql/binlog_position_file'
 require 'flydata/fluent-plugins/mysql/binlog_record_dispatcher'
 require 'flydata/fluent-plugins/mysql/context'
 require 'flydata/fluent-plugins/idle_event_detector'
 require 'flydata/fluent-plugins/mysql/table_meta'
+require 'flydata/mysql/table_ddl'
 require 'flydata-core/fluent/config_helper'
 
 #Monkey-patch fluentd class (EngineClass) to support shutdown for input plugin.
@@ -75,11 +76,11 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     end
 
     # Db access opts
-    db_opts = { host: @host, port: @port, username: @username, password: @password, database: @database, ssl_ca: @ssl_ca_path }
+    @db_opts = { host: @host, port: @port, username: @username, password: @password, database: @database, ssl_ca: @ssl_ca_path }
 
     $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
     $log.info "mysql client version: #{`mysql -V`}"
-    server_version = `echo 'select version();' | #{Flydata::Util::MysqlUtil.generate_mysql_cmd(db_opts)} 2>/dev/null`
+    server_version = `echo 'select version();' | #{Flydata::Mysql::MysqlUtil.generate_mysql_cmd(@db_opts)} 2>/dev/null`
     $log.info "mysql server version: #{server_version}"
 
     @tables = @tables.split(/,\s*/)
@@ -94,7 +95,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     @tables -= new_tables
     $log.info "Not watching these tables: #{new_tables.join(", ")}"
 
-    table_meta = Mysql::TableMeta.new(db_opts.merge(tables: @tables))
+    table_meta = Flydata::Mysql::TableMeta.new(@db_opts.merge(tables: @tables))
 
     table_revs = tables.inject({}) do |h, table_name|
       h[table_name] = @sync_fm.table_rev(table_name)
@@ -145,6 +146,11 @@ EOS
 
   def run
     @context.table_meta.update
+    Flydata::Mysql::TableDdl.migrate_tables(@context.tables, @db_opts,
+                                            @context.sync_fm, @position_file,
+                                            @context) do |event|
+      @record_dispatcher.dispatch(event)
+    end
     start_kodama(mysql_url) do |c|
       c.binlog_position_file = @position_file
       if @ssl_ca_path.to_s != '' && c.respond_to?(:ssl_ca=)
@@ -167,7 +173,7 @@ EOS
   rescue => e
     # HACK: mysql-replication-listener has a network connection leak bug which doesn't release a connection
     # to MySQL.  Rather than fixing the bug, restarting the fluentd process for now.
-    $log.warn "kodama died with an error.  Restart the process after #{@retry_wait} seconds.  error: #{e.class.to_s} '#{e.to_s}'"
+    $log.warn "kodama died with an error.  Restart the process after #{@retry_wait} seconds.  error: #{e.class.to_s} '#{e.to_s}'\n#{e.backtrace.join("\n")}"
     sleep @retry_wait
     Process.kill(:HUP, Process.ppid) # Send SIGHUP to the supervisor to restart fluentd
   rescue SignalException
@@ -260,16 +266,15 @@ class Client
         def rows
           rs = super
           # HACK
-          # Assuming all string values are UTF-8
-          # To make this right, MySQL client's encoding must be set to UTF-8
-          # But how?
+          # Set string encoding to binary because ruby-binlog has no knowledge
+          # about the encoding of strings.
           new_rs = rs.collect {|row|
                      row.collect{|value|
                        if (value.kind_of?(Array))
                          # Update has two rows in it
-                         value.collect{|val| val.force_encoding("UTF-8") if val.respond_to?(:force_encoding); val}
+                         value.collect{|val| val.force_encoding("binary") if val.respond_to?(:force_encoding); val}
                        else
-                         value.force_encoding("UTF-8") if value.respond_to?(:force_encoding); value
+                         value.force_encoding("binary") if value.respond_to?(:force_encoding); value
                        end
                      }
                    }

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -1,6 +1,7 @@
 require 'fluent/plugin/in_mysql_binlog'
 require 'binlog'
-require 'flydata/fluent-plugins/mysql/binlog_position'
+require 'flydata/mysql/binlog_position'
+require 'flydata-core/record/record'
 
 module Mysql
   class BinlogRecordHandler
@@ -10,6 +11,7 @@ module Mysql
     RESPECT_ORDER = :respect_order
     SRC_POS = :src_pos
     TABLE_REV = :table_rev
+    V = :v  # FlyData record format version
 
     def initialize(context)
       @context = context
@@ -21,7 +23,7 @@ module Mysql
       @context.tables.each do |table_name|
         table_binlog_content = @context.sync_fm.get_table_binlog_pos(table_name)
         if table_binlog_content
-          @table_binlog_pos[table_name] = BinLogPosition.new(table_binlog_content)
+          @table_binlog_pos[table_name] = Flydata::Mysql::BinLogPosition.new(table_binlog_content)
         end
       end
     end
@@ -40,7 +42,7 @@ module Mysql
       acceptable = @context.tables.include?(table)
 
       if acceptable and @table_binlog_pos[record['table_name']]
-        if @table_binlog_pos[record['table_name']] >= BinLogPosition.new(
+        if @table_binlog_pos[record['table_name']] >= Flydata::Mysql::BinLogPosition.new(
           "#{@context.current_binlog_file}\t#{record['next_position'] - record['event_length']}")
           acceptable = false
         else
@@ -84,6 +86,7 @@ module Mysql
         r[TABLE_NAME] = table
         r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
         r[TABLE_REV] = table_rev
+        r[V] = FlydataCore::Record::V2
       end
 
       # Use binlog's timestamp

data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb

@@ -49,7 +49,7 @@ module Mysql
           row_values = { ROW => row }
           row_values = yield(row) if block_given?  # Give the caller a chance to generate the correct row values
           row_types = row_values.keys
-          row_kinds = row_types.inject({}) {|m, k| m[k] = convert_to_flydata_row_format(table_name, row_values[k]); m}
+          row_kinds = row_types.inject({}) {|m, k| m[k] = convert_to_flydata_row_format(row_values[k]); m}
           encode_signless_integer(row_kinds, record["columns"], row_types)
           row_kinds
         end
@@ -57,22 +57,29 @@ module Mysql
       end
     end
 
-    def convert_to_flydata_row_format(table_name, row)
+    def convert_to_flydata_row_format(row)
       row.each.with_index(1).inject({}) do |h, (v, i)|
         if v.kind_of?(String)
-          v = encode_row_value(table_name, v)
+          v = encode_row_value(v)
+          if v.encoding == Encoding::BINARY
+            h['attrs'] ||= {}
+            h['attrs'][i.to_s] ||= {}
+            h['attrs'][i.to_s]['enc'] = 'b'
+          end
         end
         h[i.to_s] = v
         h
       end
     end
 
-    def encode_row_value(table_name, value)
-      if src_encoding = @table_meta[table_name][:encoding]
-        value.encode('utf-8', src_encoding, :undef => :replace, :invalid => :replace)
-      else
-        value.encode('utf-16', :undef => :replace, :invalid => :replace).encode('utf-8')
+    def encode_row_value(value)
+      value.force_encoding('utf-8')
+      unless value.valid_encoding?
+        # value can't be treated as UTF-8.  Convert to Flydata binary format.
+        value = "0x" + value.unpack('c*').collect{|c| "%02X" % (c < 0 ? c + 256 : c)}.join('')
+        value.force_encoding('binary')  # setting to 'binary' to differentiate from UTF-8.  Downstream code may rely on it.
       end
+      value
     end
 
     def encode_signless_integer(record, column_types, row_types)

data/lib/flydata/fluent-plugins/mysql/table_meta.rb

@@ -1,5 +1,7 @@
 require 'mysql2'
+require 'flydata-core/table_def/mysql_table_def'
 
+module Flydata
 module Mysql
   class TableMeta
     MANDATORY_OPTS = [
@@ -41,45 +43,14 @@ EOT
         database: @database, tables: @tables.collect{|t| "'#{t}'"}.join(',') }
       columns = conn.query(sql)
       columns.collect do |col|
-        @table_meta[col['table_name'].to_sym][:encoding] = get_ruby_charset_name(col['character_set_name'])
+        mysql_charset = col['character_set_name']
+        @table_meta[col['table_name'].to_sym][:encoding] = FlydataCore::TableDef::MysqlTableDef.ruby_encoding(mysql_charset)
+        @table_meta[col['table_name'].to_sym][:mysql_charset] = mysql_charset
       end
     ensure
       conn.close rescue nil if conn
     end
 
-    # Charset naming conversion rule. mysql => ruby
-    #
-    # mysql
-    #   http://dev.mysql.com/doc/refman/5.6/en/charset-charsets.html
-    # mysql(supported CJK character sets)
-    #   http://dev.mysql.com/doc/refman/5.6/en/faqs-cjk.html#qandaitem-A-11-1-1
-    # For ruby, you can see encoding list with "Encoding.list"
-    CHARSET_ENCODE_RULE = {
-      'ascii' => nil, # 'ASCII-8BIT', (not need to be encoded)
-      'utf8' => nil, # 'UTF-8', (not need to be encoded)
-      'utf8mb4' => nil, # 'UTF-8', (not need to be encoded)
-      'utf16' => 'UTF-16',
-      'utf32' => 'UTF-32',
-      'latin1' => 'ISO-8859-1',
-      'latin2' => 'ISO-8859-2',
-      'big5' => 'Big5',
-      'cp932' => 'CP932',
-      'eucjpms' => 'eucJP-ms',
-      'euckr' => 'EUC-KR',
-      'gb18030' => 'GB18030',
-      'gb2312' => 'GB2312',
-      'gbk' => 'GBK',
-      'sjis' => 'Shift_JIS',
-      'ujis' => 'EUC-JP',
-    }
-
-    def get_ruby_charset_name(mysql_charset)
-      return nil if mysql_charset.to_s.empty?
-      raise "Unsupported charset:#{mysql_charset}." unless CHARSET_ENCODE_RULE.has_key?(mysql_charset)
-      charset = CHARSET_ENCODE_RULE[mysql_charset]
-      return charset
-    end
-
     # Return table meta
     #   :character_set_name
     def [](table_name)
@@ -87,3 +58,4 @@ EOT
     end
   end
 end
+end

data/lib/flydata/{fluent-plugins/mysql → mysql}/binlog_position.rb

@@ -1,3 +1,4 @@
+module Flydata
 module Mysql
   class BinLogPosition
     include Comparable
@@ -18,3 +19,4 @@ module Mysql
     end
   end
 end
+end

data/lib/flydata/{util → mysql}/mysql_util.rb

@@ -1,5 +1,8 @@
+require 'open3'
+require 'flydata-core/table_def/mysql_table_def'
+
 module Flydata
-  module Util
+  module Mysql
     class MysqlUtil
       DEFAULT_MYSQL_CMD_OPTION = "--default-character-set=utf8 --protocol=tcp"
 
@@ -73,6 +76,36 @@ module Flydata
         generate_mysql_cmd(opt)
       end
 
+      def self.each_mysql_tabledef(tables, option)
+        command = generate_mysql_ddl_dump_cmd(option.merge(tables: tables))
+
+        create_opt = {}
+        if option.has_key?(:skip_primary_key_check)
+          create_opt[:skip_primary_key_check] = option[:skip_primary_key_check]
+        end
+
+        Open3.popen3(command) do |stdin, stdout, stderr|
+          stdin.close
+          stdout.set_encoding("utf-8", "utf-8") # mysqldump output must be in UTF-8
+          create_flydata_ctl_table = true
+          while !stdout.eof?
+            begin
+              mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, create_opt)
+              break if mysql_tabledef.nil?
+              yield(mysql_tabledef, nil)
+            rescue FlydataCore::TableDefError=> e
+              yield(nil, e)
+            end
+          end
+          errors = ""
+          while !stderr.eof?
+            line = stderr.gets.gsub('mysqldump: ', '')
+            errors << line unless /Warning: Using a password on the command line interface can be insecure./ === line
+          end
+          raise errors unless errors.empty?
+        end
+      end
+
       private
 
       def self.convert_keys_to_sym(hash)

data/lib/flydata/mysql/table_ddl.rb

@@ -0,0 +1,118 @@
+require 'flydata/mysql/binlog_position'
+require 'flydata/mysql/mysql_util'
+
+module Flydata
+module Mysql
+
+class TableDdl
+  VERSION0 = 0 # the version where no .generated_ddl file was generated.
+               # Therefore, this version never shows up in anywhere.
+  VERSION1 = 1 # the version which doesn't handle server side encoding support.
+  VERSION2 = 2 # the version with server side encoding support, migrated from
+               # the previous versions.  Format/functionality-wise, it's the
+               # same as Version 3.
+  VERSION3 = 3 # the version with server side encoding support, generated by
+               # sync:generated_table_ddl command.
+  VERSION4 = 4 # the version with server side encoding support, generated by
+               # the auto-generated CREATE TABLE event.
+  VERSION = VERSION3
+
+  def self.migrate_tables(tables, mysql_opts, sync_fm, position_file, context,
+                          &block)
+    migrate_to_v2(tables, mysql_opts, sync_fm, position_file, context, &block)
+  end
+
+  private
+
+  V2_TARGET_VERSION = VERSION2
+  EVENT_TYPE = 'Query'
+  ALTER_TABLE_CHARSET_SQL = <<EOS.strip
+ALTER TABLE `%s`.`%s` CHARACTER SET = %s;
+EOS
+  CHANGE_COLUMN_SQL = <<EOS.strip
+CHANGE COLUMN `%s` %s
+EOS
+  ALTER_TABLE_SQL = <<EOS.strip
+ALTER TABLE `%s`.`%s` %s;
+EOS
+
+  def self.migrate_to_v2(tables, mysql_opts, sync_fm, position_file, context)
+    database = mysql_opts[:database]
+    mysql_tabledefs = nil
+    original_binlog_file = nil
+    binlog_pos = nil
+    event_size = 0 # so that the next binlog position becomes the master binlog position
+    tables.each do |table|
+      # check the current ddl version
+      contents = sync_fm.load_generated_ddl([table])
+      version = contents.first.to_i
+      # return if no need to migrate
+      next if version >= V2_TARGET_VERSION
+
+      if mysql_tabledefs.nil?
+        MysqlUtil.each_mysql_tabledef(tables, mysql_opts) do |mysql_tabledef, error|
+          raise error if error
+
+          mysql_tabledefs ||= {}
+          mysql_tabledefs[mysql_tabledef.table_name] = mysql_tabledef
+        end
+      end
+      mysql_tabledef = mysql_tabledefs[table]
+      if binlog_pos.nil?
+        # get binlog position
+        binlog_pos = BinLogPosition.new(File.open(position_file){|f| f.read })
+        original_binlog_file = context.current_binlog_file
+        context.current_binlog_file = binlog_pos.file
+      end
+      # get charset
+      charset = mysql_tabledef.default_charset_mysql
+      # construct queries
+      # column charset
+      column_event = nil
+      at_subquery = mysql_tabledef.column_def.select{|col, coldef| /CHARACTER SET/.match(coldef) }.collect{|col, coldef| CHANGE_COLUMN_SQL % [col, coldef]}.join(",")
+      unless at_subquery.empty?
+        column_query = ALTER_TABLE_SQL % [database, table, at_subquery]
+        column_event = QueryEvent.new(EVENT_TYPE, database, table,
+                                      binlog_pos.pos, event_size, column_query,
+                                      Time.now.to_i)
+        yield column_event
+      end
+      # table charset
+      table_query = ALTER_TABLE_CHARSET_SQL % [database, table, charset]
+      # create events and yield
+      table_event = QueryEvent.new(EVENT_TYPE, database, table, binlog_pos.pos,
+                                   event_size, table_query, Time.now.to_i)
+      yield table_event
+      $log.info "migrating table `#{table}` from version #{version} to version #{V2_TARGET_VERSION}.  Table event #{table_event} Column event #{column_event}"
+
+      # update generated_ddl
+      sync_fm.save_generated_ddl([table], V2_TARGET_VERSION.to_s)
+    end
+    context.current_binlog_file = original_binlog_file if binlog_pos
+  end
+end
+
+# mimics RubyBinlog::QueryEvent
+class QueryEvent
+  def initialize(event_type, database_name, event_table_name, next_position,
+                 event_length, query, timestamp)
+    @event_type = event_type
+    @database_name = database_name
+    @event_table_name = event_table_name
+    @next_position = next_position
+    @event_length = event_length
+    @query = query
+    @timestamp = timestamp
+  end
+  attr_reader :event_type, :database_name, :event_table_name, :next_position,
+              :event_length, :query, :timestamp
+
+  def to_s
+    <<EOS
+event_type;#{event_type} database_name:#{database_name} event_table_name:#{event_table_name} next_position:#{next_position} event_length:#{event_length} timestamp:#{timestamp} query:#{query}
+EOS
+  end
+end
+
+end
+end

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -1,5 +1,5 @@
 require 'fiber'
-require 'flydata/util/mysql_util'
+require 'flydata/mysql/mysql_util'
 
 module Flydata
   module Parser
@@ -73,16 +73,16 @@ EOS
 
           begin
             # create pipe for callback function
-            rd_io, wr_io = IO.pipe("utf-8")
+            rd_io, wr_io = IO.pipe("utf-8", "utf-8")
             wr_io.sync = true
-            wr_io.set_encoding("utf-8")
+            wr_io.set_encoding("utf-8", "utf-8")
             rd_io.extend(DumpStreamIO)
             binlog_pos = nil
 
             # start mysqldump
             Open3.popen3 dump_cmd do |cmd_in, cmd_out, cmd_err, wait_thr|
               cmd_in.close_write
-              cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
+              cmd_out.set_encoding("utf-8", "utf-8") # mysqldump output must be in UTF-8
 
               first_line = cmd_out.gets # wait until first line comes
               binfile, pos = table_locker.resume
@@ -135,7 +135,7 @@ EOS
         end
 
         def generate_dump_cmd(conf, file_path = nil)
-          Util::MysqlUtil.generate_mysqldump_without_master_data_cmd(conf.merge(result_file: file_path))
+          Flydata::Mysql::MysqlUtil.generate_mysqldump_without_master_data_cmd(conf.merge(result_file: file_path))
         end
 
         private