flydata 0.1.7 → 0.1.8

data/lib/flydata/fluent-plugins/mysql/binlog_query_handler.rb

@@ -0,0 +1,11 @@
+require_relative 'binlog_record_handler'
+
+module Mysql
+  class BinlogQueryHandler < BinlogRecordHandler
+    # Return regexp
+    # This class will be used if the pattern matches with the query
+    def pattern
+      raise "Not implemented."
+    end
+  end
+end

data/lib/flydata/fluent-plugins/mysql/binlog_record_dispatcher.rb

@@ -0,0 +1,50 @@
+require 'fluent/plugin/in_mysql_binlog'
+require 'binlog'
+require_relative 'dml_record_handler'
+require_relative 'binlog_query_dispatcher'
+
+module Mysql
+  class BinlogRecordDispatcher
+    def dispatch(event)
+      method_name = "on_#{event.event_type.downcase}"
+      if self.respond_to?(method_name)
+        # TODO to_hash method call below can fail if event.event_type is
+        # "Update_rows".  This seems to be a bug of ruby-binlog.  The bug must
+        # be fixed when we support record update.
+        record = Fluent::MysqlBinlogInput::BinlogUtil.to_hash(event)
+        self.send(method_name, record)
+      else
+        # $log.trace "Unhandled type: #{record["event_type"]}"
+      end
+    end
+  end
+
+  class FlydataBinlogRecordDispatcher < BinlogRecordDispatcher
+    def initialize(context)
+      context.current_binlog_file = ""
+      @context = context
+      @query_dispatcher = FlydataBinlogQueryDispatcher.new(context)
+      @dml_record_handler = DmlRecordHandler.new(context)
+    end
+
+    def on_rotate(record)
+      @context.current_binlog_file = record["binlog_file"]
+    end
+
+    def on_write_rows(record)
+      @dml_record_handler.process(record, :write_rows)
+    end
+
+    def on_update_rows(record)
+      @dml_record_handler.process(record, :update_rows)
+    end
+
+    def on_delete_rows(record)
+      @dml_record_handler.process(record, :delete_rows)
+    end
+
+    def on_query(record)
+      @query_dispatcher.dispatch(record)
+    end
+  end
+end

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -0,0 +1,100 @@
+require 'fluent/plugin/in_mysql_binlog'
+require 'binlog'
+require_relative 'binlog_position'
+require_relative 'query_parser'
+
+module Mysql
+  class BinlogRecordHandler
+    TABLE_NAME = :table_name # A Flydata JSON tag to specify a table name
+    TYPE = :type
+    SEQ = :seq
+    RESPECT_ORDER = :respect_order
+    SRC_POS = :src_pos
+    TABLE_REV = :table_rev
+
+    def initialize(context)
+      @context = context
+      @first_empty_binlog = true
+
+      # Load per-table binlog position
+      @table_binlog_pos = {}
+      @context.tables.each do |table_name|
+        table_binlog_content = @context.sync_fm.get_table_binlog_pos(table_name)
+        if table_binlog_content
+          @table_binlog_pos[table_name] = BinLogPosition.new(table_binlog_content)
+        end
+      end
+    end
+
+    private
+
+    def acceptable_db?(db_name)
+      @context.database == db_name
+    end
+
+    def acceptable_table?(record, table)
+      acceptable = @context.tables.include?(table)
+
+      if acceptable and @table_binlog_pos[record['table_name']]
+        if @table_binlog_pos[record['table_name']] >= BinLogPosition.new(
+          "#{@context.current_binlog_file}\t#{record['next_position'] - record['event_length']}")
+          acceptable = false
+        else
+          @context.sync_fm.delete_table_binlog_pos(record['table_name'])
+          @table_binlog_pos.delete(record['table_name'])
+        end
+      end
+      acceptable
+    end
+
+    def emit_record(type, record, opt = {})
+      return unless acceptable_db?(record["db_name"])
+      return unless record["table_name"].nil? or acceptable_table?(record, record["table_name"])
+
+      check_empty_binlog
+
+      records = yield
+      records = [records] unless records.kind_of?(Array)
+
+      table = records.first[TABLE_NAME] || record['table_name']
+      raise "Missing table name. #{record}" if table.to_s.empty?
+      return unless acceptable_table?(record, table)
+
+      table_rev = @context.sync_fm.table_rev(table)
+      position = record['next_position'] - record['event_length']
+
+      # Add common information to each record
+      records.each do |r|
+        if opt[:increment_table_rev]
+          table_rev = @context.sync_fm.increment_table_rev(table, table_rev)
+        end
+        r[TYPE] = type
+        r[RESPECT_ORDER] = true
+        r[TABLE_NAME] = table
+        r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
+        r[TABLE_REV] = table_rev
+      end
+
+      # Use binlog's timestamp
+      timestamp = record["timestamp"].to_i
+      records.each do |row|
+        @context.sync_fm.increment_and_save_table_position(row[TABLE_NAME]) do |seq|
+          row[SEQ] = seq
+          Fluent::Engine.emit(@context.tag, timestamp, row)
+        end
+      end
+    end
+
+    def check_empty_binlog
+      #Log one warning per consecutive records that have empty binlog filename
+      if @context.current_binlog_file.to_s.empty?
+        if @first_empty_binlog
+          $log.warn "Binlog file name is empty. Rotate event not received!"
+          @first_empty_binlog = false
+        end
+      else
+        @first_empty_binlog = true
+      end
+    end
+  end
+end

data/lib/flydata/fluent-plugins/mysql/context.rb

@@ -0,0 +1,25 @@
+module Mysql
+  class Context
+    MANDATORY_OPTS = [
+      :database, :tables, :tag, :sync_fm,
+    ]
+    OPTIONAL_OPTS = [
+      :current_binlog_file,
+    ]
+
+    (MANDATORY_OPTS + OPTIONAL_OPTS).each do |opt|
+      attr_accessor opt
+    end
+
+    def initialize(opts)
+      missing_opts = MANDATORY_OPTS - opts.keys
+      unless (missing_opts.empty?)
+        raise "Mandatory option(s) are missing: #{missing_opts.join(', ')}"
+      end
+
+      opts.each do |k, v|
+        self.instance_variable_set(:"@#{k}", v)
+      end
+    end
+  end
+end

data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb

@@ -0,0 +1,82 @@
+require_relative 'binlog_record_handler'
+
+module Mysql
+  class DmlRecordHandler < BinlogRecordHandler
+    ROW = :row
+    INTEGER_TYPES = {'TINY' => 1,
+                     'SHORT' => 2,
+                     'INT24' => 3,
+                     'LONG' => 4,
+                     'LONGLONG' => 8
+    }
+    SIGNLESS_INTEGER_PREFIX = '0SL'
+
+    def process(record, type)
+      case type
+      when :write_rows
+        emit_insert(record)
+      when :delete_rows
+        emit_delete(record)
+      when :update_rows
+        emit_update(record)
+      else
+        raise "Invalid type:#{type}"
+      end
+    end
+
+    private
+
+    def emit_insert(record)
+      emit_rows(:insert, record)
+    end
+
+    def emit_delete(record)
+      emit_rows(:delete, record)
+    end
+
+    def emit_update(record)
+      emit_rows(:update, record) do |row|
+        row.last # For update, row has two arrays (old and new values)  Use new values
+      end
+    end
+
+    def emit_rows(type, record)
+      emit_record(type, record) do |table|
+        records = record["rows"].collect do |row|
+          row = yield(row) if block_given?  # Give the caller a chance to generate the correct row
+          { ROW => convert_to_flydata_row_format(row) }
+        end
+        encode_signless_integer(records, record["columns"])
+        records
+      end
+    end
+
+    def convert_to_flydata_row_format(row)
+      row.each.with_index(1).inject({}) do |h, (v, i)|
+        if v.kind_of?(String)
+          v = v.encode('utf-16', :undef => :replace, :invalid => :replace).encode('utf-8')
+        end
+        h[i.to_s] = v
+        h
+      end
+    end
+
+    def encode_signless_integer(records, column_types)
+      records.each do |record|
+        record[ROW].keys.each do |position|
+          index = position.to_i - 1
+          column_type = column_types[index]
+          if INTEGER_TYPES.keys.include?(column_type)
+            # It's a signless integer.
+            intval = record[ROW][position]
+            next unless (intval.kind_of?(Numeric) || intval =~ /^-?[\d]+$/)
+            width = INTEGER_TYPES[column_type] * 2 # * 2 because a single byte requires two characters (e.g. ff)
+            signless_val = SIGNLESS_INTEGER_PREFIX
+            signless_val += sprintf("%0#{width}x", intval).gsub(/\.\.f/, 'f' * width).slice(-width..-1)
+            record[ROW][position] = signless_val
+          end
+        end
+      end
+    end
+  end
+end

data/lib/flydata/fluent-plugins/mysql/query_parser.rb

@@ -0,0 +1,69 @@
+require_relative '../../table_def'
+
+module Mysql
+  class QueryParser
+    # Return hash object
+    def parse(query)
+    end
+  end
+
+  class AlterTableAddColumnParser < QueryParser
+    # Return hash object or array
+    # {
+    #   alter_action : "add_column",
+    #   table_name : table_name,
+    #   column : {
+    #     name: column_name,
+    #     type: column_type,
+    #     not_null: [true|false],
+    #     default: default_value,
+    #     position: [first|last],
+    #     after: column_name
+    #   }
+    # }
+    def parse(query)
+      do_parse(query)
+    rescue
+      $log.error("Failed to parse query. query:'#{query}' error:#{$!}")
+      raise
+    end
+
+    private
+
+    def do_parse(query)
+      m = /^\s*alter\s+table\s+`?(?<table_name>[^\s]+)`?\s+add\s+column\s+/i.match(query)
+      table_name = m['table_name']
+      query = query[m[0].length..-1]
+
+      columns = if query[0] == '('
+                  query = query[1..-2] # delete parenthesis (...,...,...)
+                  parse_multiple_columns(query)
+                else
+                  [parse_one_column(query)]
+                end
+      columns.collect do |col|
+        {
+          subtype: :add_column,
+          table_name: table_name,
+          column: col
+        }
+      end
+    end
+
+    def parse_one_column(query)
+      Flydata::TableDef::MysqlTableDef.parse_one_column_def(query) do |column, query, pos|
+        option_str = " " + query[pos..-1]
+        if /\sFIRST/i.match(option_str)
+          column[:position] = :first
+        elsif /\sAFTER\s+`?([^`]+)`?/i.match(option_str)
+          column[:after] = $1
+        end
+        column
+      end
+    end
+
+    def parse_multiple_columns(query)
+      raise "Not supported to add multiple columns"
+    end
+  end
+end

data/lib/flydata/sync_file_manager.rb

@@ -2,26 +2,16 @@ module Flydata
   module FileUtil
     class SyncFileManager
       DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
+      BACKUP_DIR = ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
       TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
       def initialize(data_entry)
         @data_entry = data_entry
       end
 
       def dump_file_path
-        dump_dir = @data_entry['mysql_data_entry_preference']['mysqldump_dir']
-        if dump_dir
-          dump_dir = dump_dir.dup
-          dump_dir[0] = ENV['HOME'] if dump_dir.match(/^~$|^~\//)
-        else
-          dump_dir = DUMP_DIR.dup
-        end
-        if File.exists?(dump_dir) and not Dir.exists?(dump_dir)
-          raise "'mysqldump_dir'(#{dump_dir}) must be a directory."
-        end
-        FileUtils.mkdir_p(dump_dir) unless Dir.exists?(dump_dir)
         File.join(dump_dir, @data_entry['name']) + ".dump"
       end
-
+            
       # dump pos file for resume
       def dump_pos_path
         dump_file_path + ".pos"
@@ -67,6 +57,13 @@ module Flydata
         File.join(FLYDATA_HOME, @data_entry['name'] + ".binlog.pos")
       end
 
+      def reset_table_position_files(tables)
+        tables.each do |table_name|
+          file = File.join(table_positions_dir_path, table_name + ".pos")
+          File.open(file, "w") {|f| f.write('0') }
+        end
+      end
+      
       def table_positions_dir_path
         TABLE_POSITIONS_DIR
       end
@@ -99,7 +96,97 @@ module Flydata
           retry
         end
       end
+      
+      def sync_info_file
+        File.join(dump_dir, "sync.info")
+      end
+      
+      def save_sync_info(initial_sync, tables)
+        File.open(sync_info_file, "w") do |f|
+          f.write([initial_sync, tables].join("\t"))
+        end        
+      end
+      
+      def load_sync_info
+        return nil unless File.exists?(sync_info_file)
+        items = File.open(sync_info_file, 'r').readline.split("\t")
+        { initial_sync: (items[0] == 'true'),
+          tables: items[1] }
+      end
+      
+      def get_table_binlog_pos(table_name)
+        file = File.join(table_positions_dir_path, table_name + ".binlog.pos")
+        return nil unless File.exists?(file)
+        File.open(file, 'r').readline
+      end
+
+      def table_rev_file_path(table_name)
+        File.join(table_positions_dir_path, table_name + ".rev")
+      end
+
+      def table_rev_file_paths
+        Dir.glob(File.join(table_positions_dir_path, "*.rev"))
+      end
+
+      def table_rev(table_name)
+        file = table_rev_file_path(table_name)
+        return 1 unless File.exists?(file) #default revision is 1
+        File.open(file, "r+") do |f|
+          seq = f.read
+          if seq.empty?
+            return 1
+          else
+            return seq.to_i
+          end
+        end
+      end
+
+      def increment_table_rev(table_name, base_rev)
+        file = table_rev_file_path(table_name)
+        new_rev = base_rev + 1
+        File.open(file, "w") do |f|
+          f.write(new_rev)
+        end
+        new_rev
+      end
 
+      def delete_table_binlog_pos(table_name)
+        file = File.join(table_positions_dir_path, table_name + ".binlog.pos")
+        if File.exists?(file)
+          FileUtils.rm(file, :force => true)
+        else
+          puts "#{file} does not exist. Something is wrong. Did you delete the file manually when flydata was running?"
+        end  
+      end
+      
+      def save_table_binlog_pos(tables, binlog_pos)        
+        tables.split(" ").each do |table_name|
+          file = File.join(dump_dir, table_name + ".binlog.pos")
+          File.open(file, "w") do |f|
+            f.write(binlog_content(binlog_pos))
+          end
+        end
+      end
+      
+      def move_table_binlog_files(tables)
+        FileUtils.mkdir_p(table_positions_dir_path) unless Dir.exists?(table_positions_dir_path)
+        tables.each do |table_name|
+          file = File.join(dump_dir, table_name + ".binlog.pos")
+          if ! File.exists?(file)
+            raise "#{file} does not exist. Error!!"
+          end
+          FileUtils.mv(file, table_positions_dir_path)
+        end
+      end
+      
+      def backup_dump_dir
+        backup_dir = BACKUP_DIR.dup
+        FileUtils.mkdir_p(backup_dir) unless Dir.exists?(backup_dir)
+        dest_dir = File.join(backup_dir, Time.now.strftime("%Y%m%d%H%M%S"))
+        FileUtils.mkdir(dest_dir)        
+        FileUtils.mv(dump_dir, dest_dir)
+      end
+            
       private
 
       def dump_pos_content(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
@@ -115,6 +202,26 @@ module Flydata
         return nil unless File.exists?(path)
         Marshal.load(File.open(path, 'r'))
       end
+
+      def dump_dir
+        pref = @data_entry['mysql_data_entry_preference']
+        dump_dir = if pref and pref['mysqldump_dir']
+                     pref['mysqldump_dir']
+                   else
+                     nil
+                   end
+        if dump_dir
+          dump_dir = dump_dir.dup
+          dump_dir[0] = ENV['HOME'] if dump_dir.match(/^~$|^~\//)
+        else
+          dump_dir = DUMP_DIR.dup
+        end
+        if File.exists?(dump_dir) and not Dir.exists?(dump_dir)
+          raise "'mysqldump_dir'(#{dump_dir}) must be a directory."
+        end
+        FileUtils.mkdir_p(dump_dir) unless Dir.exists?(dump_dir)
+        dump_dir      
+      end
     end
   end
 end