flydata 0.6.11 → 0.6.12

data/lib/flydata/parser/source_table.rb

@@ -2,15 +2,16 @@ module Flydata
 module Parser
 
 class SourceTable
-  def initialize(table_name, columns = {}, primary_keys = [])
+  def initialize(table_name, columns = {})
     @table_name = table_name
     @columns = columns
     @column_names = columns.collect{|k,v| v[:column_name]}
-    @primary_keys = primary_keys
+    @primary_keys = [] # no longer used.  keeping the instance variable for
+                       # mashall dump compatibility
     @value_converters = {}
   end
 
-  attr_accessor :table_name, :columns, :column_names, :primary_keys, :value_converters
+  attr_accessor :table_name, :column_names, :value_converters
 
   def add_column(column)
     cn = column[:column_name]

data/lib/flydata/plugin_support/context.rb

@@ -0,0 +1,46 @@
+module Flydata
+module PluginSupport
+  class Context
+
+    def self.mandatory_opts
+      @mandatory_opts ||= []
+    end
+
+    def self.optional_opts
+      @optional_opts ||= []
+    end
+
+    def self.register_mandatory_opts(*opts)
+      @mandatory_opts ||= []
+      opts.each {|opt|
+        @mandatory_opts << opt.to_sym
+        attr_accessor opt
+      }
+      nil
+    end
+
+    def self.register_optional_opts(*opts)
+      @optional_opts ||= []
+      opts.each {|opt|
+        @optional_opts << opt.to_sym
+        attr_accessor opt
+      }
+      nil
+    end
+
+    register_mandatory_opts :tables, :tag, :sync_fm, :omit_events, :table_revs
+    register_optional_opts :current_binlog_file
+
+    def initialize(opts)
+      missing_opts = self.class.mandatory_opts - opts.keys
+      unless (missing_opts.empty?)
+        raise "Mandatory option(s) are missing: #{missing_opts.join(', ')}"
+      end
+
+      opts.each do |k, v|
+        self.instance_variable_set(:"@#{k}", v)
+      end
+    end
+  end
+end
+end

data/lib/flydata/plugin_support/sync_record_emittable.rb

@@ -0,0 +1,69 @@
+require 'flydata-core/record/record'
+
+module Flydata
+module PluginSupport
+  module SyncRecordEmittable
+    TABLE_NAME = :table_name # A Flydata JSON tag to specify a table name
+    TYPE = :type
+    SEQ = :seq
+    RESPECT_ORDER = :respect_order
+    SRC_POS = :src_pos
+    TABLE_REV = :table_rev
+    V = :v  # FlyData record format version
+
+    attr_accessor :context # required
+
+    # Public Interface: Emit sync records to fluent engine
+    #
+    #   "records" : A record or records for emitting
+    #               Each record needs to be Hash
+    #   "options"
+    #     tag       : (optional) tag (default: @context.tag)
+    #     timestamp : (optional) timestamp (default: current timestamp)
+    #     src_pos   : (required) source position (used for sync:repair)
+    #     table     : (optional) table name
+    #     increment_table_rev : (optional) set true when incrementing table revision
+    def emit_sync_records(records, options)
+      return if records.nil? # skip
+      records = [records] unless records.kind_of?(Array)
+
+      # Check options
+      tag = options[:tag] || @context.tag
+      timestamp = options[:timestamp] || Time.now.to_i
+      type = options[:type]
+      raise "type option must be set" if type.to_s.empty?
+      src_pos = options[:src_pos]
+      raise "src_pos option must be set" if src_pos.to_s.empty?
+
+      seq = nil
+      if table = options[:table]
+        table_rev = @context.table_revs[table]
+        if options[:increment_table_rev]
+          table_rev = @context.sync_fm.increment_table_rev(table, table_rev)
+          @context.table_revs[table] = table_rev
+        end
+        seq = @context.sync_fm.get_table_position(table)
+      end
+
+      # Add common information to each record
+      array = records.collect do |r|
+        r[TYPE] = type
+        r[RESPECT_ORDER] = true
+        r[SRC_POS] = src_pos
+        r[V] = FlydataCore::Record::V2
+
+        if table
+          seq = @context.sync_fm.increment_table_position(seq)
+          r[SEQ] = seq
+          r[TABLE_NAME] = table
+          r[TABLE_REV] = table_rev
+        end
+        [timestamp, r]
+      end
+      Fluent::Engine.emit_array(tag, array)
+      @context.sync_fm.save_table_position(table, seq) if table
+    end
+
+  end
+end
+end

data/lib/flydata/source/component.rb

@@ -13,7 +13,7 @@ class Component
 
   protected
 
-  attr_reader :options
+  attr_reader :source, :options
   def_delegators :@source, :de
 end
 

data/lib/flydata/source/generate_source_dump.rb

@@ -55,8 +55,9 @@ class GenerateSourceDump < Component
   #
   # tables: An array of tables to be dumped.
   # file_path: A file path string of the dump file to which data is written.
-  #         This value may be nil, in which case contents are written to a
-  #         pipe.
+  #         This value may be nil, in which case contents are written to
+  #         a pipe to which the caller can access via `io` passed in the
+  #         callback.
   # src_pos_callback: A callback called when the source position of the dump
   #         becomes available. The callback takes the following arguments.
   #           io: Input IO to the dump.

data/lib/flydata/source_mysql/mysql_compatibility_check.rb

@@ -43,17 +43,24 @@ module SourceMysql
         stdin.close
         while !stderr.eof?
           lines = []
-          while line = stderr.gets; lines << line.strip; end
-          err_reason = lines.join(" ")
-          log_error("Error occured during access to mysql server.", {err: err_reason})
-
-          unless /Warning: Using a password on the command line interface can be insecure/ === err_reason
+          while line = stderr.gets do
+            lines << line.strip unless line =~ /Warning: Using a password on the command line interface can be insecure/
+          end
+          unless lines.empty?
+            err_reason = lines.join(" ")
+            log_error("Error occured during access to mysql server.", err: err_reason)
             raise FlydataCore::MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n      $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
           end
         end
       end
     end
 
+    def check_rds_master_status
+      if is_rds?
+        FlydataCore::Mysql::RdsMasterStatusChecker.new(@db_opts).do_check
+      end
+    end
+
     def check_mysql_parameters_compat
       begin
         FlydataCore::Mysql::OptionalBinlogParameterChecker.new(@db_opts).do_check
@@ -63,12 +70,6 @@ module SourceMysql
       FlydataCore::Mysql::RequiredBinlogParameterChecker.new(@db_opts).do_check
     end
 
-    def check_rds_master_status
-      if is_rds?
-        FlydataCore::Mysql::RdsMasterStatusChecker.new(@db_opts).do_check
-      end
-    end
-
     def check_mysql_binlog_retention
       if is_rds?
         run_rds_retention_check

data/lib/flydata/source_mysql/parser/dump_parser.rb

@@ -326,10 +326,6 @@ EOS
               create_table_block.call(current_table)
               current_state = Flydata::Parser::State::INSERT_RECORD
               check_point_block.call(current_table, dump_io.pos, bytesize, @binlog_pos, current_state)
-            elsif m = /^PRIMARY KEY \((?<primary_keys>[^\)]+)\)/.match(line)
-              current_table.primary_keys = m[:primary_keys].split(',').collect do |pk_str|
-                pk_str[1..-2]
-              end
             end
           end
 

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/alter_table_query_handler.rb

@@ -1,7 +1,10 @@
 require 'flydata/parser/parser_provider'
-require 'flydata/fluent-plugins/mysql/ddl_query_handler'
+require 'flydata/source_mysql/plugin_support/ddl_query_handler'
 
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
   class AlterTableQueryHandler < TableDdlQueryHandler
     PATTERN = /^ALTER TABLE/i
 
@@ -39,3 +42,6 @@ EOS
     end
   end
 end
+
+end
+end

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_position_file.rb

@@ -1,4 +1,7 @@
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
   class BinLogPositionFile
     attr_accessor :path
 
@@ -15,3 +18,6 @@ module Mysql
     end
   end
 end
+
+end
+end

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_dispatcher.rb

@@ -1,8 +1,11 @@
-require 'flydata/fluent-plugins/mysql/alter_table_query_handler'
-require 'flydata/fluent-plugins/mysql/truncate_table_query_handler'
-require 'flydata/fluent-plugins/mysql/drop_database_query_handler'
+require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
+require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
+require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
 
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
   class BinlogQueryDispatcher
     def initialize
       @handlers = []
@@ -64,3 +67,6 @@ module Mysql
     end
   end
 end
+
+end
+end

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_handler.rb

@@ -1,6 +1,9 @@
-require 'flydata/fluent-plugins/mysql/binlog_record_handler'
+require 'flydata/source_mysql/plugin_support/binlog_record_handler'
 
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
   class BinlogQueryHandler < BinlogRecordHandler
     # Return regexp
     # This class will be used if the pattern matches with the query
@@ -9,3 +12,6 @@ module Mysql
     end
   end
 end
+
+end
+end

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_record_dispatcher.rb

@@ -1,9 +1,12 @@
 require 'fluent/plugin/in_mysql_binlog'
 require 'binlog'
-require 'flydata/fluent-plugins/mysql/dml_record_handler'
-require 'flydata/fluent-plugins/mysql/binlog_query_dispatcher'
+require 'flydata/source_mysql/plugin_support/dml_record_handler'
+require 'flydata/source_mysql/plugin_support/binlog_query_dispatcher'
 
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
   class BinlogRecordDispatcher
     def dispatch(event)
       method_name = "on_#{event.event_type.downcase}"
@@ -48,3 +51,6 @@ module Mysql
     end
   end
 end
+
+end
+end

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_record_handler.rb

@@ -1,17 +1,14 @@
 require 'fluent/plugin/in_mysql_binlog'
 require 'binlog'
-require 'flydata-core/record/record'
 require 'flydata-core/mysql/binlog_pos'
+require 'flydata/plugin_support/sync_record_emittable'
 
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
   class BinlogRecordHandler
-    TABLE_NAME = :table_name # A Flydata JSON tag to specify a table name
-    TYPE = :type
-    SEQ = :seq
-    RESPECT_ORDER = :respect_order
-    SRC_POS = :src_pos
-    TABLE_REV = :table_rev
-    V = :v  # FlyData record format version
+    include Flydata::PluginSupport::SyncRecordEmittable
 
     def initialize(context)
       @context = context
@@ -29,6 +26,7 @@ module Mysql
     end
 
     private
+
     def binlog_pos(record)
       "#{@context.current_binlog_file}\t#{record['next_position'] - record['event_length']}"
     end
@@ -70,34 +68,15 @@ module Mysql
       return if records.nil? # skip
       records = [records] unless records.kind_of?(Array)
 
-      table = records.first[TABLE_NAME] || record['table_name']
+      table = records.first[:table_name] || record['table_name']
       raise "Missing table name. #{record}" if table.to_s.empty?
       return unless acceptable_table?(record, table) && acceptable_event?(type, table)
 
-      table_rev = @context.table_revs[table]
-
-      # Add common information to each record
-      records.each do |r|
-        if opt[:increment_table_rev]
-          table_rev = @context.sync_fm.increment_table_rev(table, table_rev)
-          @context.table_revs[table] = table_rev
-        end
-        r[TYPE] = type
-        r[RESPECT_ORDER] = true
-        r[TABLE_NAME] = table
-        r[SRC_POS] = binlog_pos(record)
-        r[TABLE_REV] = table_rev
-        r[V] = FlydataCore::Record::V2
-      end
-
-      # Use binlog's timestamp
-      timestamp = record["timestamp"].to_i
-      records.each do |row|
-        @context.sync_fm.increment_and_save_table_position(row[TABLE_NAME]) do |seq|
-          row[SEQ] = seq
-          Fluent::Engine.emit(@context.tag, timestamp, row)
-        end
-      end
+      emit_sync_records(records, opt.merge(
+        timestamp: record["timestamp"].to_i,
+        type: type,
+        table: table,
+        src_pos: binlog_pos(record)))
     end
 
     def check_empty_binlog
@@ -113,3 +92,6 @@ module Mysql
     end
   end
 end
+
+end
+end

data/lib/flydata/source_mysql/plugin_support/context.rb

@@ -0,0 +1,7 @@
+require 'flydata/plugin_support/context'
+
+module Flydata::SourceMysql::PluginSupport
+  class Context < ::Flydata::PluginSupport::Context
+    register_mandatory_opts :database, :table_meta
+  end
+end

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/ddl_query_handler.rb

@@ -1,6 +1,9 @@
-require 'flydata/fluent-plugins/mysql/binlog_query_handler'
+require 'flydata/source_mysql/plugin_support/binlog_query_handler'
 
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
 
 class DdlQueryHandler < BinlogQueryHandler
   DDL_TABLE_QUERY = /^(?:(?:ALTER|CREATE|DROP|RENAME) +(?:\w+ +)*TABLE +([^ ]+)|TRUNCATE +(?:TABLE +)?([^ ;]+))/i
@@ -39,32 +42,21 @@ end
 class DatabaseDdlQueryHandler < DdlQueryHandler
   def emit_record(type, record)
     return unless acceptable_db?(record)
-
     check_empty_binlog
 
     opt = {}
     records = yield(opt) # The block may set options as necessary
     return if records.nil? # skip
     records = [records] unless records.kind_of?(Array)
-
-    database = records.first[DB_NAME] || record['db_name']
-
     return unless acceptable_event?(type)
 
-    # Add common information to each record
-    records.each do |r|
-      r[TYPE] = type
-      r[RESPECT_ORDER] = true
-      r[SRC_POS] = binlog_pos(record)
-      r[V] = FlydataCore::Record::V2
-    end
-
-    # Use binlog's timestamp
-    timestamp = record["timestamp"].to_i
-    records.each do |row|
-      Fluent::Engine.emit(@context.tag, timestamp, row)
-    end
+    emit_sync_records(records, opt.merge(
+      timestamp: record["timestamp"].to_i,
+      type: type,
+      src_pos: binlog_pos(record)))
   end
 end
 
 end
+end
+end