flydata 0.6.3 → 0.6.4

data/lib/flydata/compatibility_check.rb

@@ -1,8 +1,4 @@
-require 'mysql2'
 require 'flydata/command_loggable'
-require 'flydata-core/mysql/config'
-require 'flydata-core/mysql/command_generator'
-require 'flydata-core/mysql/compatibility_checker'
 require 'flydata-core/errors'
 
 module Flydata
@@ -39,6 +35,30 @@ module Flydata
 
   end
 
+  # Collection of checks for initial sync
+  module InitialSyncChecks
+    # Checks permission of @dump_dir and @backup_dir
+    #
+    # Requires
+    #   @dump_dir
+    #   @backup_dir
+    def check_writing_permissions
+      write_errors = []
+      paths_to_check = [FLYDATA_HOME]
+      paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
+      paths_to_check << @backup_dir unless @backup_dir.to_s.empty?
+      paths_to_check.each do |path|
+        full_path = File.expand_path(path)
+        full_path = File.dirname(full_path) unless File.directory?(full_path)
+        write_errors << full_path unless File.writable?(full_path) and File.executable?(full_path)
+      end
+      unless write_errors.empty?
+        error_dir = write_errors.join(", ")
+        raise FlydataCore::CompatibilityError, "We cannot access the directories: #{error_dir}"
+      end
+    end
+  end
+
   class AgentCompatibilityCheck < CompatibilityCheck
 
     TCP_PORT=45326
@@ -69,117 +89,4 @@ module Flydata
     end
   end
 
-  class MysqlCompatibilityCheck < CompatibilityCheck
-
-    def initialize(dp_hash, de_hash, options={})
-      super
-      @db_opts = FlydataCore::Mysql::Config.build_mysql_db_opts(de_hash)
-      @dump_dir = options[:dump_dir] || nil
-      @backup_dir = options[:backup_dir] || nil
-      @tables = de_hash['tables']
-    end
-
-    def print_errors
-      return if @errors.empty?
-      log_error_stderr "There may be some compatibility issues with your MySQL credentials: "
-      @errors.each do |error|
-        log_error_stderr "  * #{error.message}"
-      end
-      raise "Please correct these errors if you wish to run FlyData Sync"
-    end
-
-    def check_mysql_user_compat
-      FlydataCore::Mysql::SyncPermissionChecker.new(@db_opts).do_check
-    end
-
-    def check_mysql_protocol_tcp_compat
-      query = FlydataCore::Mysql::CommandGenerator.generate_mysql_show_grants_cmd(@db_opts)
-
-      Open3.popen3(query) do |stdin, stdout, stderr|
-        stdin.close
-        while !stderr.eof?
-          lines = []
-          while line = stderr.gets; lines << line.strip; end
-          err_reason = lines.join(" ")
-          log_error("Error occured during access to mysql server.", {err: err_reason})
-
-          unless /Warning: Using a password on the command line interface can be insecure/ === err_reason
-            raise FlydataCore::MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n      $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
-          end
-        end
-      end
-    end
-
-    def check_mysql_parameters_compat
-      begin
-        FlydataCore::Mysql::OptionalBinlogParameterChecker.new(@db_opts).do_check
-      rescue FlydataCore::MysqlCompatibilityError => e
-        log_warn_stderr(e.to_s)
-      end
-      FlydataCore::Mysql::RequiredBinlogParameterChecker.new(@db_opts).do_check
-    end
-
-    def check_rds_master_status
-      if is_rds?
-        FlydataCore::Mysql::RdsMasterStatusChecker.new(@db_opts).do_check
-      end
-    end
-
-    def check_mysql_binlog_retention
-      if is_rds?
-        run_rds_retention_check
-      else
-        run_mysql_retention_check
-      end
-    end
-
-    def check_writing_permissions
-      write_errors = []
-      paths_to_check = [FLYDATA_HOME]
-      paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
-      paths_to_check << @backup_dir unless @backup_dir.to_s.empty?
-      paths_to_check.each do |path|
-        full_path = File.expand_path(path)
-        full_path = File.dirname(full_path) unless File.directory?(full_path)
-        write_errors << full_path unless File.writable?(full_path) and File.executable?(full_path)
-      end
-      unless write_errors.empty?
-        error_dir = write_errors.join(", ")
-        raise FlydataCore::MysqlCompatibilityError, "We cannot access the directories: #{error_dir}"
-      end
-    end
-
-    # If table_type='VIEW' or engine='MEMORY', raise error.
-    def check_mysql_table_types
-      return if @tables.empty?
-      option = @db_opts.dup.merge(tables: @tables)
-      FlydataCore::Mysql::TableTypeChecker.new(option).do_check
-    end
-
-    def run_mysql_retention_check
-      FlydataCore::Mysql::NonRdsRetentionChecker.new(@db_opts).do_check
-    end
-
-    def run_rds_retention_check
-      FlydataCore::Mysql::RdsRetentionChecker.new(@db_opts).do_check
-    rescue Mysql2::Error => e
-      if e.message =~ /command denied to user/
-        retention_hours = FlydataCore::Mysql::RdsRetentionChecker::BINLOG_RETENTION_HOURS
-        log_warn_stderr("[WARNING]Cannot verify RDS retention period on current MySQL user account.\n" +
-                        "To see retention period, please run this on your RDS:\n" +
-                        "  $> call mysql.rds_show_configuration;\n" +
-                        "Please verify that the hours is not nil and is at least #{retention_hours} hours\n" +
-                        "To set binlog retention hours, you can run this on your RDS:\n" +
-                        "  $> call mysql.rds_set_configuration('binlog retention hours', #{retention_hours});\n"
-                       )
-      else
-        raise e
-      end
-    end
-
-    def is_rds?(hostname = @db_opts[:host])
-      hostname.match(/rds.amazonaws.com$/) != nil
-    end
-  end
-
 end

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -16,7 +16,7 @@ require 'flydata/fluent-plugins/mysql/binlog_record_dispatcher'
 require 'flydata/fluent-plugins/mysql/context'
 require 'flydata/fluent-plugins/idle_event_detector'
 require 'flydata/fluent-plugins/mysql/table_meta'
-require 'flydata/mysql/table_ddl'
+require 'flydata/source_mysql/table_ddl'
 require 'flydata-core/fluent/config_helper'
 require 'flydata-core/mysql/ssl'
 
@@ -83,7 +83,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     end
 
     @sync_fm = Flydata::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
-    @sent_position_file_path = @sync_fm.sent_binlog_path(@position_file)
+    @sent_position_file_path = @sync_fm.sent_source_pos_path(@position_file)
 
     load_custom_conf
 
@@ -184,7 +184,7 @@ EOS
     return if instance_variable_defined? :@abort
 
     @context.table_meta.update
-    Flydata::Mysql::TableDdl.migrate_tables(@context.tables, @db_opts,
+    Flydata::SourceMysql::TableDdl.migrate_tables(@context.tables, @db_opts,
                                             @context.sync_fm, @position_file,
                                             @context) do |event|
       @record_dispatcher.dispatch(event)

data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb

@@ -1,4 +1,4 @@
-require 'flydata/parser_provider'
+require 'flydata/parser/parser_provider'
 require 'flydata/fluent-plugins/mysql/ddl_query_handler'
 
 module Mysql
@@ -17,7 +17,7 @@ module Mysql
       emit_record(:alter_table, record) do |opt|
         ret = nil
         begin
-          result = ParserProvider.parser(:mysql, :mysql_alter_table).new.parse(record["normalized_query"])
+          result = Flydata::Parser::ParserProvider.parser(:mysql, :mysql_alter_table).new.parse(record["normalized_query"])
           if result.nil?
             $log.error("Received unsupported alter table query. normalized query:'#{record['normalized_query']}', raw query: '#{record['query']}'")
           else

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -1,7 +1,7 @@
 require 'fluent/plugin/in_mysql_binlog'
 require 'binlog'
-require 'flydata/mysql/binlog_position'
 require 'flydata-core/record/record'
+require 'flydata-core/mysql/binlog_pos'
 
 module Mysql
   class BinlogRecordHandler
@@ -21,9 +21,9 @@ module Mysql
       # Load per-table binlog position
       @table_binlog_pos = {}
       @context.tables.each do |table_name|
-        table_binlog_content = @context.sync_fm.get_table_binlog_pos(table_name)
-        if table_binlog_content
-          @table_binlog_pos[table_name] = Flydata::Mysql::BinLogPosition.new(table_binlog_content)
+        table_binlog_pos_str = @context.sync_fm.get_table_source_raw_pos(table_name)
+        if table_binlog_pos_str
+          @table_binlog_pos[table_name] = FlydataCore::Mysql::BinlogPos.new(table_binlog_pos_str)
         end
       end
     end
@@ -45,10 +45,10 @@ module Mysql
       acceptable = @context.tables.include?(table)
 
       if acceptable and @table_binlog_pos[record['table_name']]
-        if @table_binlog_pos[record['table_name']] >= Flydata::Mysql::BinLogPosition.new(binlog_pos(record))
+        if @table_binlog_pos[record['table_name']] >= FlydataCore::Mysql::BinlogPos.new(binlog_pos(record))
           acceptable = false
         else
-          @context.sync_fm.delete_table_binlog_pos(record['table_name'])
+          @context.sync_fm.delete_table_source_pos(record['table_name'])
           @table_binlog_pos.delete(record['table_name'])
         end
       end

data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb

@@ -1,4 +1,3 @@
-require 'flydata/parser_provider'
 require 'flydata/fluent-plugins/mysql/ddl_query_handler'
 
 module Mysql

data/lib/flydata/parser.rb

@@ -0,0 +1,14 @@
+module Flydata
+
+module Parser
+  module State
+    START = 'START'
+    CREATE_TABLE = 'CREATE_TABLE'
+    CREATE_TABLE_COLUMNS = 'CREATE_TABLE_COLUMNS'
+    CREATE_TABLE_CONSTRAINTS = 'CREATE_TABLE_CONSTRAINTS'
+    INSERT_RECORD = 'INSERT_RECORD'
+    PARSING_INSERT_RECORD = 'PARSING_INSERT_RECORD'
+  end
+end
+
+end

data/lib/flydata/{parser_provider.rb → parser/parser_provider.rb}

@@ -2,7 +2,8 @@ require 'thread'
 require 'treetop'
 require 'flydata-core/table_def'
 
-module Mysql
+module Flydata
+module Parser
   class ParserProvider
 
     @@mutex = Mutex.new
@@ -15,9 +16,9 @@ module Mysql
           return @@parsers[key]
         else
           # lazy load
-          treetop_path = File.join(
-            File.expand_path(File.dirname(__FILE__)),
-            "parser/#{db_type}/#{parser_type}.treetop")
+          treetop_path = File.expand_path(
+               "../source_#{db_type}/parser/#{parser_type}.treetop",
+               File.dirname(__FILE__))
           Treetop.load treetop_path
           @@parsers[key] = "#{parser_type}_parser".camelize.constantize
         end
@@ -25,3 +26,4 @@ module Mysql
     end
   end
 end
+end

data/lib/flydata/parser/source_table.rb

@@ -0,0 +1,33 @@
+module Flydata
+module Parser
+
+class SourceTable
+  def initialize(table_name, columns = {}, primary_keys = [])
+    @table_name = table_name
+    @columns = columns
+    @column_names = columns.collect{|k,v| v[:column_name]}
+    @primary_keys = primary_keys
+    @value_converters = {}
+  end
+
+  attr_accessor :table_name, :columns, :column_names, :primary_keys, :value_converters
+
+  def add_column(column)
+    cn = column[:column_name]
+    @columns[cn] = column
+    @column_names << cn
+  end
+
+  def set_value_converters(converter_hash)
+    @value_converters ||= {}  # for backward compatibility with an old marshal dump object
+    @columns.each_with_index do |(k, v), i|
+      type = v[:format_type]
+      if converter_hash.has_key?(type)
+        @value_converters[i] = converter_hash[type]
+      end
+    end
+  end
+end
+
+end
+end

data/lib/flydata/source.rb

@@ -0,0 +1,105 @@
+require 'active_support'
+require 'flydata'
+
+module Flydata
+
+module Source
+  def self.create(de)
+    MasterContext.new(de)
+  end
+
+  def self.register(component_class, type_class)
+    @component_classes ||= {}
+    source_sym = underscored_source_module_sym(component_class)
+    unless source_sym
+      raise "Class must must be under a 'SourceXxxx' module"
+    end
+    component_sym = underscored_class_name_sym(type_class)
+    @component_classes[component_key(source_sym, component_sym)] = component_class
+  end
+
+  DATA_ENTRY_TYPE_MAP = {
+    "RedshiftMysqlDataEntry" => :source_mysql,
+    "RedshiftFileDataEntry" => :source_file,
+    "FileDataEntry" => :source_file,
+    "RedshiftPostgresDataEntry" => :source_postgres,
+    "RedshiftZendeskDataEntry" => :source_zendesk,
+  }
+  def self.component_class_for(component_sym, de)
+    source_sym = DATA_ENTRY_TYPE_MAP[de['type']]
+    unless source_sym
+      raise "No source components are available for this data entry"
+    end
+    @component_classes ||= {}
+    component_key = component_key(source_sym, component_sym)
+    unless @component_classes.has_key?(component_key)
+      require_component(source_sym, component_sym)
+      unless @component_classes.has_key?(component_key)
+        @component_classes[component_key] = nil
+      end
+    end
+    klass = @component_classes[component_key]
+    unless klass
+      raise "Component '#{component_sym}' is not defined in source '#{source_sym}'"
+    end
+    klass
+  end
+
+  COMMAND_SUBDIR = "command"
+  # Returns require-friendly paths to the source specific command directory.
+  #  ex) ["flydata/source_mysql/command", "flydata/source_postgres/command"]
+  def self.command_dir_paths
+    result = []
+    DATA_ENTRY_TYPE_MAP.values.each do |source_key|
+      path = File.join(source_dir_path(source_key), COMMAND_SUBDIR)
+      if File.exists?(path)
+        result << File.join("flydata", source_key.to_s, COMMAND_SUBDIR)
+      end
+    end
+    result
+  end
+
+  private
+
+  def self.underscored_source_module_sym(klass)
+    source_module = klass.name.split("::").detect{|m| m.start_with?("Source")}
+    source_module ? underscore_sym(source_module) : nil
+  end
+
+  def self.underscored_class_name_sym(klass)
+    underscore_sym(klass.name.split("::").last)
+  end
+
+  def self.underscore_sym(name)
+    ActiveSupport::Inflector.underscore(name).to_sym
+  end
+
+  def self.component_key(source_key, component_key)
+    "#{source_key}:#{component_key}"
+  end
+
+  def self.require_component(source_key, component_key)
+    path = source_dir_path(source_key)
+    path = File.join(path, "**", "#{component_key}.rb")
+    paths = Dir.glob(path)
+    paths.each {|pth| require pth}
+  end
+
+  def self.source_dir_path(source_key)
+    File.absolute_path(File.join(__FILE__, "..", source_key.to_s))
+  end
+
+  class MasterContext
+    def initialize(de)
+      @de = de
+    end
+    attr_reader :de
+
+    def method_missing(method_sym, *args)
+      source_class = Source.component_class_for method_sym, de
+      source_class.new(self, *args)
+    end
+  end
+end
+
+end

data/lib/flydata/source/component.rb

@@ -0,0 +1,21 @@
+require 'forwardable'
+
+module Flydata
+module Source
+
+class Component
+  extend Forwardable
+
+  def initialize(source, options = {})
+    @source = source
+    @options = options
+  end
+
+  protected
+
+  attr_reader :options
+  def_delegators :@source, :de
+end
+
+end
+end