flydata 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c46afc6ca25cb5f8969e22aac032415a315eb0e
-  data.tar.gz: a95cdb85e846ae1b56f4ad3d2802fc163ec9df99
+  metadata.gz: 83cb9df24d4fe0b983b20bfd30d1798aa4cbecdd
+  data.tar.gz: a00a24a84acf688553503bf97b156acdd57fc8ab
 SHA512:
-  metadata.gz: 56cb5f8d6e58f71598bfc370ea31a24784c1132b5fcadbae263cb29c85a1cd950ff0c2169698722f73058c9d44b16424a57d441c4b98f1ee914ddd57062d91df
-  data.tar.gz: c0b6985cafa887eb2a8aac272c57431f1bb72343b937be7ebb55ef5dfdfec989778acf160119bd0205e4978289fd4134c04890a468bc655022dc99d17f65107f
+  metadata.gz: 335e28547f000af29cf5df8d7a28632fe8fa5356298016f80819622c2a8299ae8469aa46a4e2ec8aa6179c472d151ea90583da1a7ab5f9367aec7668200c9ebc
+  data.tar.gz: 78049be459795e1eccb4b486c55e74989b8f2fae7cce9e5a8ce79236a12ad54d990550b30e71f9027e3ff299598d9f586a63fff8911ae159c6c0935f95e2bfeb

data/Gemfile

@@ -13,7 +13,7 @@ gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
 gem "io-console", '~> 0.4.2', '>= 0.4.2'
-gem "kodama", '~> 0.1.2', '>= 0.1.5'
+gem "kodama", '~> 0.1.2', '>= 0.1.6'
 gem "serverengine", '~> 1.5'
 
 group :development do

data/Gemfile.lock

@@ -61,7 +61,7 @@ GEM
       rdoc
     json (1.8.1)
     jwt (1.0.0)
-    kodama (0.1.5)
+    kodama (0.1.6)
       ruby-binlog (~> 1.0, >= 1.0.4)
     method_source (0.8.2)
     mime-types (2.3)
@@ -135,7 +135,7 @@ DEPENDENCIES
   io-console (~> 0.4.2, >= 0.4.2)
   jeweler (~> 1.8, >= 1.8.8)
   json (~> 1.8, >= 1.8.0)
-  kodama (~> 0.1.2, >= 0.1.5)
+  kodama (~> 0.1.2, >= 0.1.6)
   mysql2 (~> 0.3, >= 0.3.17)
   protected_attributes (~> 1.0, >= 1.0.8)
   pry

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.4.1

data/flydata-core/lib/flydata-core/errors.rb

@@ -304,6 +304,18 @@ class TableDefError < StandardError
   end
 end
 
+## Compatibility check error
+
+class CompatibilityError < StandardError
+end
+
+class AgentCompatibilityError < StandardError
+end
+
+class MysqlCompatibilityError < StandardError
+end
+
+
 ## Error container
 
 class DataDeliveryErrorThreadContext

data/{lib/flydata/mysql/mysql_util.rb → flydata-core/lib/flydata-core/mysql/command_generator.rb}

@@ -1,9 +1,9 @@
 require 'open3'
 require 'flydata-core/table_def/mysql_table_def'
 
-module Flydata
+module FlydataCore
   module Mysql
-    class MysqlUtil
+    class CommandGenerator
       DEFAULT_MYSQL_CMD_OPTION = "--default-character-set=utf8 --protocol=tcp"
 
       # Generate mysql/mysqldump command with options
@@ -76,7 +76,34 @@ module Flydata
         generate_mysql_cmd(opt)
       end
 
-      def self.each_mysql_tabledef(tables, option)
+      def self.each_mysql_tabledef(tables, options, &block)
+        tables = tables.clone
+        missing_tables = []
+        begin
+          if tables.to_s == '' || tables.to_s == '[]'
+            raise ArgumentError, "tables is nil or empty"
+          end
+          _each_mysql_tabledef(tables, options, &block)
+        rescue TableMissingError => e
+          tables.delete e.table
+          missing_tables << e.table
+          return missing_tables if tables.empty?
+          retry
+        end
+        missing_tables
+      end
+
+      private
+
+      class TableMissingError < RuntimeError
+        def initialize(message, table)
+          super(message)
+          @table = table
+        end
+         attr_reader :table
+      end
+
+      def self._each_mysql_tabledef(tables, option)
         command = generate_mysql_ddl_dump_cmd(option.merge(tables: tables))
 
         create_opt = {}
@@ -100,14 +127,20 @@ module Flydata
           errors = ""
           while !stderr.eof?
             line = stderr.gets.gsub('mysqldump: ', '')
-            errors << line unless /Warning: Using a password on the command line interface can be insecure./ === line
+            case line
+            when /Couldn't find table: "([^"]+)"/
+              missing_table = $1
+              raise TableMissingError.new(line, missing_table)
+            when /Warning: Using a password on the command line interface can be insecure./
+              # Ignore
+            else
+              errors << line
+            end
           end
           raise errors unless errors.empty?
         end
       end
 
-      private
-
       def self.convert_keys_to_sym(hash)
         hash.inject(hash.dup) do |ret, (k, v)|
           if k.kind_of?(String) && ret[k.to_sym].nil?

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -0,0 +1,222 @@
+require 'flydata-core/errors'
+require 'mysql2'
+
+module FlydataCore
+  module Mysql
+    class CompatibilityChecker
+      def initialize(option = {})
+        @option = option || {}
+      end
+
+      def do_check(option = @option, &block)
+        result = block.call create_query(option)
+        check_result(result, option)
+      end
+
+      # Override
+      #def create_query(option = @option)
+      #end
+
+      # Override
+      #def validate_result(result, option = @option)
+      #end
+    end
+
+    class MysqlCompatibilityChecker < CompatibilityChecker
+      def do_check(option = @option, &block)
+        query = create_query(option)
+        result = if block
+                   block.call query
+                 else
+                   exec_query(query)
+                 end
+        check_result(result, option)
+      end
+
+      def exec_query(query)
+        begin
+          client = Mysql2::Client.new(@option)
+          client.query(query)
+        ensure
+          client.close rescue nil if client
+        end
+      end
+    end
+
+    class SyncPermissionChecker < MysqlCompatibilityChecker
+      def create_query(option = @option)
+        "SHOW GRANTS"
+      end
+
+      def check_result(result, option = @option)
+        databases = ['mysql', @option[:database]]
+        get_grant_regex = /GRANT (?<privs>.*) ON (`)?(?<db_name>[^`]*)(`)?\.\* TO '#{@option[:username]}/
+        necessary_permission_fields = ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
+        all_privileges_field = ["ALL PRIVILEGES"]
+
+        found_priv = Hash[databases.map {|d| [d,[]]}]
+        missing_priv = {}
+
+        result.each do |res|
+          # SHOW GRANTS should only return one column
+          res_value = res.values.first
+          matched_values = res_value.match(get_grant_regex)
+          next unless matched_values
+          line_priv = matched_values["privs"].split(", ")
+          if matched_values["db_name"] == "*"
+            return true if (all_privileges_field - line_priv).empty?
+            databases.each {|d| found_priv[d] << line_priv }
+          elsif databases.include? matched_values["db_name"] 
+            if (all_privileges_field - line_priv).empty?
+              found_priv[matched_values["db_name"]] = necessary_permission_fields
+            else
+              found_priv[matched_values["db_name"]] << line_priv
+            end
+          end
+          missing_priv = get_missing_privileges(found_priv, necessary_permission_fields)
+          return true if missing_priv.empty?
+        end
+        error_text = "The user '#{@option[:username]}' does not have the correct permissions to run FlyData Sync\n"
+        error_text << "    * These privileges are missing...\n"
+        missing_priv.each_key {|db| error_text << "      for the database '#{db}': #{missing_priv[db].join(", ")}\n"}
+        raise MysqlCompatibilityError, error_text
+      end
+
+      private
+
+      def get_missing_privileges(found_priv, all_priv)
+        return_hash = {}
+        found_priv.each_key do |key|
+          missing_priv = all_priv - found_priv[key].flatten.uniq
+          return_hash[key] = missing_priv unless missing_priv.empty?
+        end
+        return_hash
+      end
+    end
+
+    module MysqlVariablesHandling
+      def create_query(option = @option)
+        "SHOW VARIABLES;"
+      end
+
+      def convert_result_to_hash(result)
+        ret = {}
+        result.each do |record|
+          k = record['Variable_name']
+          v = record['Value']
+          ret[k] = v
+        end
+        ret
+      end
+    end
+
+    class BinlogParameterChecker < MysqlCompatibilityChecker
+      include MysqlVariablesHandling
+
+      SYS_VAR_TO_CHECK = {
+        # parameter => expected value
+        'binlog_format'=>'ROW',
+        'binlog_checksum'=>'NONE',
+        'log_bin_use_v1_row_events'=>'ON',
+        'log_slave_updates'=>'ON'
+      }
+
+      def check_result(result, option = @option)
+        errors = {}
+        param_hash = convert_result_to_hash(result)
+
+        SYS_VAR_TO_CHECK.each_key do |sys_var|
+          if param_hash.has_key?(sys_var)
+            actual_val = param_hash[sys_var]
+            expected_val = SYS_VAR_TO_CHECK[sys_var]
+            unless actual_val == expected_val
+              errors[sys_var] = actual_val
+            end
+          elsif not %w(binlog_checksum log_bin_use_v1_row_events).include?(sys_var)
+            # Mark variables that do not exist as error
+            errors[sys_var] = false
+          end
+        end
+
+        unless errors.empty?
+          error_explanation = ""
+          errors.each_key do |err_key|
+            error_explanation << "\n    * #{err_key} is #{errors[err_key]} but should be #{SYS_VAR_TO_CHECK[err_key]}"
+          end
+          raise FlydataCore::MysqlCompatibilityError,
+            "These system variable(s) are not the correct value: #{error_explanation}\n" +
+            "   Please change these system variables for FlyData Sync to run correctly"
+        end
+      end
+    end
+
+    class TableTypeChecker < MysqlCompatibilityChecker
+      SELECT_TABLE_INFO_TMPLT = 
+        "SELECT table_name, table_type, engine " +
+        "FROM information_schema.tables " +
+        "WHERE table_schema = '%s' and table_name in (%s)"
+
+      # option[:client] : Mysql2::Client object
+      # option[:tables] : target table list
+      def create_query(option = @option)
+        SELECT_TABLE_INFO_TMPLT % [
+          Mysql2::Client.escape(option[:database]),
+          option[:tables].collect{|t| "'#{Mysql2::Client.escape(t)}'"}.join(", ")
+        ]
+      end
+
+      def check_result(result, option = @option)
+        invalid_tables = []
+        result.each do |r|
+          invalid_tables.push(r['table_name']) if r['table_type'] == 'VIEW' || r['engine'] == 'MEMORY'
+        end
+
+        unless invalid_tables.empty?
+          raise FlydataCore::MysqlCompatibilityError,
+            "FlyData does not support VIEW and MEMORY ENGINE table. " +
+            "Remove following tables from data entry: #{invalid_tables.join(", ")}"
+        end
+      end
+    end
+
+    class NonRdsRetentionChecker < MysqlCompatibilityChecker
+      include MysqlVariablesHandling
+
+      BINLOG_RETENTION_HOURS = 24
+      EXPIRE_LOGS_DAYS_LIMIT = BINLOG_RETENTION_HOURS / 24
+
+      def check_result(result, option = @option)
+        param_hash = convert_result_to_hash(result)
+
+        if param_hash["expire_logs_days"].to_s != '0' &&
+           param_hash["expire_logs_days"].to_i <= EXPIRE_LOGS_DAYS_LIMIT
+          raise FlydataCore::MysqlCompatibilityError,
+            "Binary log retention is too short\n " +
+            "  We recommend the system variable '@@expire_logs_days' to be either set to 0 or at least #{EXPIRE_LOGS_DAYS_LIMIT} day(s)"
+        end
+      end
+    end
+
+    class RdsRetentionChecker < MysqlCompatibilityChecker
+      BINLOG_RETENTION_HOURS = NonRdsRetentionChecker::BINLOG_RETENTION_HOURS
+
+      def create_query(option = @option)
+        "call mysql.rds_show_configuration;"
+      end
+
+      def check_result(result, option = @option)
+        if result.first["name"] == "binlog retention hours"
+          if result.first["value"].nil? ||
+             result.first["value"].to_i <= BINLOG_RETENTION_HOURS
+            raise FlydataCore::MysqlCompatibilityError,
+              "Binary log retention is too short\n" +
+              "  We recommend setting RDS binlog retention " +
+              "to be at least #{BINLOG_RETENTION_HOURS} hours. To do this, " +
+              "run this on your RDS MySQL database:\n" +
+              "  $> call mysql.rds_set_configuration('binlog retention hours', 94);"
+          end
+        end
+      end
+    end
+  end
+end

data/flydata-core/lib/flydata-core/query_job.rb

@@ -0,0 +1,7 @@
+# QueryJob is a library provding an interfact to the Query Job Processor (also
+# known as Copy Handler)
+# SimpleDB related classes and methods defined in flydata and flydata-web will
+# be eventually consolidated into this library.
+Dir[File.join(File.dirname(__FILE__), "query_job/**/*.rb")].each do |f|
+  require f
+end

data/flydata-core/lib/flydata-core/query_job/redshift.rb

@@ -0,0 +1,27 @@
+require 'flydata-core/table_def/redshift_table_def'
+
+module FlydataCore
+module QueryJob
+
+class Redshift
+  # returns a list of table names used in the Query Job system.
+  def self.target_table_names(flydata_table_names)
+    flydata_table_names = [ flydata_table_names ] unless flydata_table_names.kind_of?(Array)
+
+    flydata_table_names.collect {|flydata_table_name|
+      redshift_table_name = TableDef::RedshiftTableDef.convert_to_valid_table_name(flydata_table_name)
+      table_names = [ redshift_table_name ]
+      if redshift_table_name != flydata_table_name
+        # for backward compatibility
+        # Old implementation used FlyData table names.  To support it, the
+        # method adds the flydata_table_name as well.
+        table_names << flydata_table_name
+      end
+
+      table_names
+    }.flatten
+  end
+end
+
+end
+end

data/{spec/flydata/mysql/mysql_util_spec.rb → flydata-core/spec/mysql/command_generator_spec.rb}

@@ -1,10 +1,10 @@
 require 'spec_helper'
-require 'flydata/mysql/mysql_util'
+require 'flydata-core/mysql/command_generator'
 
-module Flydata
+module FlydataCore
   module Mysql
 
-    describe MysqlUtil do
+    describe CommandGenerator do
       let(:default_option) { {
         command: 'mysqldump',
         host: 'test-host',
@@ -139,6 +139,25 @@ module Flydata
           'mysql -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp -e "SHOW GRANTS;" testdb'
         ) }
       end
+
+      describe '.each_mysql_tabledef' do
+        context 'with empty tables' do
+          subject { described_class.each_mysql_tabledef([], nil) }
+          it do
+            expect{subject}.to raise_error(
+              ArgumentError,"tables is nil or empty")
+          end
+        end
+
+        context 'when all tables are missing' do
+          before do
+            allow(described_class).to receive(:_each_mysql_tabledef).
+              and_raise(CommandGenerator::TableMissingError.new('table, missing error', 'table1'))
+          end
+          subject { described_class.each_mysql_tabledef(%w(table1), nil) }
+          it { is_expected.to eq(%w(table1)) }
+        end
+      end
     end
   end
 end

data/flydata-core/spec/mysql/compatibility_checker.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+module FlydataCore
+  module Mysql
+    describe SyncPermissionChecker do
+
+    end
+  end
+end

data/flydata-core/spec/query_job/redshift_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+require 'flydata-core/query_job/redshift'
+
+module FlydataCore::QueryJob
+
+describe Redshift do
+  describe '.target_table_names' do
+    subject { described_class.target_table_names(flydata_table_names) }
+
+    context 'with a single table name' do
+      let(:flydata_table_names) { "my_table" }
+
+      it { is_expected.to eq [ flydata_table_names ] }
+    end
+
+    context 'with multiples table names' do
+      let(:flydata_table_names) { %w(my_table1 my_table2) }
+
+      it { is_expected.to eq flydata_table_names }
+    end
+    context 'with a mixed case table name' do
+      let(:flydata_table_names) { "MyTable" }
+
+      it { is_expected.to eq [ flydata_table_names.downcase, flydata_table_names ] }
+    end
+    context 'with table names including mixed case one' do
+      let(:flydata_table_names) { %w(MyTable my_table2) }
+
+      it { is_expected.to eq %w(mytable MyTable my_table2) }
+    end
+  end
+end
+
+end