flydata 0.8.9 → 0.8.9.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 18f14f5aa7cb056405bcddd04145128861746e81
-  data.tar.gz: 7137aa3faffbc50cdadf71a54e22f50c59f7beb4
+SHA256:
+  metadata.gz: 35a3d0077661045cac6a3431b1e26370d9d4b3c3734ad13fd41ab3e06e99530e
+  data.tar.gz: 32e5f53766192e1a9ecb384176314095018ee5ee2c1b751ec140a479e8c578ed
 SHA512:
-  metadata.gz: 515cc0f4a0520fc9c6f2573eb457d903dd83c43c6ff8ef114659b8e6050d67c3984e2ecc4acbfeba8307b12e4db383b1039df4647ede956c0d52799a465b6a6c
-  data.tar.gz: e2ea17f5471e190d8bcc718db2da8136bff4677337f60fb5e1f7da4011fb41de5081c9efad491bd8ebe1043296e26a969a55845a6b3446764e573d022cfad3f8
+  metadata.gz: cd8938432ec340ae74f8f4968d4282c3e42532c54a61c2fa7aef43177d0d0a72847d3059e21a9c777d12ba2a3940a51c93505048c6ab00e5a53e1d4092fe646e
+  data.tar.gz: 280fda7d5b9cff89f079196d5ea86c35178afafc94bef9e3be48fb0cb4c0594a08201568d56eef7dea3ffcd1e9f6d48181986c71f276db0f3f602aa963afaddc

data/.gitsha

@@ -0,0 +1 @@
+d37dcb936ffe8940b3dd34695df8291b8ea92e5b

data/.ruby-version

@@ -0,0 +1 @@
+2.0.0-p648

data/Gemfile

@@ -18,7 +18,7 @@ gem "kodama", '~> 0.1', '>= 0.1.12'
 gem "serverengine", '~> 1.5'
 
 group :development do
-  gem "jeweler", '~> 1.8', '>= 1.8.8'
+  gem "jeweler", '~> 2.0', '>= 2.0.1'
   gem "rspec", '~> 3.0'
   gem 'test-unit'
   gem 'timecop', '~> 0.7', '>= 0.7.1'
@@ -31,4 +31,5 @@ group :development do
   gem 'rake'
   #gem 'ruby-oci8'
   gem 'rake-compiler', '~> 0.9', '>= 0.9.5'
+  gem 'rollbar'
 end

data/Gemfile.lock

@@ -17,14 +17,17 @@ GEM
       multi_json (~> 1.3)
       thread_safe (~> 0.1)
       tzinfo (~> 0.3.37)
-    addressable (2.3.6)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
     arel (4.0.2)
     builder (3.1.4)
     coderay (1.1.0)
     cool.io (1.4.1)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
-    faraday (0.8.9)
-      multipart-post (~> 1.2.0)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
     ffi (1.9.6)
     fluent-plugin-mysql-binlog (0.0.2)
       activesupport
@@ -37,48 +40,52 @@ GEM
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
       sigdump (~> 0.2.2)
       yajl-ruby (~> 1.0)
-    git (1.2.8)
-    github_api (0.10.1)
-      addressable
-      faraday (~> 0.8.1)
+    git (1.5.0)
+    github_api (0.11.3)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.1)
+      faraday (~> 0.8, < 0.10)
       hashie (>= 1.2)
-      multi_json (~> 1.4)
-      nokogiri (~> 1.5.2)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.0)
       oauth2
-    hashie (3.2.0)
+    hashie (3.6.0)
     highline (1.6.21)
     http_parser.rb (0.6.0)
     i18n (0.6.11)
     io-console (0.4.6)
-    jeweler (1.8.8)
+    jeweler (2.1.2)
       builder
-      bundler (~> 1.0)
+      bundler (>= 1.0)
       git (>= 1.2.5)
-      github_api (= 0.10.1)
+      github_api (~> 0.11.0)
       highline (>= 1.6.15)
-      nokogiri (= 1.5.10)
+      nokogiri (>= 1.5.10)
       rake
       rdoc
+      semver
     json (1.8.1)
-    jwt (1.0.0)
+    jwt (1.5.6)
     kodama (0.1.12)
       ruby-binlog (~> 1.0, >= 1.0.10)
     method_source (0.8.2)
     mime-types (2.3)
+    mini_portile2 (2.1.0)
     minitest (4.7.5)
     msgpack (0.5.8)
     multi_json (1.10.1)
-    multi_xml (0.5.5)
-    multipart-post (1.2.0)
+    multi_xml (0.6.0)
+    multipart-post (2.0.0)
     mysql2 (0.3.18)
     netrc (0.7.7)
-    nokogiri (1.5.10)
-    oauth2 (1.0.0)
-      faraday (>= 0.8, < 0.10)
-      jwt (~> 1.0)
+    nokogiri (1.6.8.1)
+      mini_portile2 (~> 2.1.0)
+    oauth2 (1.4.1)
+      faraday (>= 0.8, < 0.16.0)
+      jwt (>= 1.0, < 3.0)
       multi_json (~> 1.3)
       multi_xml (~> 0.5)
-      rack (~> 1.2)
+      rack (>= 1.2, < 3)
     pg (0.18.4)
     polyglot (0.3.5)
     power_assert (1.0.1)
@@ -88,15 +95,17 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    rack (1.5.2)
+    public_suffix (2.0.5)
+    rack (1.6.11)
     rake (10.3.2)
     rake-compiler (0.9.5)
       rake
-    rdoc (4.1.1)
-      json (~> 1.4)
+    rdoc (5.1.0)
     rest-client (1.7.2)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
+    rollbar (2.18.0)
+      multi_json
     rspec (3.0.0)
       rspec-core (~> 3.0.0)
       rspec-expectations (~> 3.0.0)
@@ -111,6 +120,7 @@ GEM
     rspec-support (3.0.3)
     ruby-binlog (1.0.10)
     ruby-prof (0.15.1)
+    semver (1.0.1)
     serverengine (1.5.10)
       sigdump (~> 0.2.2)
     sigdump (0.2.2)
@@ -139,7 +149,7 @@ DEPENDENCIES
   highline (~> 1.6, >= 1.6.19)
   i18n (~> 0.6, >= 0.6.5)
   io-console (~> 0.4.6, >= 0.4.6)
-  jeweler (~> 1.8, >= 1.8.8)
+  jeweler (~> 2.0, >= 2.0.1)
   json (~> 1.8, >= 1.8.0)
   kodama (~> 0.1, >= 0.1.12)
   mysql2 (~> 0.3, >= 0.3.17)
@@ -149,6 +159,7 @@ DEPENDENCIES
   rake
   rake-compiler (~> 0.9, >= 0.9.5)
   rest-client (~> 1.6, >= 1.6.7)
+  rollbar
   rspec (~> 3.0)
   ruby-prof (~> 0.15, >= 0.15.1)
   serverengine (~> 1.5)
@@ -160,4 +171,4 @@ DEPENDENCIES
   treetop (~> 1.5, >= 1.5.3)
 
 BUNDLED WITH
-   1.13.7
+   1.17.1

data/Rakefile

@@ -21,7 +21,10 @@ Jeweler::Tasks.new do |gem|
   gem.description = %Q{FlyData Agent}
   gem.email = "sysadmin@flydata.com"
   gem.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS", "Chie Inada"]
-  # dependencies defined in Gemfile
+
+  sha=`git rev-parse --verify HEAD`
+  File.open(File.join(File.dirname(__FILE__), ".gitsha"), "w") { |f| f << sha }
+  gem.files << ".gitsha"
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-0.8.9
+0.8.9.11

data/circle.yml

@@ -6,7 +6,7 @@ dependencies:
   pre:
     - sudo dpkg -l |grep libboost|grep 1.48|awk '{print $2}'|xargs sudo apt-get purge -y
     - sudo apt-get autoremove
-    - sudo apt-get update
+    - sudo apt-get update || true
     - sudo apt-get install -y libboost-system1.54.0
     - sudo apt-get install -y libboost-system1.54-dev
     - sudo apt-get install -y libboost-thread1.54.0

data/fd-build

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+set -e
+set -o pipefail
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+FD_BUILD=$(which fd-build)
+source $(dirname $FD_BUILD)/../lib/fd-scripts/utils.sh
+
+cd $DIR
+
+# bundle to install gems.
+nix-shell --run "bundle install --path vendor/bundle"
+nix-shell --run "bundle exec rake compile"
+
+# Bundix updates the gemset.nix file so that nix-build can operate using fixed builds.
+$(nix-build '<nixpkgs>' -A fd-bundix --no-out-link --show-trace)/bin/bundix --ruby=fd-ruby
+
+echo -e "${BLUE}Done!${RESTORE}  Everything should be up to date!"

data/flydata-core/lib/flydata-core/errors.rb

@@ -380,6 +380,9 @@ end
 class PostgresqlCompatibilityError < CompatibilityError
 end
 
+class MissingExecutePermissionMysqlCompatibilityError < MysqlCompatibilityError
+end
+
 ## Error container
 
 class DataDeliveryErrorThreadContext

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -61,6 +61,8 @@ module FlydataCore
       def rds?(hostname = @option[:host])
         return true if hostname.match(/rds.amazonaws.com$/) != nil
 
+        ExecutePrivilegeChecker.new(@option).do_check
+
         # To distinguish RDS customers using Secure Tunnel
         begin
           exec_query(RDS_CHECK_QUERY)
@@ -358,7 +360,7 @@ EOT
         param_hash = convert_result_to_hash(result)
 
         if param_hash["expire_logs_days"].to_s != '0' &&
-           param_hash["expire_logs_days"].to_i <= EXPIRE_LOGS_DAYS_LIMIT
+           param_hash["expire_logs_days"].to_i < EXPIRE_LOGS_DAYS_LIMIT
           raise FlydataCore::MysqlCompatibilityError,
             "Binary log retention is too short\n " +
             "  We recommend the system variable '@@expire_logs_days' to be either set to 0 or at least #{EXPIRE_LOGS_DAYS_LIMIT} day(s)"
@@ -389,6 +391,12 @@ EOT
         "call mysql.rds_show_configuration;"
       end
 
+      def do_check(option = @option, &block)
+        ExecutePrivilegeChecker.new(option).do_check
+
+        super(option, &block)
+      end
+
       def check_result(result, option = @option)
         if result.first["name"] == "binlog retention hours"
           if result.first["value"].nil? ||
@@ -403,5 +411,35 @@ EOT
         end
       end
     end
+
+    class ExecutePrivilegeChecker < MysqlCompatibilityChecker
+      def create_query(option = @option)
+        "SHOW GRANTS FOR '#{option[:username]}'"
+      end
+
+      def check_result(result, option = @option)
+        valid_privileges = ['EXECUTE', 'ALL PRIVILEGES']
+
+        get_grant_regex = /GRANT (?<privs>.*) ON (`)?(?<db_name>[^`]*)(`)?\.\* TO '#{option[:username]}/
+
+        result.each do |res|
+          # SHOW GRANTS should only return one column
+          res_value = res.values.first
+          matched_values = res_value.match(get_grant_regex)
+
+          next unless matched_values
+          matched_privileges = matched_values['privs'].split(', ')
+
+          next unless ['*', option[:database]].include?(matched_values['db_name'])
+          next if (matched_privileges & valid_privileges).empty? # array intersection
+
+          return true
+        end
+
+        error_text = "The user '#{option[:username]}' does not have the EXECUTE permission"
+
+        raise MissingExecutePermissionMysqlCompatibilityError, error_text
+      end
+    end
   end
 end

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -301,7 +301,12 @@ EOS
         "'#{self.parse_timestamp(remove_single_quote(default_value))}'"
       end
     when 'date'
-      "'#{self.parse_date(remove_single_quote(default_value))}'"
+      val = self.parse_date(remove_single_quote(default_value))
+      if val =~ /SYSDATE/
+        "#{val}"
+      else
+        "'#{val}'"
+      end
     when 'boolean'
       default_value.to_s.upcase
     else
@@ -517,9 +522,14 @@ EOS
 
   def self.parse_date(value)
     return nil if value.nil?
-    value_str = value.to_s
-    dt = Date.parse(convert_date(value_str))
-    dt.strftime('%Y-%m-%d')
+    case value.to_s.upcase
+    when /^CURRENT_TIMESTAMP\b/, /^CURRENT_TIME\b/, /^NOW\(\)/
+      'SYSDATE'
+    else
+      value_str = value.to_s
+      dt = Date.parse(convert_date(value_str))
+      dt.strftime('%Y-%m-%d')
+    end
   rescue ArgumentError => ae
     # '0000-00-00' is valid for mysql date column
     return '0001-01-01' if value_str == '0000-00-00'

data/flydata-core/spec/mysql/compatibility_checker_spec.rb

@@ -52,6 +52,21 @@ module FlydataCore
         describe '#rds?' do
           subject { subject_object.rds? }
 
+          let(:execute_privilege_checker) { ExecutePrivilegeChecker.new }
+
+          before do
+            allow(ExecutePrivilegeChecker).to receive(:new).and_return(execute_privilege_checker)
+            allow(execute_privilege_checker).to receive(:do_check).and_return(nil)
+          end
+
+          it 'calls ExecutePrivilegeChecker' do
+            allow(subject_object).to receive(:exec_query).with(MysqlCompatibilityChecker::RDS_CHECK_QUERY).and_return(nil)
+
+            expect(execute_privilege_checker).to receive(:do_check)
+
+            subject
+          end
+
           context 'When called for non-rds host' do
             let(:mysql2_error_msg) { "PROCEDURE mysql.rds_show_configuration does not exist" }
             before do
@@ -107,6 +122,18 @@ module FlydataCore
               expect { subject }.to raise_error(Mysql2::Error, mysql2_error_msg)
             end
           end
+
+          context 'when FlydataCore::MissingExecutePermissionMysqlCompatibilityError raised' do
+            let(:error_msg) { "The user 'sync' does not have the EXECUTE permission" }
+
+            before do
+              allow(ExecutePrivilegeChecker).to receive(:new).and_return(execute_privilege_checker)
+              allow(execute_privilege_checker).to receive(:do_check).and_raise(FlydataCore::MissingExecutePermissionMysqlCompatibilityError, error_msg)
+            end
+            it do
+              expect { subject }.to raise_error(FlydataCore::MissingExecutePermissionMysqlCompatibilityError, error_msg)
+            end
+          end
         end
       end
 
@@ -339,6 +366,119 @@ EOT
           end
         end
       end
+
+      describe ExecutePrivilegeChecker do
+        let(:subject_object) { described_class.new(option.merge(username: 'sync')) }
+
+        describe '#check_result' do
+          subject { subject_object.check_result(result) }
+
+          context 'when ALL PRIVILEGES is granted' do
+            let(:result) do
+              [{"Grants for sync@%" => "GRANT ALL PRIVILEGES ON *.* TO 'sync'@'%' IDENTIFIED BY PASSWORD 'password'"}]
+            end
+
+            it { expect{subject}.to_not raise_error }
+          end
+
+          context 'when EXECUTE privilege is granted in all databases' do
+            let(:result) do
+              [{"Grants for sync@%" => "GRANT SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT, EXECUTE ON *.* TO 'sync'@'%' IDENTIFIED BY PASSWORD 'password'"}]
+            end
+
+            it { expect{subject}.to_not raise_error }
+          end
+
+          context 'when EXECUTE privilege is granted in the test_db database' do
+            let(:result) do
+              [{"Grants for sync@%" => "GRANT SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT, EXECUTE ON test_db.* TO 'sync'@'%' IDENTIFIED BY PASSWORD 'password'"}]
+            end
+
+            it { expect{subject}.to_not raise_error }
+          end
+
+          context 'when ALL PRIVILEGES is granted in a different database' do
+            let(:result) do
+              [{"Grants for sync@%" => "GRANT ALL PRIVILEGES ON different_db.* TO 'sync'@'%' IDENTIFIED BY PASSWORD 'password'"}]
+            end
+
+            it { expect{subject}.to raise_error(FlydataCore::MissingExecutePermissionMysqlCompatibilityError, "The user 'sync' does not have the EXECUTE permission") }
+          end
+
+          context 'when EXECUTE privilege is granted in a different database' do
+            let(:result) do
+              [{"Grants for sync@%" => "GRANT SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT, EXECUTE ON different_db.* TO 'sync'@'%' IDENTIFIED BY PASSWORD 'password'"}]
+            end
+
+            it { expect{subject}.to raise_error(FlydataCore::MissingExecutePermissionMysqlCompatibilityError, "The user 'sync' does not have the EXECUTE permission") }
+          end
+
+          context 'when EXECUTE privilege is not granted' do
+            let(:result) do
+              [{"Grants for sync@%" => "GRANT SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'sync'@'%' IDENTIFIED BY PASSWORD 'password'"}]
+            end
+
+            it { expect{subject}.to raise_error(FlydataCore::MissingExecutePermissionMysqlCompatibilityError, "The user 'sync' does not have the EXECUTE permission") }
+          end
+        end
+      end
+
+      describe RdsRetentionChecker do
+        let(:options) { option.merge(username: 'sync') }
+        let(:mysql_options) { options.merge(port: nil, password: nil, ssl_ca: nil, ssl_cipher: nil) }
+        let(:subject_object) { described_class.new(options) }
+
+        describe '#do_check' do
+          subject { subject_object.do_check }
+
+          let(:execute_privilege_checker) { ExecutePrivilegeChecker.new(options) }
+
+          before do
+            @rds_show_configuration_result = [{name: 'binlog retention hours', value: '144'}]
+
+            allow(subject_object).to receive(:create_query).and_return('call mysql.rds_show_configuration;')
+            allow(subject_object).to receive(:exec_query).with('call mysql.rds_show_configuration;').and_return(@rds_show_configuration_result)
+            allow(subject_object).to receive(:check_result).with(@rds_show_configuration_result, mysql_options).and_return(nil)
+
+            allow(ExecutePrivilegeChecker).to receive(:new).and_return(execute_privilege_checker)
+            allow(execute_privilege_checker).to receive(:do_check).and_return(nil)
+          end
+
+          it { expect{subject}.to_not raise_error }
+
+          it 'calls ExecutePrivilegeChecker' do
+            expect(execute_privilege_checker).to receive(:do_check)
+
+            subject
+          end
+
+          context 'when database user has EXECUTE privilege' do
+            it { expect{subject}.to_not raise_error }
+
+            context 'when binlog is too low' do
+              before do
+                @error_message = "Binary log retention is too short\n" +
+                  "  We recommend setting RDS binlog retention " +
+                  "to be at least 96 hours. To do this, " +
+                  "run this on your RDS MySQL database:\n" +
+                  "  $> call mysql.rds_set_configuration('binlog retention hours', 96);"
+
+                allow(subject_object).to receive(:check_result).with(@rds_show_configuration_result, mysql_options).and_raise(FlydataCore::MysqlCompatibilityError, @error_message)
+              end
+
+              it { expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, @error_message) }
+            end
+          end
+
+          context 'when database user does not have EXECUTE privilege' do
+            before do
+              allow(execute_privilege_checker).to receive(:do_check).and_raise(FlydataCore::MissingExecutePermissionMysqlCompatibilityError, "The user 'sync' does not have the EXECUTE permission")
+            end
+
+            it { expect{subject}.to raise_error(FlydataCore::MissingExecutePermissionMysqlCompatibilityError, "The user 'sync' does not have the EXECUTE permission") }
+          end
+        end
+      end
     end
 
     describe BinlogParameterChecker do