flydata 0.8.9 → 0.8.9.11

data/lib/flydata/cli.rb

@@ -3,6 +3,7 @@ require 'flydata/command_loggable'
 require 'flydata/errors'
 require 'flydata/helpers'
 require 'flydata/source'
+require 'flydata/error_reporting'
 
 module Flydata
   class Cli
@@ -18,6 +19,8 @@ module Flydata
           "#{datetime} command.#{severity.to_s.downcase}: #{msg}\n"
         end
       end
+
+      Flydata::RollbarHookSetup.new(get_logger).setup
       log_info("Start command.", {cmd: @args.join(' '), ver:flydata_version})
     end
 

data/lib/flydata/error_reporting.rb

@@ -0,0 +1,89 @@
+require 'flydata' # For FLYDATA_HOME
+require 'rubygems'
+
+module Flydata
+
+class RollbarHookSetup
+  def initialize(log,
+      access_token=ENV['ROLLBAR_ACCESS_TOKEN'],
+      environment=ENV['ROLLBAR_ENV'])
+    @log = log
+    @access_token = access_token
+    @environment = environment
+  end
+
+  def should_setup?
+    (!already_setup?) && rollbar_installed? && rollbar_flag? && config_available?
+  end
+
+  def rollbar_flag?
+    File.exists?(File.join(FLYDATA_HOME, 'use_rollbar'))
+  end
+
+  def config_available?
+    return false unless @access_token
+    return false if @access_token.empty?
+
+    return false unless @environment
+    return false if @environment.empty?
+    true
+  end
+
+  def already_setup?
+    @log.instance_variable_defined?('@_rollbar_hook_enabled')
+  end
+
+  # Use to stub in tests
+  def rollbar_gem_name
+    'rollbar'
+  end
+
+  def rollbar_installed?
+    if Gem::Specification.respond_to?(:find_all_by_name)
+      Gem::Specification.find_all_by_name(rollbar_gem_name).any?
+    else
+      Gem.source_index.find_name(rollbar_gem_name).first
+    end
+  end
+
+  def rollbar_configure
+    return unless rollbar_installed?
+    require rollbar_gem_name
+
+    Rollbar.configure do |config|
+      # Only actually send rollbar errors to endpoint in production and staging.
+      config.enabled = ['production', 'staging'].include?(@environment)
+      config.access_token = @access_token
+      config.environment = @environment || 'development'
+
+      config.custom_data_method = lambda do |message, exception, context|
+        { flydata_home: FLYDATA_HOME }
+      end
+    end
+  end
+
+  def setup
+    return false unless should_setup?
+
+    rollbar_configure
+
+    old_error = @log.method(:error)
+
+    # Replace the existing logger's error method so that it will report to rollbar messages or exceptions
+    # when .error is called.
+    @log.define_singleton_method :error do |*args|
+      old_error.call(*args)
+
+      if $!.nil?
+        Rollbar.error(args.first) # Just the message portion
+      else
+        Rollbar.error($!) # Send the exception details
+      end
+    end
+
+    # Make this method idempotent: future calls will have no effect now.
+    @log.instance_variable_set('@_rollbar_hook_enabled', true)
+    true
+  end
+end
+end

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -18,6 +18,7 @@ require 'flydata/source_mysql/table_meta'
 require 'flydata/source_mysql/table_ddl'
 require 'flydata-core/fluent/config_helper'
 require 'flydata-core/mysql/ssl'
+require 'flydata/error_reporting'
 
 
 class MysqlBinlogFlydataInput < MysqlBinlogInput
@@ -53,6 +54,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
   def configure(conf)
     super
+    Flydata::RollbarHookSetup.new($log).setup
 
     # SSL configuration
     unless @ssl_ca_content.to_s.strip.empty?

data/lib/flydata/fluent-plugins/in_postgresql_query_based_flydata.rb

@@ -7,6 +7,7 @@ require 'flydata/source_postgresql/plugin_support/context'
 require 'flydata/source_postgresql/table_meta'
 require 'flydata/source_postgresql/query_based_sync/client'
 require 'flydata-core/postgresql/config'
+require 'flydata/error_reporting'
 
 module Fluent
 
@@ -20,6 +21,9 @@ class PostgresqlQueryBasedFlydataInput < Input
 
   def configure(conf)
     super
+
+    Flydata::RollbarHookSetup.new($log).setup
+
     @dbconf = FlydataCore::Postgresql::Config.opts_for_pg(@data_entry['postgresql_data_entry_preference'])
     $log.info "postgresql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{@tables_append_only}\""
 
@@ -42,7 +46,7 @@ class PostgresqlQueryBasedFlydataInput < Input
       },
     )
 
-    @client = Flydata::SourcePostgresql::QueryBasedSync::Client.new(@context)
+    @client = Flydata::SourcePostgresql::QueryBasedSync::Client.new(@context, true)
   end
 
   def start

data/lib/flydata/query_based_sync/client.rb

@@ -1,5 +1,6 @@
 require 'thread'
 require 'flydata-core/logger'
+require 'flydata/error_reporting'
 
 module Flydata
 module QueryBasedSync
@@ -13,7 +14,7 @@ module QueryBasedSync
     # params
     #   fetch_interval
     #   resource_names
-    def initialize(context)
+    def initialize(context, with_rollbar=false)
       @context = context
       @resource_requester = self.class::RESOURCE_REQUESTER_CLASS.new(context)
       @response_handler = self.class::RESPONSE_HANDLER_CLASS.new(context)
@@ -22,6 +23,10 @@ module QueryBasedSync
         @fetch_interval = c.nil? ? DEFAULT_FETCH_INTERVAL : c[:fetch_interval]
         @retry_interval = c.nil? ? DEFAULT_RETRY_INTERVAL : c[:retry_interval]
       end
+
+      if with_rollbar
+        Flydata::RollbarHookSetup.new(get_logger).setup
+      end
     end
 
     attr_reader :context

data/lib/flydata/source_mysql/mysql_compatibility_check.rb

@@ -17,7 +17,13 @@ module SourceMysql
       @dump_dir = options[:dump_dir] || nil
       @backup_dir = options[:backup_dir] || nil
       @tables = de_hash['tables']
-      @rds = FlydataCore::Mysql::MysqlCompatibilityChecker.new(@db_opts).rds?
+
+      begin
+        @rds = FlydataCore::Mysql::MysqlCompatibilityChecker.new(@db_opts).rds?
+      rescue FlydataCore::MissingExecutePermissionMysqlCompatibilityError => e
+        @rds = false
+        log_warn_stderr("[WARNING] #{e.message}")
+      end
     end
 
     def print_errors
@@ -98,6 +104,8 @@ module SourceMysql
       else
         raise e
       end
+    rescue FlydataCore::MissingExecutePermissionMysqlCompatibilityError => e
+      log_warn_stderr("[WARNING] #{e.message}")
     end
 
     def is_rds?

data/replication.nix

@@ -0,0 +1,30 @@
+{ pkgs ? import <nixpkgs> {}
+, fd-pkgs ? pkgs.fd-pkgs
+}:
+fd-pkgs.callPackage (
+  { fd-base-dev-env
+  , stdenv
+  , cmake
+  , openssl
+  , gnustep
+  , fd-mysql
+  , fetchFromGitHub
+  , boost159
+  }:
+
+stdenv.mkDerivation {
+  name = "mysql-replication-listener";
+  src = fetchFromGitHub {
+    owner = "flydata";
+    repo = "mysql-replication-listener";
+    rev = "80d10ad92a78833fc3d145f86152a0ba99891725";
+    sha256 = "11020wifib046yq9j0n1fjz4i6cq28md8p549w9phgjz13a74f5n";
+  };
+
+  buildInputs = [
+    fd-base-dev-env
+    cmake
+    openssl
+    boost159
+  ];
+}) {}

data/shell.nix

@@ -0,0 +1,47 @@
+{ pkgs ? import <nixpkgs> {}
+, fd-pkgs ? pkgs.fd-pkgs
+}:
+
+let
+  replication = fd-pkgs.callPackage ./replication.nix {};
+in
+
+fd-pkgs.callPackage (
+  { fd-base-dev-env
+  , stdenv
+  , openssl
+  , gnustep
+  , libiconv
+  , libxml2
+  , pkgconfig
+  , libxslt
+  , fd-mysql
+  , postgresql
+  , boost159
+  , sqlite
+  }:
+
+stdenv.mkDerivation {
+  name = "flydata-agent-dev";
+  buildInputs = [
+    postgresql
+    gnustep.libobjc
+    libiconv
+    libxml2
+    pkgconfig
+    libxslt
+    openssl
+    fd-mysql.connector-c
+    replication
+    boost159
+    sqlite
+    fd-base-dev-env
+  ];
+
+  projectDir = builtins.toString ./.;
+
+  shellHook = ''
+    ! (gem list | grep bundler &>/dev/null) && gem install bundler
+    export PATH=$projectDir/bin:$PATH
+  '';
+}) {}

data/spec/flydata/error_reporting_spec.rb

@@ -0,0 +1,107 @@
+require 'flydata/error_reporting'
+require 'logger'
+require 'rollbar'
+
+module Flydata
+  describe RollbarHookSetup do
+    before :each do
+      allow(setup_obj).to receive(:rollbar_gem_name).and_return(rollbar_gem_name)
+      allow(File).to receive(:exists?).and_return(use_rollbar_exists)
+    end
+
+    let(:setup_obj) {RollbarHookSetup.new(log, token, env)}
+    let(:log) {Logger.new(STDOUT)}
+    let(:token) {'sometoken'}
+    let(:env) {'test'}
+    let(:rollbar_gem_name) {'rollbar'}
+    let(:use_rollbar_exists) {true}
+    let(:log_method) {:error}
+    let(:log_message) {"my message"}
+    let(:log_args) {[log_message]}
+
+    describe 'custom data method' do
+      subject {setup_obj.setup; Rollbar.notifier.configuration.custom_data_method.call('a', Exception.new, Object.new)}
+
+      it {should include(:flydata_home)}
+    end
+
+    describe 'log calls after #setup' do
+      subject {setup_obj.setup; log.send(log_method, *log_args)}
+
+      it do
+        expect(Rollbar.notifier).to receive(:configure)
+        expect(Rollbar.notifier).to receive(:log).with("error", log_message)
+        subject
+      end
+
+      context 'during an exception' do
+        let (:exception) { Exception.new("oh no") }
+        subject do
+          setup_obj.setup
+          begin
+            raise exception
+          rescue Exception
+            log.send(log_method, *log_args)
+          end
+        end
+
+        it do
+          expect(Rollbar.notifier).to receive(:log).with("error", exception)
+          subject
+        end
+      end
+
+      context 'no environment is specified' do
+        let(:env) { nil }
+
+        it do
+          expect(Rollbar.notifier).to_not receive(:log)
+          expect(Rollbar.notifier).to_not receive(:configure)
+          subject
+        end
+      end
+
+
+      context 'no token is specified' do
+        let(:token) { nil }
+
+        it do
+          expect(Rollbar.notifier).to_not receive(:log)
+          expect(Rollbar.notifier).to_not receive(:configure)
+          subject
+        end
+      end
+
+      context 'the log method is not error' do
+        let(:log_method) { :warn }
+
+        it do
+          expect(Rollbar.notifier).to_not receive(:log)
+          expect(Rollbar.notifier).to receive(:configure)
+          subject
+        end
+      end
+
+
+      context 'the use_rollbar file does not exist' do
+        let(:use_rollbar_exists) { false }
+
+        it do
+          expect(Rollbar.notifier).to_not receive(:log)
+          expect(Rollbar.notifier).to_not receive(:configure)
+          subject
+        end
+      end
+
+      context 'when the rollbar gem is not installed' do
+        let(:rollbar_gem_name) {'rollbar2'} # Fake name that should not be installed
+
+        it do
+          expect(Rollbar.notifier).to_not receive(:log)
+          expect(Rollbar.notifier).to_not receive(:configure)
+          subject
+        end
+      end
+    end
+  end
+end

data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb

@@ -31,6 +31,7 @@ module SourceMysql
 
     let(:client) { double('client') }
     before do
+      allow_any_instance_of(FlydataCore::Mysql::MysqlCompatibilityChecker).to receive(:rds?).and_return(false)
       allow(Mysql2::Client).to receive(:new).and_return(client)
       allow(client).to receive(:query).with("call mysql.rds_show_configuration;")
       allow(client).to receive(:close)
@@ -156,6 +157,7 @@ module SourceMysql
           allow(subject_object).to receive(:is_rds?).and_return(true)
         end
         before do
+          allow_any_instance_of(FlydataCore::Mysql::ExecutePrivilegeChecker).to receive(:do_check)
           allow(client).to receive(:query).with("SELECT @@expire_logs_days").and_return([{"@@expire_logs_days"=>0}])
         end
 
@@ -220,6 +222,7 @@ module SourceMysql
       context 'when host is rds' do
         before do
           de_hash['host'] = 'rdrss.xxyyzz.rds.amazonaws.com'
+          allow_any_instance_of(FlydataCore::Mysql::MysqlCompatibilityChecker).to receive(:rds?).and_call_original
         end
 
         context "where backup retention period is not set" do
@@ -241,6 +244,41 @@ module SourceMysql
         end
       end
     end
+
+    describe '#run_rds_retention_check' do
+      subject { subject_object.run_rds_retention_check }
+
+      before do
+        allow_any_instance_of(FlydataCore::Mysql::ExecutePrivilegeChecker).to receive(:do_check)
+        allow_any_instance_of(FlydataCore::Mysql::RdsRetentionChecker).to receive(:do_check)
+      end
+
+      context 'when RdsRetentionChecker succeeds' do
+        it { expect{subject}.to_not raise_error }
+      end
+
+      context 'when RdsRetentionChecker fails with command denied' do
+        before do
+          allow_any_instance_of(FlydataCore::Mysql::RdsRetentionChecker).to receive(:do_check).and_raise(Mysql2::Error, "execute command denied to user 'test'@'%' for routine 'mysql.rds_show_configuration'")
+        end
+
+        it { expect{subject}.to_not raise_error }
+
+        it 'logs the error message' do
+          expect(subject_object).to receive(:log_warn_stderr)
+
+          subject
+        end
+      end
+
+      context 'when RdsRetentionChecker fails with unexpected error' do
+        before do
+          allow_any_instance_of(FlydataCore::Mysql::RdsRetentionChecker).to receive(:do_check).and_raise(Mysql2::Error, "error")
+        end
+
+        it { expect{subject}.to raise_error }
+      end
+    end
   end
 
 end