flydata 0.2.22 → 0.2.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99feeb76fa109f4255e1d76c881d1631e82fdfbe
-  data.tar.gz: 8c080c31ac049d2ad3e30bef154507a6ec15aec7
+  metadata.gz: b1b0c1f87ad981df879da76c17b2e4fc99246273
+  data.tar.gz: aa8b2361f3639d1891019339df98c84ab3f95d3d
 SHA512:
-  metadata.gz: f51151ca084d61e1fdf9bea821151f86f54c773c53223daa35a0ac5e173125906cc2cee2ab53ef691bcebcd7f1ae59b96476977786e6d076b3542021dea982cb
-  data.tar.gz: fbca06f1f714479abcf848e16cd9905579ed857ffbcf7a1b9b3d42895f17fb5446f7fc5e17ffa103595e25c87ac1c06e011c03449a16b9c1f4c5afc6f52968ed
+  metadata.gz: a7225b7a8d99fe0087f55aea9da3b1f514386b08ab47da2e100ecea32f030f9353743827b394f50ee805d37791004a0338c4992fcde9968ea9c64be7af2ea02a
+  data.tar.gz: ce6c3c67dc8730820ba5ebf3a9f46d8645f1dc1eb4d29dad40164015d72ddc29c49ad608fdc877b68f8ad21daac757a36bf9caa494696a00a66e2567e098a9a6

data/Gemfile

@@ -12,6 +12,7 @@ gem "fluent-plugin-mysql-binlog", '~> 0.0', '>= 0.0.2'
 gem "mysql2", '~> 0.3', '>= 0.3.11'
 gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
+gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
 
 group :development do
   gem "jeweler", '~> 1.8', '>= 1.8.8'

data/Gemfile.lock

@@ -25,6 +25,7 @@ GEM
     diff-lcs (1.2.5)
     faraday (0.8.9)
       multipart-post (~> 1.2.0)
+    ffi (1.9.6)
     fluent-plugin-mysql-binlog (0.0.2)
       activesupport
       fluentd
@@ -108,6 +109,8 @@ GEM
     sigdump (0.2.2)
     slop (3.6.0)
     sqlite3 (1.3.9)
+    sys-filesystem (1.1.3)
+      ffi
     thread_safe (0.3.4)
     timecop (0.7.1)
     treetop (1.5.3)
@@ -137,5 +140,6 @@ DEPENDENCIES
   ruby-prof (~> 0.15, >= 0.15.1)
   slop (~> 3.4, >= 3.4.6)
   sqlite3 (~> 1.3, >= 1.3.9)
+  sys-filesystem (~> 1.1, >= 1.1.3)
   timecop (~> 0.7, >= 0.7.1)
   treetop (~> 1.5, >= 1.5.3)

data/VERSION

@@ -1 +1 @@
-0.2.22
+0.2.23

data/flydata.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: flydata 0.2.23 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.22"
+  s.version = "0.2.23"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2014-12-15"
+  s.date = "2014-12-17"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -143,8 +145,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.14"
+  s.rubygems_version = "2.2.2"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then
@@ -162,6 +163,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
       s.add_runtime_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
       s.add_runtime_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
+      s.add_runtime_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_development_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_development_dependency(%q<rspec>, ["~> 3.0"])
       s.add_development_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -183,6 +185,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
       s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
       s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
+      s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_dependency(%q<rspec>, ["~> 3.0"])
       s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -205,6 +208,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
     s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
     s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
+    s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
     s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
     s.add_dependency(%q<rspec>, ["~> 3.0"])
     s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])

data/lib/flydata/command/sync.rb

@@ -2,6 +2,7 @@ require 'fiber'
 require 'msgpack'
 require 'open3'
 require 'mysql2'
+require 'sys/filesystem'
 require 'flydata/sync_file_manager'
 require 'flydata/compatibility_check'
 require 'flydata/output/forwarder'
@@ -24,7 +25,8 @@ module Flydata
         Slop.new do
           on 'c', 'skip-cleanup', 'Skip server cleanup'
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
-          on 'd', 'dump-file', 'Dump mysqldump into a file. Use this for debugging after making sure the free space.'
+          on 'd', 'dump-file', 'Dump mysqldump into a file. Use this for debugging after making sure the free space.' # dummy for compatibility
+          on 's', 'dump-stream', 'Dump mysqldump stream instead of saving dump file. It might cause timeout error if db size is larger than 10GB.'
           on 'n', 'no-flydata-start', 'Don\'t start the flydata agent after initial sync.'
         end
       end
@@ -290,7 +292,7 @@ module Flydata
           Flydata::Command::Conf.new.copy_templates
         end
 
-        generate_mysqldump(de, sync_fm, opts.dump_file?) do |mysqldump_io, db_bytesize|
+        generate_mysqldump(de, sync_fm, !opts.dump_stream?) do |mysqldump_io, db_bytesize|
           sync_fm.save_sync_info(de['mysql_data_entry_preference']['initial_sync'], de['mysql_data_entry_preference']['tables'])
           parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize)
         end
@@ -341,10 +343,22 @@ EOM
         log_info confirmation_text.strip
 
         if ask_yes_no('Start Sync?')
-          log_info_stdout("Checking database size(not same as mysqldump data size)...")
+          log_info_stdout("Checking database size...")
 
           db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
           log_info_stdout("  -> #{as_size(db_bytesize)} (#{db_bytesize} byte)")
+
+          # check free disk space
+          free_disk_bytesize = free_disk_space(File.dirname(fp))
+          if (free_disk_bytesize - db_bytesize) < (1024 * 1024 * 1024) # 1GB
+            log_warn_stderr("!!WARNING There may not be enough disk space for a DB dump. We recommend 1GB free disk space after the dump. free disk space:#{as_size(free_disk_bytesize)}(#{free_disk_bytesize} byte) /" +
+                            " db size:#{as_size(db_bytesize)}(#{db_bytesize} byte)")
+            unless ask_yes_no('Do you want to continue?')
+              log_warn_stderr("To change the dump file directory, delete '#' and modify the path of 'mysqldump_dir:' in '#{Preference::DataEntryPreference.conf_path(de)}'")
+              exit 1
+            end
+          end
+
           log_info_stdout("Exporting data from database.")
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
           Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], fp).check
@@ -362,6 +376,12 @@ EOM
         end
       end
 
+      # return available disk size(byte)
+      def free_disk_space(dump_path)
+        stat = Sys::Filesystem.stat(dump_path)
+        stat.block_size * stat.blocks_available
+      end
+
       def call_block_or_return_io(fp, &block)
         if block
           f_io = open_file_io(fp)

data/lib/flydata/compatibility_check.rb

@@ -75,6 +75,7 @@ module Flydata
     end
 
     SELECT_QUERY_TMPLT = "SELECT %s"
+    BINLOG_RETENTION_HOURS = 24
 
     #def initialize(de_hash, dump_dir=nil)
     def initialize(dp_hash, de_hash, options={})
@@ -169,6 +170,19 @@ module Flydata
       end
     end
 
+    def check_mysql_binlog_retention
+      client = Mysql2::Client.new(@db_opts)
+      begin
+        if is_rds?(@db_opts[:host])
+          run_rds_retention_check(client)
+        else
+          run_mysql_retention_check(client)
+        end
+      ensure
+        client.close
+      end
+    end
+
     def check_writing_permissions
       write_errors = []
       paths_to_check = ["~/.flydata"]
@@ -183,6 +197,48 @@ module Flydata
         raise MysqlCompatibilityError, "We cannot access the directories: #{error_dir}"
       end
     end
+
+    def run_mysql_retention_check(mysql_client)
+      expire_logs_days_limit = BINLOG_RETENTION_HOURS / 24
+      sel_query = SELECT_QUERY_TMPLT % '@@expire_logs_days'
+      result = mysql_client.query(sel_query)
+      if result.first["@@expire_logs_days"]!=0 and result.first["@@expire_logs_days"] <= expire_logs_days_limit
+        raise MysqlCompatibilityError, "Binary log retention is too short\n " +
+          "  We recommend the system variable '@@expire_logs_days' to be either set to 0 or at least #{expire_logs_days_limit} days"
+      end
+    end
+
+    def run_rds_retention_check(mysql_client)
+      sql_query =  "call mysql.rds_show_configuration;"
+
+      begin
+        result = mysql_client.query(sql_query)
+        if result.first["name"]=="binlog retention hours"
+          if result.first["value"].nil? or result.first["value"].to_i <= BINLOG_RETENTION_HOURS
+            raise MysqlCompatibilityError, "Binary log retention is too short\n" +
+                                           "  We recommend setting RDS binlog retention to be at least #{BINLOG_RETENTION_HOURS} hours. To do this, run this on your RDS MySQL database:\n" +
+                                           "  $> call mysql.rds_set_configuration('binlog retention hours', 94);"
+          end
+        end
+      rescue Mysql2::Error => e
+        if e.message =~ /command denied to user/
+          log_warn_stderr("[WARNING]Cannot verify RDS retention period on currend MySQL user account.\n" +
+                          "To see retention period, please run this on your RDS:\n" +
+                          "  $> call mysql.rds_show_configuration;\n" +
+                          "Please verify that the hours is not nil and is at least #{BINLOG_RETENTION_HOURS} hours\n" +
+                          "To set binlog retention hours, you can run this on your RDS:\n" +
+                          "  $> call mysql.rds_set_configuration('binlog retention hours', #{BINLOG_RETENTION_HOURS});\n"
+                         )
+        else
+          raise e
+        end
+      end
+
+    end
+
+    def is_rds?(hostname)
+      hostname.match(/rds.amazonaws.com$/) != nil
+    end
   end
 
 end

data/spec/flydata/compatibility_check_spec.rb

@@ -33,4 +33,111 @@ module Flydata
       end
     end
   end
+
+  describe MysqlCompatibilityCheck do
+    let(:default_mysql_cred) do
+      {
+        "host" => "test",
+        "port" => 1234,
+        "username" => "test",
+        "password" => "password",
+        "database" => "test_db"
+      }
+    end
+    describe "#check_mysql_binlog_retention" do
+      context "on on-premise mysql server" do
+        subject { MysqlCompatibilityCheck.new({},:default_mysql_cred) }
+        context "where retention is below limit" do
+          let(:client) { double('client') }
+          before do
+            allow(Mysql2::Client).to receive(:new).and_return(client)
+            allow(client).to receive(:query).and_return([{"@@expire_logs_days"=>1}])
+            allow(client).to receive(:close)
+            allow(subject).to receive(:is_rds?).and_return(false)
+          end
+          it do
+            expect{subject.check_mysql_binlog_retention}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /expire_logs_days/)
+          end
+        end
+        context "where retention is 0" do
+          let(:client) { double('client') }
+          before do
+            allow(Mysql2::Client).to receive(:new).and_return(client)
+            allow(client).to receive(:query).and_return([{"@@expire_logs_days"=>0}])
+            allow(subject).to receive(:is_rds?).and_return(false)
+          end
+          it do
+            expect{subject.check_mysql_binlog_retention}
+          end
+        end
+        context "where retention is above limit" do
+          let(:client) { double('client') }
+          before do
+            allow(Mysql2::Client).to receive(:new).and_return(client)
+            allow(client).to receive(:query).and_return([{"@@expire_logs_days"=>11}])
+            allow(subject).to receive(:is_rds?).and_return(false)
+          end
+          it do
+            expect{subject.check_mysql_binlog_retention}
+          end
+        end
+      end
+      context "on RDS" do
+        subject { MysqlCompatibilityCheck.new({},:default_mysql_cred) }
+        context "where retention period is nil" do
+          let(:client) { double('client') }
+          before do
+            allow(Mysql2::Client).to receive(:new).and_return(client)
+            allow(client).to receive(:query).with("SELECT @@expire_logs_days").and_return([{"@@expire_logs_days"=>0}])
+            allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>nil}])
+            allow(client).to receive(:close)
+            allow(subject).to receive(:is_rds?).and_return(true)
+          end
+          it do
+            expect{subject.check_mysql_binlog_retention}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /rds_set_config/)
+          end
+        end
+        context "where retention period is too low" do
+           let(:client) { double('client') }
+          before do
+            allow(Mysql2::Client).to receive(:new).and_return(client)
+            allow(client).to receive(:query).with("SELECT @@expire_logs_days").and_return([{"@@expire_logs_days"=>0}])
+            allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>4}])
+            allow(client).to receive(:close)
+            allow(subject).to receive(:is_rds?).and_return(true)
+          end
+          it do
+            expect{subject.check_mysql_binlog_retention}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /rds_set_config/)
+          end
+        end
+        context "where retention period is over recommended limit" do
+         let(:client) { double('client') }
+          before do
+            allow(Mysql2::Client).to receive(:new).and_return(client)
+            allow(client).to receive(:query).with("SELECT @@expire_logs_days").and_return([{"@@expire_logs_days"=>0}])
+            allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>120}])
+            allow(client).to receive(:close)
+            allow(subject).to receive(:is_rds?).and_return(true)
+          end
+          it do
+            expect{subject.check_mysql_binlog_retention}
+          end
+        end
+        context "where user has no access to rds configuration" do
+          let(:client) { double('client') }
+          before do
+            allow(Mysql2::Client).to receive(:new).and_return(client)
+            allow(client).to receive(:query).with("SELECT @@expire_logs_days").and_return([{"@@expire_logs_days"=>0}])
+            allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_raise(Mysql2::Error, "execute command denied to user")
+            allow(client).to receive(:close)
+            allow(subject).to receive(:is_rds?).and_return(true)
+          end
+          it do
+            expect{subject.check_mysql_binlog_retention}
+          end
+        end
+      end
+    end
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.22
+  version: 0.2.23
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-15 00:00:00.000000000 Z
+date: 2014-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -222,6 +222,26 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: sys-filesystem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.3
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.3
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: jeweler
   requirement: !ruby/object:Gem::Requirement
@@ -545,7 +565,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: FlyData Agent