flydata 0.2.27 → 0.2.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31c9862228f6affa3e27e1ffbe4bbd7bd330774e
-  data.tar.gz: 5a447f0bb5cc7db2997d90580d7b59360834743f
+  metadata.gz: bde8ac1d5079ec1086baca972d51548f98e5b96b
+  data.tar.gz: 5d6ddbacb6f4e16e9f502623f6c8a197efdf03f3
 SHA512:
-  metadata.gz: 7bdec134c9d33a26ab24c734220ba639465f2b7f1bc6e49333beebf80a6ee73794d603566aa5ca429b296718f1cdaa52d4ee98d5044b96981a60d64aeca724df
-  data.tar.gz: 7242fb4e8e3d1c19da49557238921b5ee5f91b7891f5710d39a136ed25127781c0db7269aece6ab08d167d53196dea16fbf98ddb07ef7b28c82b46d0c38d474d
+  metadata.gz: 922db0c489085b5b86160658965b8154701f09ab826c5b5eed58547dcc61b13c89ebc1bbe98aba9eb44c63615dafb9039ca0e4214fbc31b6215466b82722ba3b
+  data.tar.gz: fdafc2ffdd7995f6c7c91e8af09845c58e3ac682924c79f23b2f5ce1108bc9a671f435ff7af985e55b74ed69e8f8b5ad22d8cd92ad73001f00c1b9353f1f7daa

data/VERSION

@@ -1 +1 @@
-0.2.27
+0.2.28

data/flydata.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: flydata 0.2.28 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.27"
+  s.version = "0.2.28"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-01-08"
+  s.date = "2015-01-16"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -144,8 +146,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.14"
+  s.rubygems_version = "2.2.2"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/sync.rb

@@ -86,6 +86,8 @@ module Flydata
           sync_fm.mysql_table_marshal_dump_path,
           sync_fm.sync_info_file,
           sync_fm.table_position_file_paths(*tables),
+          sync_fm.table_binlog_pos_paths(*tables),
+          sync_fm.table_binlog_pos_init_paths(*tables),
           sync_fm.table_rev_file_paths(*tables)
         ]
         delete_files << sync_fm.binlog_path if tables.empty?
@@ -352,20 +354,23 @@ EOM
           db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
           log_info_stdout("  -> #{as_size(db_bytesize)} (#{db_bytesize} byte)")
 
-          # check free disk space
-          free_disk_bytesize = free_disk_space(File.dirname(fp))
-          if (free_disk_bytesize - db_bytesize) < (1024 * 1024 * 1024) # 1GB
-            log_warn_stderr("!!WARNING There may not be enough disk space for a DB dump. We recommend 1GB free disk space after the dump. free disk space:#{as_size(free_disk_bytesize)}(#{free_disk_bytesize} byte) /" +
-                            " db size:#{as_size(db_bytesize)}(#{db_bytesize} byte)")
-            unless ask_yes_no('Do you want to continue?')
-              log_warn_stderr("To change the dump file directory, delete '#' and modify the path of 'mysqldump_dir:' in '#{Preference::DataEntryPreference.conf_path(de)}'")
-              exit 1
+          if file_dump
+            # check free disk space
+            free_disk_bytesize = free_disk_space(File.dirname(fp))
+            if (free_disk_bytesize - db_bytesize) < (1024 * 1024 * 1024) # 1GB
+              log_warn_stderr("!!WARNING There may not be enough disk space for a DB dump. We recommend 1GB free disk space after the dump. free disk space:#{as_size(free_disk_bytesize)}(#{free_disk_bytesize} byte) /" +
+                              " db size:#{as_size(db_bytesize)}(#{db_bytesize} byte)")
+              unless ask_yes_no('Do you want to continue?')
+                log_warn_stderr("To change the dump file directory, delete '#' and modify the path of 'mysqldump_dir:' in '#{Preference::DataEntryPreference.conf_path(de)}'")
+                exit 1
+              end
             end
           end
 
+
           log_info_stdout("Exporting data from database.")
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
-          Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], fp).check
+          Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], dump_dir: fp, backup_dir: sync_fm.backup_dir).check
           if file_dump
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
               new(de['mysql_data_entry_preference']).dump(fp)
@@ -541,7 +546,7 @@ Thank you for using FlyData!
           if de['mysql_data_entry_preference']['initial_sync']
             sync_fm.save_binlog(info[:binlog_pos])
           end
-          sync_fm.move_table_binlog_files(de['mysql_data_entry_preference']['tables'].split(','))
+          sync_fm.install_table_binlog_files(de['mysql_data_entry_preference']['tables'].split(','))
           sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'].split(','))
           sync_fm.delete_dump_file
           sync_fm.backup_dump_dir

data/lib/flydata/compatibility_check.rb

@@ -81,7 +81,8 @@ module Flydata
     def initialize(dp_hash, de_hash, options={})
       super
       @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
-      @dump_dir = options['dump_dir'] || nil
+      @dump_dir = options[:dump_dir] || nil
+      @backup_dir = options[:backup_dir] || nil
     end
 
     def print_errors
@@ -187,10 +188,11 @@ module Flydata
       write_errors = []
       paths_to_check = ["~/.flydata"]
       paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
+      paths_to_check << @backup_dir unless @backup_dir.to_s.empty?
       paths_to_check.each do |path|
         full_path = File.expand_path(path)
         full_path = File.dirname(full_path) unless File.directory?(full_path)
-        write_errors << full_path unless File.writable?(full_path)
+        write_errors << full_path unless File.writable?(full_path) and File.executable?(full_path)
       end
       unless write_errors.empty?
         error_dir = write_errors.join(", ")
@@ -242,6 +244,3 @@ module Flydata
   end
 
 end
-
-
-

data/lib/flydata/sync_file_manager.rb

@@ -79,6 +79,16 @@ module Flydata
           tables.map{|table| File.join(table_positions_dir_path, table + '.pos')}
       end
 
+      def table_binlog_pos_paths(*tables)
+        tables.empty? ? Dir.glob(File.join(table_positions_dir_path, '*.binlog.pos')) :
+          tables.map{|table| File.join(table_positions_dir_path, table + '.binlog.pos')}
+      end
+
+      def table_binlog_pos_init_paths(*tables)
+        tables.empty? ? Dir.glob(File.join(table_positions_dir_path, '*.binlog.pos.init')) :
+          tables.map{|table| File.join(table_positions_dir_path, table + '.binlog.pos.init')}
+      end
+
       # Read a sequence number from the table's position file,
       # increment the number and pass the number to a block.
       # After executing the block, saves the value to the position
@@ -184,14 +194,18 @@ module Flydata
         end
       end
 
-      def move_table_binlog_files(tables)
+      def install_table_binlog_files(tables)
         FileUtils.mkdir_p(table_positions_dir_path) unless Dir.exists?(table_positions_dir_path)
         tables.each do |table_name|
-          file = File.join(dump_dir, table_name + ".binlog.pos")
-          if ! File.exists?(file)
-            raise "#{file} does not exist. Error!!"
+          file_name = table_name + ".binlog.pos"
+          src_file = File.join(dump_dir, file_name)
+          if ! File.exists?(src_file)
+            raise "#{src_file} does not exist. Error!!"
           end
-          FileUtils.mv(file, table_positions_dir_path)
+          FileUtils.mv(src_file, table_positions_dir_path)
+          # save the position at initial sync.  this is used for repair if
+          # necessary.
+          FileUtils.cp(File.join(table_positions_dir_path, file_name), File.join(table_positions_dir_path, file_name + ".init"))
         end
       end
 
@@ -204,7 +218,11 @@ module Flydata
         FileUtils.mkdir_p(backup_dir) unless Dir.exists?(backup_dir)
         dest_dir = File.join(backup_dir, Time.now.strftime("%Y%m%d%H%M%S"))
         FileUtils.mkdir(dest_dir)
-        FileUtils.mv(dump_dir, dest_dir)
+        FileUtils.mv(Dir.glob("#{dump_dir}/*"), dest_dir)
+      end
+
+      def backup_dir
+        BACKUP_DIR
       end
 
       private

data/spec/flydata/sync_file_manager_spec.rb

@@ -60,6 +60,7 @@ module Flydata
             default_data_entry['mysql_data_entry_preference'].delete('mysqldump_dir')
           end
           it do
+            stub_const('Flydata::FileUtil::SyncFileManager::DUMP_DIR', File.join(Flydata::FLYDATA_HOME, 'dump'))
             expect(default_sfm.dump_file_path).to eq(
               File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.27
+  version: 0.2.28
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-08 00:00:00.000000000 Z
+date: 2015-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -566,7 +566,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: FlyData Agent