flydata 0.3.12 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df760e4cdf0cdaa8feea1af41d0441dfd0d1b25f
-  data.tar.gz: 48ccb962e9470ac32df552f0d7c330de57c69019
+  metadata.gz: 317626ac900b0a0adf96c84648e554f6d2e7436a
+  data.tar.gz: e54dcf2b46068be7c27e3d6ce63c338d111e0c20
 SHA512:
-  metadata.gz: 5b9558a312db75ae958d258a59b953a646b5a577d14647b032f7d2797ee464474a8e0c6058abc1c16563b2aaaa486635e94b807c2052a861ed9b7ad798cd3ca1
-  data.tar.gz: 745fc312f4ab0f4aaaca698e21d9ae128281fc2681e28baddbffdbba0e486716b098047cad438f4159a029142b18ad2c520e7890cfefa991f6ecb7172ccc9e66
+  metadata.gz: 935c6cc893c91fba078e92306b88a06697195c3a21b5241f0d08936a9cfb8acdff7c9eb32febbdc5c77a04b5dbe9627c757cf123ceee0420ae29784e4e10e1b4
+  data.tar.gz: 593a4ade45a37713981a11c5d98456eaaa04a592870cab4b2aff48a162e17d36c4780a138df5c38636d07bc9c3879d3ddde02e6331c94be916c62cc9b1d2f58a

data/VERSION

@@ -1 +1 @@
-0.3.12
+0.3.13

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.12 ruby lib
+# stub: flydata 0.3.13 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.12"
+  s.version = "0.3.13"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-04-08"
+  s.date = "2015-04-09"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -172,7 +172,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.2.2"
+  s.rubygems_version = "2.4.3"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/sender.rb

@@ -35,7 +35,7 @@ module Flydata
         Flydata::Command::Sync.new.try_mysql_sync
 
         # Start sender(fluentd) process
-        log_info_stdout("Starting sender process.") unless options[:quiet]
+        log_info_stdout("Starting FlyData Agent sender process.") unless options[:quiet]
         raw_start(options)
 
         wait_until_client_ready(options)

data/lib/flydata/command/sync.rb

@@ -285,13 +285,15 @@ EOS
         de = data_entry
 
         unless @new_tables.empty?
-          say("We've noticed that these tables have not been synced yet: #{@new_tables.join(", ")}")
+          unsynced_table_message = "We've noticed that these tables have not been synced yet: #{@new_tables.join(", ")}\n"
           unless @ddl_tables.empty?
-            say("  WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.")
-            say("    We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'")
-            say("    to generate SQL to run on Redshift to create the correct tables")
-            say("    Without running this sql on your Redshift cluster, there may be issues with your data")
+            unsynced_table_message << 
+            "  WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.\n" +
+            "    We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'\n" +
+            "    to generate SQL to run on Redshift to create the correct tables\n" +
+            "    Without running this sql on your Redshift cluster, there may be issues with your data"
           end
+          log_info_stdout(unsynced_table_message)
           if ask_yes_no("Do you want to run initial sync on all of these tables now?")
             initial_sync(de)
           else
@@ -308,6 +310,7 @@ EOS
 
       def initial_sync(de)
         # Load sync information from file
+        log_info_stdout("Checking FlyData Servers for existing buffer data...")
         load_sync_info(de)
         validate_initial_sync_status
         begin
@@ -393,7 +396,7 @@ EOM
         if ask_yes_no('Start Sync?')
           log_info_stdout("Checking MySQL server connection and configuration...")
           Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], dump_dir: fp, backup_dir: sync_fm.backup_dir).check
-          log_info_stdout("Checking database size...")
+          log_info_stdout("Checking the database size...")
 
           db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).get_db_bytesize
           log_info_stdout("  -> #{as_size(db_bytesize)} (#{db_bytesize} byte)")
@@ -411,12 +414,20 @@ EOM
             end
           end
 
-          log_info_stdout("Exporting data from database.")
+          log_info_stdout("Exporting data from the database.")
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
           if file_dump
-            binlog_pos = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
-              new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp)
-            log_info_stdout("  -> Done")
+            begin
+              binlog_pos = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
+                new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp)
+              log_info_stdout("  -> Database dump done")
+            rescue Exception => e
+              #Catch all exceptions including SystemExit and Interrupt.
+              log_info_stdout "Quit while running mysqldump, deleting dump file..."
+              sync_fm.delete_dump_file
+              log_info_stdout "Dump file deleted, to restart the FlyData Agent, please run the 'flydata start' command "
+              raise e
+            end
             call_block_or_return_io(fp, binlog_pos, &block)
           else
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
@@ -581,15 +592,16 @@ EOM
 
         state = :PROCESS
         start_time = Time.now
-        log_info_stdout("Uploading data to Redshift...")
+        #log_info_stdout("Uploading data to Redshift...")
+        log_info_stdout("Processing and uploading your data on FlyData Servers...")
         sleep 10
         status = nil
         prev_message =nil
         while (status = check_server_status(tables))
           if state == :PROCESS && status['state'] == 'uploading'
-            log_info_stdout("  -> Done")
+            log_info_stdout("  -> Data processing done")
             state = :UPLOAD
-            log_info_stdout("Finishing data upload...")
+            log_info_stdout("Uploading remaining data chunks...")
           end
 
           #TODO This is based on a temporary option
@@ -683,8 +695,8 @@ EOM
       end
 
       def cleanup_sync_server(de, tables = [])
-        print "Cleaning the server."
-        log_info("Cleaning the server.")
+        print("Cleaning the queued items on the FlyData Servers.")
+        log_info("Cleaning the queued items on the FlyData Servers.")
         worker = Thread.new do
           begin
             flydata.data_entry.cleanup_sync(de['id'], tables)

data/lib/flydata/compatibility_check.rb

@@ -226,7 +226,7 @@ module Flydata
         end
       rescue Mysql2::Error => e
         if e.message =~ /command denied to user/
-          log_warn_stderr("[WARNING]Cannot verify RDS retention period on currend MySQL user account.\n" +
+          log_warn_stderr("[WARNING]Cannot verify RDS retention period on current MySQL user account.\n" +
                           "To see retention period, please run this on your RDS:\n" +
                           "  $> call mysql.rds_show_configuration;\n" +
                           "Please verify that the hours is not nil and is at least #{BINLOG_RETENTION_HOURS} hours\n" +

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -268,7 +268,7 @@ EOS
               line = buffered_line
               buffered_line = nil
             else
-              rawline = dump_io.readline
+              rawline = dump_io.readline.encode('utf-16', :undef => :replace, :invalid => :replace).encode('utf-8')
               bytesize += rawline.bytesize
               line = rawline.strip
             end

data/spec/flydata/command/sync_spec.rb

@@ -65,6 +65,64 @@ module Flydata
           end
         end
       end
+      describe '#generate_mysqldump' do
+        let (:flydata) { double('flydata') }
+        let (:dp) { double('dp') }
+        let (:default_data_port) { double('default_data_port') }
+        let (:default_sync_fm) { double('default_sync_fm') }
+        let (:default_dump_pos) do {
+          status: "DO_NOT_SKIP_DUMP"
+        }
+        end
+        let (:default_fp) { double('default_fp') }
+        let (:default_backup) { double('default_backup') }
+        let (:target_tables) { ["test_table_1"] }
+        let (:db_byte) { 1 }
+        let (:disk_byte) { 100 }
+        before do
+          expect(subject).to receive(:flydata).and_return(flydata)
+          expect(flydata).to receive(:data_port).and_return(dp)
+          expect(dp).to receive(:get).and_return(default_data_port)
+          allow(File).to receive(:exists?).and_return(false)
+          expect(default_sync_fm).to receive(:load_dump_pos).and_return(default_dump_pos)
+          expect(default_sync_fm).to receive(:dump_file_path).and_return(default_fp)
+          expect(default_sync_fm).to receive(:backup_dir).and_return(default_backup)
+          expect(subject).to receive(:target_tables).and_return(target_tables).at_least(:once)
+          expect(subject).to receive(:print)
+          expect(subject).to receive(:log_info)
+          expect(subject).to receive(:log_info_stdout).at_least(:once)
+          expect(subject).to receive(:ask_yes_no).and_return(true).at_least(:once)
+          Flydata::Parser::Mysql::DatabaseSizeCheck.any_instance.should_receive(:get_db_bytesize).and_return(db_byte)
+          Flydata::MysqlCompatibilityCheck.any_instance.should_receive(:check)
+        end
+        context 'with no stream option' do
+          before do
+            expect(subject).to receive(:free_disk_space).and_return(disk_byte)
+          expect(File).to receive(:dirname)
+          end
+          it 'will export to dump file' do
+            expect(subject).to receive(:call_block_or_return_io)
+            Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.any_instance.should_receive(:dump).with(default_fp)
+
+            subject.send(:generate_mysqldump, default_data_entry, default_sync_fm)
+          end
+          it 'will remove dump file on interrupt' do
+            expect(default_sync_fm).to receive(:delete_dump_file)
+
+            expect {
+              Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.any_instance.should_receive(:dump).and_raise(Interrupt)
+              subject.send(:generate_mysqldump, default_data_entry, default_sync_fm)
+            }.to raise_error
+          end
+        end
+        context 'with stream option' do
+          it 'will export to io' do
+            Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.any_instance.should_receive(:dump)
+
+            subject.send(:generate_mysqldump, default_data_entry, default_sync_fm, false)
+          end
+        end
+      end
       describe '#do_generate_table_ddl' do
         before do
           allow(subject).to receive(:data_entry).and_return(default_data_entry)

data/spec/flydata/parser/mysql/dump_parser_spec.rb

@@ -242,7 +242,7 @@ EOT
             end
           end
 
-          context 'when dump contains create table with multi inserts' do
+          context 'when dump contains create table with multi inserts and invalid utf8 byte sequences' do
             let(:dump_content) { <<EOT
 #{DUMP_HEADER}
 
@@ -265,7 +265,7 @@ CREATE TABLE `users_login` (
 
 LOCK TABLES `users_login` WRITE;
 /*!40000 ALTER TABLE `users_login` DISABLE KEYS */;
-INSERT INTO `users_login` VALUES (15,46,'moodier','2001-10-02 08:18:08','1986-06-11 22:50:10'),(35,6,'missあいうえおteer','1991-10-15 19:38:07','1970-10-01 22:03:10'),(52,33,'sub\\\\field','1972-08-23 20:16:08','1974-10-10 23:28:11');
+INSERT INTO `users_login` VALUES (15,46,'moodier\xED\xB1\xA2','2001-10-02 08:18:08','1986-06-11 22:50:10'),(35,6,'missあいうえおteer','1991-10-15 19:38:07','1970-10-01 22:03:10'),(52,33,'sub\\\\field','1972-08-23 20:16:08','1974-10-10 23:28:11');
 INSERT INTO `users_login` VALUES (373,31,'out\\'swearing','1979-10-07 08:10:08','2006-02-22 16:26:04'),(493,8,'schizophrenic','1979-07-06 07:34:07','1970-08-09 01:21:01');
 /*!40000 ALTER TABLE `users_login` ENABLE KEYS */;
 UNLOCK TABLES;
@@ -284,7 +284,7 @@ EOT
 
               # insert_record_block
               [
-                [ %w(15 46 moodier 2001-10-02\ 08:18:08 1986-06-11\ 22:50:10),
+                [ %W(15 46 moodier\uFFFD\uFFFD\uFFFD 2001-10-02\ 08:18:08 1986-06-11\ 22:50:10),
                   %w(35 6 missあいうえおteer  1991-10-15\ 19:38:07 1970-10-01\ 22:03:10),
                   %w(52 33 sub\\field 1972-08-23\ 20:16:08 1974-10-10\ 23:28:11),],
                 [ %w(373 31 out'swearing 1979-10-07\ 08:10:08 2006-02-22\ 16:26:04),

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.13
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-08 00:00:00.000000000 Z
+date: 2015-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -632,7 +632,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent