flydata 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 197b34bb3ccfc2ed09015f6a13413ecdbda4f5ed
-  data.tar.gz: 0f04fd5270dee95ac4fe648ab5a8b2716302c5ce
+  metadata.gz: bfd589bd9943f41f8e817ec796aa74aef758d1f9
+  data.tar.gz: dc62b1dde189197fb3552abde072b40697e92dd5
 SHA512:
-  metadata.gz: ff9fc62260afc8ce527de57edb70052f5cb35357e87f7db5ad12207ea2f0237c672bf3667eea92a79a8dc4b882ec2e484ca9e3e6aa58d52dcae24a56377e87a3
-  data.tar.gz: 98a99de769067b51ce61f92a4bc6bc40e2651dbf5886aa0c4006cc1121136869984d0d97e50c27b409e4cfa1bd84c5a5c96406831cfd5496bcfb1839b11a800b
+  metadata.gz: 34006fd5e1f6052f901b92df572ae71c64e3deea4449d3c92aded6cf7013ecf5cd142e2d875c800c5778a5962af5d8add5627e315c3a46806df5980e8bbabceb
+  data.tar.gz: 9a7b312e2c03d2048878f211a6c0233d5d9d31bf7cce0cf655a75bc14287e226094779a45d0dd67df81d1b9f6d19aca1f74a4c04870da4fe6a969bcb4fa34ac5

data/VERSION

@@ -1 +1 @@
-0.2.6
+0.2.7

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.2.6 ruby lib
+# stub: flydata 0.2.7 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.6"
+  s.version = "0.2.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2014-09-26"
+  s.date = "2014-10-07"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -52,6 +52,7 @@ Gem::Specification.new do |s|
     "lib/flydata/command/version.rb",
     "lib/flydata/credentials.rb",
     "lib/flydata/cron.rb",
+    "lib/flydata/errors.rb",
     "lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb",
     "lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb",
     "lib/flydata/fluent-plugins/mysql/binlog_position.rb",

data/lib/flydata/command/sync.rb

@@ -3,6 +3,7 @@ require 'msgpack'
 require 'open3'
 require 'mysql2'
 require 'flydata/sync_file_manager'
+require 'flydata/errors'
 require 'flydata/table_def'
 #require 'ruby-prof'
 
@@ -179,24 +180,32 @@ module Flydata
           raise "mysqldump is not installed.  mysqldump is required to run the command"
         end
 
+        error_list = []
+
         schema_name = (de['schema_name'] || nil)
 
         mp = de['mysql_data_entry_preference']
         params = []
-        if mp['host'] then params << mp['host'] else raise "MySQL `host` is not defined in the data entry" end
+        if !mp['host'].empty? then params << mp['host'] else raise "MySQL `host` is neither defined in the data entry nor the local config file" end
         params << (mp['port'] or '3306')
-        if mp['username'] then params << mp['username'] else raise "MySQL `username` is not defined is not defined in the data entry" end
+        if !mp['username'].empty? then params << mp['username'] else raise "MySQL `username` is neither defined in the data entry nor the local config file" end
         params << (mp['password'].to_s.empty? ? "" : "-p#{mp['password']}")
-        if mp['database'] then params << mp['database'] else raise "`database` is not defined in the data entry" end
-        if mp['tables'] then params << mp['tables'].gsub(/,/, ' ') else raise "`tables` is not defined in the data entry" end
+        if !mp['database'].empty? then params << mp['database'] else raise "`database` is neither defined in the data entry nor the local config file" end
+        if !mp['tables'].empty? then params << mp['tables'].gsub(/,/, ' ') else raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file" end
 
         command = DDL_DUMP_CMD_TEMPLATE % params
 
         Open3.popen3(command) do |stdin, stdout, stderr|
           stdin.close
+          stdout.set_encoding("utf-8") # mysqldump output must be in UTF-8
           create_flydata_ctl_table = mp['initial_sync']
           while !stdout.eof?
-            mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(stdout)
+            begin
+              mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(stdout)
+            rescue TableDefError => e
+              error_list << e.err_hash
+              next
+            end
             if mysql_tabledef.nil?
               # stream had no more create table definition
               break
@@ -205,10 +214,23 @@ module Flydata
             puts Flydata::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
             create_flydata_ctl_table = false
           end
+          errors = ""
           while !stderr.eof?
-            line = stderr.gets
-            $stderr.print line unless /Warning: Using a password on the command line interface can be insecure./ === line
+            line = stderr.gets.gsub('mysqldump: ', '')
+            errors << line unless /Warning: Using a password on the command line interface can be insecure./ === line
+          end
+          raise errors unless errors.empty?
+        end
+        unless error_list.empty?
+          $stderr.puts "We have noticed the following error(s):"
+          group_error =  error_list.group_by {|d| d[:error]}
+          group_error.each_key do |a|
+            $stderr.puts "The following table(s) have #{a}:"
+            group_error[a].each do |hash|
+              $stderr.puts "  - #{hash[:table]}" if hash[:table]
+            end
           end
+          $stderr.puts "Please fix the above error(s) and try again."
         end
       end
 
@@ -851,7 +873,7 @@ EOT
     class MysqlDumpGeneratorMasterData < MysqlDumpGenerator
       EXTRA_MYSQLDUMP_PARAMS = "--flush-logs --master-data=2"
       def dump(file_path)
-        cmd = "#{@dump_cmd} > #{file_path}"
+        cmd = "#{@dump_cmd} -r #{file_path}"
         o, e, s = Open3.capture3(cmd)
         e.to_s.each_line {|l|  puts l unless /^Warning:/ =~ l } unless e.to_s.empty?
         unless s.exitstatus == 0
@@ -913,7 +935,8 @@ EOS
           # start dump
           Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err|
             cmd_in.close_write
-            File.open(file_path, "w") do |f|
+            cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
+            File.open(file_path, "w", encoding: "utf-8") do |f|
               find_insert_pos = :not_started
               cmd_out.each_line do |line|
                 if find_insert_pos == :not_started && /^-- Server version/ === line
@@ -1111,7 +1134,7 @@ EOS
         end
 
         # Start reading file from top
-        File.open(@file_path, 'r') do |f|
+        File.open(@file_path, 'r', encoding: "utf-8") do |f|
           last_saved_pos = 0
 
           # resume

data/lib/flydata/errors.rb

@@ -0,0 +1,8 @@
+module Flydata
+  class TableDefError < StandardError
+    attr_reader :err_hash
+    def initialize(err_hash)
+      @err_hash = err_hash
+    end
+  end
+end

data/lib/flydata/table_def/mysql_table_def.rb

@@ -121,7 +121,7 @@ class MysqlTableDef
 
       when :after_create_table
         unless columns.any? {|column| column[:primary_key]}
-          raise "Primary key must be defined."
+          raise TableDefError, {error: "no primary key defined", table: table_name}
         end
         break
       end

data/spec/flydata/command/sync_spec.rb

@@ -46,10 +46,17 @@ module Flydata
 
       describe '#do_generate_table_ddl' do
         subject { Sync.new }
+        shared_examples 'throws an error' do
+          it "throws an error" do
+            expect {
+              subject.send(:do_generate_table_ddl, default_data_entry)
+            }.to raise_error
+          end
+        end
         context 'with full options' do
           it 'issues mysqldump command with expected parameters' do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2').and_call_original
+             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -57,11 +64,13 @@ module Flydata
           before do
             default_data_entry['mysql_data_entry_preference'].delete('host')
           end
-          it "throws an error" do
-            expect {
-              subject.send(:do_generate_table_ddl, default_data_entry)
-            }.to raise_error
+          include_examples 'throws an error'
+        end
+        context 'with empty host' do
+          before do
+            default_data_entry['mysql_data_entry_preference']['host'] = ""
           end
+          include_examples 'throws an error'
         end
         context 'without_port' do
           before do
@@ -69,7 +78,7 @@ module Flydata
           end
           it "uses the default port" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2').and_call_original
+             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -79,7 +88,7 @@ module Flydata
           end
           it "uses the specified port" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump --protocol=tcp -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1  table2').and_call_original
+             'mysqldump --protocol=tcp -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1  table2')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -87,11 +96,13 @@ module Flydata
           before do
             default_data_entry['mysql_data_entry_preference'].delete('username')
           end
-          it "throws an error" do
-            expect {
-              subject.send(:do_generate_table_ddl, default_data_entry)
-            }.to raise_error
+          include_examples 'throws an error'
+        end
+        context 'with empty username' do
+          before do
+            default_data_entry['mysql_data_entry_preference']['username'] = ""
           end
+          include_examples 'throws an error'
         end
         context 'without_password' do
           before do
@@ -99,7 +110,7 @@ module Flydata
           end
           it "call mysqldump without -p option" do
             expect(Open3).to receive(:popen3).with(
-             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi  sync_test table1  table2').and_call_original
+             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi  sync_test table1  table2')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -107,21 +118,25 @@ module Flydata
           before do
             default_data_entry['mysql_data_entry_preference'].delete('database')
           end
-          it "throws an error" do
-            expect {
-              subject.send(:do_generate_table_ddl, default_data_entry)
-            }.to raise_error
+          include_examples 'throws an error'
+        end
+        context 'with empty database' do
+          before do
+            default_data_entry['mysql_data_entry_preference']['database'] = ""
           end
+          include_examples 'throws an error'
         end
         context 'without_tables' do
           before do
             default_data_entry['mysql_data_entry_preference'].delete('tables')
           end
-          it "throws an error" do
-            expect {
-              subject.send(:do_generate_table_ddl, default_data_entry)
-            }.to raise_error
+          include_examples 'throws an error'
+        end
+        context 'with empty tables' do
+          before do
+            default_data_entry['mysql_data_entry_preference']['tables'] = ""
           end
+          include_examples 'throws an error'
         end
       end
     end

data/spec/flydata/table_def/mysql_table_def_spec.rb

@@ -73,7 +73,7 @@ describe MysqlTableDef do
     context 'when table does not have primary key' do
       let(:dump_file_io) { file_io('mysqldump_test_table_no_pk.dump') }
       it 'should raise an error' do
-        expect{subject}.to raise_error('Primary key must be defined.')
+        expect{subject}.to raise_error(Flydata::TableDefError)
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-26 00:00:00.000000000 Z
+date: 2014-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -421,6 +421,7 @@ files:
 - lib/flydata/command/version.rb
 - lib/flydata/credentials.rb
 - lib/flydata/cron.rb
+- lib/flydata/errors.rb
 - lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb
 - lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb
 - lib/flydata/fluent-plugins/mysql/binlog_position.rb