flydata 0.1.8 → 0.1.9

data/lib/flydata/table_def/redshift_table_def.rb

@@ -22,11 +22,11 @@ class RedshiftTableDef
     'int4 unsigned' => {type: 'int8', unsigned: true},
     'int8' => {type: 'int8'},
     'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true},
-    'numeric' => {type: 'numeric', use_params: true},
-    'numeric unsigned' => {type: 'numeric', use_params: true},
+    'numeric' => {type: 'numeric', use_params: true, max_size: [38,37]},
+    'numeric unsigned' => {type: 'numeric', use_params: true, max_size: [38,37]},
     'text' => {type: 'varchar(max)'},
     'time' => {type: 'timestamp'},
-    'varbinary' => {type: 'varchar', use_params: true},
+    'varbinary' => {type: 'varchar', use_params: true, max_size: 65535},
     'varchar' => {type: 'varchar', use_params: true, max_size: 65535},
   }
   def self.from_flydata_tabledef(flydata_tabledef, options = {})
@@ -85,9 +85,7 @@ EOS
     raise "Unsupported type '#{column[:type]}'" if type_info.nil?
 
     rs_type = if type_info[:use_params] && params && !params.nil?
-                if type_info[:max_size] && /\d+/.match(params) && params.to_i > type_info[:max_size]
-                  params = type_info[:max_size]
-                end
+                params = check_and_replace_max(params, Array(type_info[:max_size])) if type_info[:max_size]
                 type_info[:type] + "(#{params})"
               else
                 type_info[:type]
@@ -150,6 +148,15 @@ EOS
   def self.escape(text)
     text.gsub("'", "\\\\'")
   end
+  
+  def self.check_and_replace_max(params, max_size_a)
+      final_params = []
+      params.split(",").each_with_index do |param, i|
+        final_params << (/\d+/.match(param) && max_size_a[i] && param.to_i > max_size_a[i].to_i ? 
+                            max_size_a[i] : param)
+      end
+      final_params.join(",")
+  end
 end
 
 end

data/spec/flydata/cli_spec.rb

@@ -0,0 +1,172 @@
+require 'spec_helper'
+
+module Flydata
+  describe Cli do
+    subject { Cli.new(args) }
+    describe '#run' do
+      let(:command_obj) { double('command_obj') }
+      let(:args) { options.unshift command }
+      let(:empty_slop) { double('empty_slop') }
+      let (:sub_command) { "subcomm" }
+      context 'target commands supporting no options' do
+        let(:main_command) { "maincomm" }
+        let(:target_class) { module Command; class Maincomm; end; end; Command::Maincomm }
+        context 'without subcommand' do
+          let(:command) { main_command }
+          context 'without options' do
+            let(:options) { [] }
+            it 'calls #run of the target class object' do
+              expect(target_class).to receive(:new){|a|
+                expect(a.to_hash).to be_empty
+              }.and_return(command_obj)
+              expect(command_obj).to receive(:run)
+  
+              subject.run
+            end
+          end
+          context 'with options' do
+            let(:options) { %w(-a --host-name localhost) }
+            it 'returns with an error' do
+              expect(subject).to receive(:puts).with('! Unknown options -a, --host-name').once
+              allow(subject).to receive(:puts)
+
+              subject.run
+            end
+          end
+          context 'with arguments' do
+            let(:options) { %w(a b c ) }
+            it 'calls #run of the target class instance with arguments' do
+              expect(target_class).to receive(:new){|a|
+                expect(a.to_hash).to be_empty
+              }.and_return(command_obj)
+              expect(command_obj).to receive(:run).with(*options)
+  
+              subject.run
+            end
+          end
+        end
+        context 'with subcommand' do
+          let (:command) { "#{main_command}:#{sub_command}" }
+          context 'without options' do
+            let(:options) { [] }
+            it 'calls subcommand method of the target class object' do
+              expect(target_class).to receive(:new){|a|
+                expect(a.to_hash).to be_empty
+              }.and_return(command_obj)
+
+              expect(command_obj).to receive(sub_command.to_sym)
+
+              subject.run
+            end
+          end
+          context 'with optionss' do
+            let(:options) { %w(-a --host-name localhost) }
+            it 'returns an error' do
+              expect(subject).to receive(:puts).with('! Unknown options -a, --host-name').once
+              allow(subject).to receive(:puts)
+
+              subject.run
+            end
+          end
+        end
+      end
+      context 'target commands supporting options' do
+        let(:main_command) { "maincommop" }
+        let(:target_class) do
+          module Command
+            class Maincommop < Base
+              # Options for the main command
+              def self.slop
+                Slop.new(strict: true) do
+                  on 'a', 'short option'
+                  on 'host-name=', 'long option taking an argument'
+                end
+              end
+
+              # Options for the sub command 'subcomm'
+              def self.slop_subcomm
+                Slop.new(strict: true) do
+                  on 'b', 'short option'
+                  on 'port=', 'Port', as: Integer
+                end
+              end
+            end
+          end
+          Command::Maincommop
+        end
+        context 'without subcommand' do
+          let(:command) { main_command }
+          context 'without options' do
+            let(:options) { [] }
+            it 'calls #run of the target class object' do
+              expect(target_class).to receive(:new) { |a|
+                expect(a.a?).to be false
+                expect(a[:"host-name"]).to be_nil
+              }.and_return(command_obj)
+
+              expect(command_obj).to receive(:run)
+
+              subject.run
+            end
+          end
+          context 'with options' do
+            let(:options) { %w(-a --host-name localhost) }
+            it 'calls #run of the target class object' do
+              expect(target_class).to receive(:new) { |a|
+                expect(a.a?).to be true
+                expect(a[:'host-name']).to eq "localhost"
+              }.and_return(command_obj)
+
+              expect(command_obj).to receive(:run)
+
+              subject.run
+            end
+          end
+          context 'with options and arguments' do
+            let(:arguments) { %w(foo bar bal) }
+            let(:options) { %w(foo -a bar --host-name localhost bal) }
+            it 'calls #run of the target class object' do
+              expect(target_class).to receive(:new) { |a|
+                expect(a.a?).to be true
+                expect(a[:'host-name']).to eq "localhost"
+              }.and_return(command_obj)
+
+              expect(command_obj).to receive(:run).with(*arguments)
+
+              subject.run
+            end
+          end
+        end
+        context 'with subcommand' do
+          let (:command) { "#{main_command}:#{sub_command}" }
+          context 'without options' do
+            let(:options) { [] }
+            it 'calls the subcommand of the target class object' do
+              expect(target_class).to receive(:new) { |a|
+                expect(a.b?).to be false
+                expect(a[:port]).to be_nil
+              }.and_return(command_obj)
+
+              expect(command_obj).to receive(sub_command.to_sym)
+
+              subject.run
+            end
+          end
+          context 'with options' do
+            let(:options) { %w(-b --port 5439) }
+            it 'calls the subcommand of the target class object' do
+              expect(target_class).to receive(:new) { |a|
+                expect(a.b?).to be true
+                expect(a[:port]).to eq 5439
+              }.and_return(command_obj)
+
+              expect(command_obj).to receive(sub_command.to_sym)
+
+              subject.run
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/flydata/command/sender_spec.rb

@@ -1,9 +1,47 @@
 require 'spec_helper'
+require 'slop'
 
 module Flydata
   module Command
     describe Sender do
-      pending '#start'
+      let(:opts) do
+        slop = Sender.slop_start
+        slop.parse(args)
+        slop
+      end
+      subject { Sender.new(opts) }
+      describe '#start' do
+        before(:each) do
+          allow(subject).to receive(:process_exist?).and_return(false)
+          expect(subject).to receive(:wait_until_server_ready)
+          expect(subject).to receive(:wait_until_client_ready)
+          allow(Kernel).to receive(:sleep)
+        end
+        context "as daemon" do
+          let(:args) { [] }
+          it "starts fluend with daemon option" do
+            expect(Kernel).to receive(:system).with( Regexp.new(
+"fluentd -d .+/flydata.pid -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
+            subject.start(false)
+          end
+        end
+        context "as regular process" do
+          let(:args) { ["-n"] }
+          it "starts fluentd with no daemon option" do
+            expect(Kernel).to receive(:system).with(Regexp.new(
+"fluentd  -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
+            subject.start(false)
+          end
+        end
+        context "as regular process with long option" do
+          let(:args) { ["--no-daemon"] }
+          it "starts fluentd with no daemon option" do
+            expect(Kernel).to receive(:system).with(Regexp.new(
+"fluentd  -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
+            subject.start(false)
+          end
+        end
+      end
       pending '#stop'
       pending '#restart'
     end

data/spec/flydata/command/sync_spec.rb

@@ -1,6 +1,5 @@
 # coding: utf-8
 require 'spec_helper'
-require 'socket'
 
 module Flydata
   module Command
@@ -49,8 +48,8 @@ module Flydata
         subject { Sync.new }
         context 'with full options' do
           it 'issues mysqldump command with expected parameters' do
-            expect(IO).to receive(:popen).with(
-             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
+            expect(Open3).to receive(:popen3).with(
+             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2').and_call_original
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -69,8 +68,8 @@ module Flydata
             default_data_entry['mysql_data_entry_preference'].delete('port')
           end
           it "uses the default port" do
-            expect(IO).to receive(:popen).with(
-             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
+            expect(Open3).to receive(:popen3).with(
+             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2').and_call_original
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -79,8 +78,8 @@ module Flydata
             default_data_entry['mysql_data_entry_preference']['port'] = 1234
           end
           it "uses the specified port" do
-            expect(IO).to receive(:popen).with(
-             'mysqldump --protocol=tcp -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
+            expect(Open3).to receive(:popen3).with(
+             'mysqldump --protocol=tcp -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1  table2').and_call_original
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -99,8 +98,8 @@ module Flydata
             default_data_entry['mysql_data_entry_preference'].delete('password')
           end
           it "call mysqldump without -p option" do
-            expect(IO).to receive(:popen).with(
-             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi  sync_test table1  table2', 'r').and_call_original
+            expect(Open3).to receive(:popen3).with(
+             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi  sync_test table1  table2').and_call_original
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -1115,35 +1114,51 @@ EOT
         end
 
         context 'when data includes carriage return' do
-          let(:target_line) { "INSERT INTO `test_table` VALUES (1,'abcd\\refgh','2014-04-15 13:49:14');" }
+          let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'abcd\refgh','2014-04-15 13:49:14');| }
           it 'should escape return carriage' do
             expect(subject).to eq([['1',"abcd\refgh",'2014-04-15 13:49:14']])
           end
         end
 
         context 'when data includes new line' do
-          let(:target_line) { "INSERT INTO `test_table` VALUES (1,'abcd\\nefgh','2014-04-15 13:49:14');" }
+          let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'abcd\nefgh','2014-04-15 13:49:14');| }
           it 'should escape new line' do
             expect(subject).to eq([['1',"abcd\nefgh",'2014-04-15 13:49:14']])
           end
         end
 
         context 'when data includes escaped single quotation' do
-          let(:target_line) { "INSERT INTO `test_table` VALUES (1,'abcd\\',\\'efgh','2014-04-15 13:49:14');" }
+          let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'abcd\',\'efgh','2014-04-15 13:49:14');| }
           it 'should escape single quotation' do
             expect(subject).to eq([['1',"abcd','efgh",'2014-04-15 13:49:14']])
           end
         end
 
         context 'when data includes back slash' do
-          let(:target_line) { "INSERT INTO `test_table` VALUES (1,'abcd\\\\efgh','2014-04-15 13:49:14');" }
+          let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'abcd\\efgh','2014-04-15 13:49:14');| }
           it 'should escape back slash' do
             expect(subject).to eq([['1',"abcd\\efgh",'2014-04-15 13:49:14']])
           end
         end
 
+        context 'when data includes back slash with double quote' do
+          # \"\'\' value on insert query shoulde be "''
+          let(:target_line) { %q|INSERT INTO `tast_table` VALUES (1,'\"\'\'','2014-04-15 13:49:14');| }
+          it 'should escape back slash' do
+            expect(subject).to eq([['1',%q|"''| ,'2014-04-15 13:49:14']])
+          end
+        end
+
+        context 'when data includes back slash with double quote another case' do
+          # \"\"\"\"\"''\"'' value on insert query shoulde be """""''"''
+          let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'\"\"\"\"\"\'\'\"\'\'','2014-04-15 13:49:14');| }
+          it 'should escape back slash' do
+            expect(subject).to eq([['1',%q|"""""''"''|,'2014-04-15 13:49:14']])
+          end
+        end
+
         context 'when data includes mixed escaped characters' do
-          let(:target_line) { "INSERT INTO `test_table` VALUES (1,'ab\\rcd\\\\e\\nf\\',\\'gh','2014-04-15 13:49:14');" }
+          let(:target_line) { %q|INSERT INTO `test_table` VALUES (1,'ab\rcd\\e\nf\',\'gh','2014-04-15 13:49:14');| }
           it 'should escape all' do
             expect(subject).to eq([['1',"ab\rcd\\e\nf','gh",'2014-04-15 13:49:14']])
           end
@@ -1211,6 +1226,62 @@ EOT
             expect{subject}.to raise_error
           end
         end
+
+        context 'when an integer that has leading zeros is given' do
+          let(:target_line) {"INSERT INTO `test_table` VALUES (1,00013);"}
+          it 'should remove the leading zeros' do
+            expect(subject).to eq([['1', '13']])
+          end
+        end
+
+        context 'when a decimal that has leading zeros is given' do
+          let(:target_line) {"INSERT INTO `test_table` VALUES (1,00013.40);"}
+          it 'should remove the leading zeros' do
+            expect(subject).to eq([['1', '13.40']])
+          end
+        end
+
+        context 'when a timestamp that has leading zeros is given' do
+          let(:target_line) {"INSERT INTO `test_table` VALUES (1,'0004-04-15 13:49:14');"}
+          it 'should not remove the leading zeros' do
+            expect(subject).to eq([['1', '0004-04-15 13:49:14']])
+          end
+        end
+
+        context 'when a string that has leading zeros is given' do
+          let(:target_line) {"INSERT INTO `test_table` VALUES (1,'00000aa');"}
+          it 'should not remove the leading zeros' do
+            expect(subject).to eq([['1', '00000aa']])
+          end
+        end
+
+        context 'when a string that has leading zeros, numbers and a comma in between is given' do
+          let(:target_line) {"INSERT INTO `test_table` VALUES (1,'0003,00033');"}
+          it 'should not remove the leading zeros' do
+            expect(subject).to eq([['1', '0003,00033']])
+          end
+        end
+
+        context 'when a string that has leading zeros, has only numbers is given' do
+          let(:target_line) {"INSERT INTO `test_table` VALUES (1,'00033');"}
+          it 'should not remove the leading zeros' do
+            expect(subject).to eq([['1', '00033']])
+          end
+        end
+
+        context 'when 0 is given' do
+          let(:target_line) {"INSERT INTO `test_table` VALUES (1,0);"}
+          it 'should not remove the leading zeros' do
+            expect(subject).to eq([['1', '0']])
+          end
+        end
+
+        context 'when 0.00 is given' do
+          let(:target_line) {"INSERT INTO `test_table` VALUES (1,0.00);"}
+          it 'should not remove the leading zeros' do
+            expect(subject).to eq([['1', '0.00']])
+          end
+        end
       end
     end
   end

data/spec/flydata/table_def/mysql_table_def_spec.rb

@@ -32,7 +32,7 @@ describe MysqlTableDef do
           [
             {:name=>"id", :type=>"int8(20)", :auto_increment=>true, :not_null=>true, :primary_key=>true},
             {:name=>"col_binary", :type=>"binary(100)", :default=>nil},
-            {:name=>"col_blob", :type=>"varbinary"},
+            {:name=>"col_blob", :type=>"varbinary(65535)"},
             {:name=>"col_bool", :type=>"int1(1)", :default=>"0"},
             {:name=>"col_char", :type=>"varchar(18)", :default=>nil},
             {:name=>"col_date", :type=>"date", :default=>nil},
@@ -43,16 +43,16 @@ describe MysqlTableDef do
             {:name=>"col_float_4_2", :type=>"float4(4,2)", :default=>nil},
             {:name=>"col_int", :type=>"int4(11)", :default=>nil},
             {:name=>"col_int_6", :type=>"int4(6)", :default=>nil},
-            {:name=>"col_longblob", :type=>"varbinary"},
+            {:name=>"col_longblob", :type=>"varbinary(4294967295)"},
             {:name=>"col_longtext", :type=>"text"},
-            {:name=>"col_mediumblob", :type=>"varbinary"},
+            {:name=>"col_mediumblob", :type=>"varbinary(16777215)"},
             {:name=>"col_mediumint", :type=>"int3(9)", :default=>nil},
             {:name=>"col_mediumtext", :type=>"text"},
             {:name=>"col_smallint", :type=>"int2(6)", :default=>nil},
             {:name=>"col_text", :type=>"text"},
             {:name=>"col_time", :type=>"time", :default=>nil},
             {:name=>"col_timestamp", :type=>"datetime", :not_null=>true, :default=>"CURRENT_TIMESTAMP"},
-            {:name=>"col_tinyblob", :type=>"varbinary"},
+            {:name=>"col_tinyblob", :type=>"varbinary(255)"},
             {:name=>"col_tinyint", :type=>"int1(4)", :default=>nil},
             {:name=>"col_tinytext", :type=>"text"},
             {:name=>"col_varbinary", :type=>"varbinary(255)", :default=>nil},
@@ -130,6 +130,23 @@ describe MysqlTableDef do
         )
       end
     end
+    
+    context 'when table has unsigned column' do
+      let(:dump_file_io) { file_io('mysqldump_test_unsigned.dump')}
+      it 'should include unsigned in column type' do
+        expect(subject[:columns]).to eq(
+          [
+            {:name=>"id", :type=>"int4(11)", :auto_increment=>true, :not_null=>true, :primary_key=>true},                             
+            {:name=>"value_int", :type=>"int4(10) unsigned", :default=>nil},
+            {:name=>"value_float", :type=>"float4 unsigned", :default=>nil}, 
+            {:name=>"value_dec", :type=>"numeric(10,2) unsigned", :default=>nil}, 
+            {:name=>"value_double", :type=>"float8 unsigned", :default=>nil}, 
+            {:name=>"name", :type=>"varchar(768)", :default=>nil}, 
+            {:name=>"value_small_int", :type=>"int2(5) unsigned", :default=>nil}
+          ]
+        )      
+      end
+    end
   end
 end