flydata 0.6.14 → 0.7.0

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -267,12 +267,17 @@ EOT
 
     shared_examples "test column types" do |*examples|
       context 'with varchar column def' do
+        let(:default_value) { "something" }
+        let(:default_value_sql) { "'#{default_value}'" }
+        let(:not_null_default_sql) { " DEFAULT ''" }
+        before do
+          column[:column] = "col_char"
+            column[:type] = "varchar(255)"
+        end
+        let(:type_sql) { %Q|"col_char" varchar(255)| }
+
         context 'when size is smaller than max size' do
-          let(:default_value) { "something" }
-          let(:default_value_sql) { "'#{default_value}'" }
-          let(:not_null_default_sql) { " DEFAULT ''" }
           before do
-            column[:column] = "col_char"
             column[:type] = "varchar(18)"
           end
           let(:type_sql) { %Q|"col_char" varchar(18)| }
@@ -281,11 +286,7 @@ EOT
         end
 
         context 'when size exceeds varchar max size' do
-          let(:default_value) { "something" }
-          let(:default_value_sql) { "'#{default_value}'" }
-          let(:not_null_default_sql) { " DEFAULT ''" }
           before do
-            column[:column] = "col_char"
             column[:type] = "varchar(1000000000)"
           end
           let(:type_sql) { %Q|"col_char" varchar(65535)| }
@@ -294,17 +295,21 @@ EOT
         end
 
         context 'when size is max size' do
-          let(:default_value) { "something" }
-          let(:default_value_sql) { "'#{default_value}'" }
-          let(:not_null_default_sql) { " DEFAULT ''" }
           before do
-            column[:column] = "col_char"
             column[:type] = "varchar(max)"
           end
           let(:type_sql) { %Q|"col_char" varchar(max)| }
 
           it_behaves_like *examples
         end
+
+        context 'when default_value has PostgreSQL-style type cast' do
+          let(:default_value) { "'2014-10-17'::text" }
+          let(:default_value_sql) { "'2014-10-17'" }
+
+          it_behaves_like *examples
+        end
+
       end
 
       context 'with medium blob column def' do
@@ -335,6 +340,34 @@ EOT
         it_behaves_like *examples
       end
 
+      context 'with boolean column def' do
+        context 'with true as column_default' do
+          let(:default_value) { true }
+          let(:default_value_sql) { "#{default_value.to_s.upcase}" }
+          let(:not_null_default_sql) { " DEFAULT FALSE" }
+          before do
+            column[:column] = "value_boolean"
+            column[:type] = "boolean"
+          end
+          let(:type_sql) { %Q|"value_boolean" boolean| }
+
+          it_behaves_like *examples
+        end
+
+        context 'with false as column_default' do
+          let(:default_value) { false }
+          let(:default_value_sql) { "#{default_value.to_s.upcase}" }
+          let(:not_null_default_sql) { " DEFAULT FALSE" }
+          before do
+            column[:column] = "value_boolean"
+            column[:type] = "boolean"
+          end
+          let(:type_sql) { %Q|"value_boolean" boolean| }
+
+          it_behaves_like *examples
+        end
+      end
+
       context 'with date column def' do
         before do
           column[:column] = "value_date"
@@ -389,6 +422,39 @@ EOT
         end
       end
 
+      context 'with datetimetz column def' do
+        before do
+          column[:column] = "value_datetimetz"
+          column[:type] = "datetimetz"
+        end
+        let(:type_sql) { %Q|"value_datetimetz" timestamp| }
+        let(:not_null_default_sql) { " DEFAULT '0000-01-01'" }
+
+        context 'when default_value is normal' do
+          let(:default_value) { "'2014-10-17 22:22:22+00'::timestamp with time zone" }
+          let(:default_value_sql) { "'2014-10-17 22:22:22.000000'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is current_timestamp' do
+          let(:default_value) { 'current_timestamp' }
+          let(:default_value_sql) { "SYSDATE" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is now()' do
+          let(:default_value) { 'now()' }
+          let(:default_value_sql) { "SYSDATE" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 0000-00-00 00:00:00-07' do
+          let(:default_value) { "'0000-00-00 00:00:00-07'" }
+          let(:default_value_sql) { "'0001-01-01 07:00:00.000000'" }
+          it_behaves_like *examples
+        end
+      end
+
       context 'with decimal column def' do
         context 'when precision exceeds max allowed' do
           let(:default_value) { "'4'" }
@@ -402,19 +468,33 @@ EOT
 
           it_behaves_like *examples
         end
-        context 'when scale exceeds max allowed' do #not possible
+        context 'when scale exceeds max allowed' do #possible in postgresql
           let(:default_value) { "'4'" }
           let(:default_value_sql) { default_value }
           let(:not_null_default_sql) { " DEFAULT '0'" }
           before do
             column[:column] = "value"
-            column[:type] = "numeric(45,44)"
+            column[:type] = "numeric(100,44)"
           end
+          #this column accepts only 1 digit above decimal point. Other values such as 10 cause DDE.
           let(:type_sql) { %Q|"value" numeric(38,37)| }
 
           it_behaves_like *examples
         end
 
+        context 'when precision and scale are not given' do #possible in postgresql
+          let(:default_value) { "'4'" }
+          let(:default_value_sql) { default_value }
+          let(:not_null_default_sql) { " DEFAULT '0'" }
+          before do
+            column[:column] = "value"
+            column[:type] = "numeric"
+          end
+          let(:type_sql) { %Q|"value" numeric(18,8)| }
+
+         it_behaves_like *examples
+        end
+
         context 'when precision and scale exceeds max allowed (for unsigned)' do
           let(:default_value) { "'4'" }
           let(:default_value_sql) { default_value }
@@ -462,6 +542,18 @@ EOT
           it_behaves_like *examples
         end
 
+        context 'when default_value uses the capital B prefix' do
+          let(:default_value) { "B'1'" }
+          let(:default_value_sql) { "1" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value has a type cast' do
+          let(:default_value) { %Q|B'1'::"bit"| }
+          let(:default_value_sql) { "1" }
+          it_behaves_like *examples
+        end
+
         # it's not real case
         context "when default_value is x'' format" do
           let(:default_value) { "x'12'" }
@@ -488,6 +580,40 @@ EOT
         end
       end
 
+      context 'with timetz column def' do
+        before do
+          column[:column] = "value_timetz"
+          column[:type] = "timetz"
+        end
+        let(:type_sql) { %Q|"value_timetz" timestamp| }
+        let(:not_null_default_sql) { " DEFAULT '0000-01-01'" }
+
+        context 'when default_value is normal' do
+          let(:default_value) { "'22:22:22+00'::time with time zone" }
+          let(:default_value_sql) { "'0001-01-01 22:22:22.000000'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is current_time' do
+          let(:default_value) { 'current_time' }
+          let(:default_value_sql) { "SYSDATE" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 00:00:00-07' do
+          let(:default_value) { "'00:00:00-07'" }
+          let(:default_value_sql) { "'0001-01-01 07:00:00.000000'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 00:00:00+07' do
+          let(:default_value) { "'00:00:00+07'" }
+          # Per ISO 8601, year 0000 is year 1 BC
+          let(:default_value_sql) { "'0000-12-31 17:00:00.000000'" }
+          it_behaves_like *examples
+        end
+      end
+
       context 'with year column def' do
         before do
           column[:column] = "value_year"
@@ -531,6 +657,52 @@ EOT
           end
         end
       end
+      context 'with money column def' do
+        before do
+          column[:column] = "value_money"
+          column[:type] = "money(19,2)"
+        end
+        let(:type_sql) { %Q|"value_money" numeric(19,2)| }
+        let(:not_null_default_sql) { " DEFAULT '0'" }
+
+        context 'when default_value is normal' do
+          let(:default_value) { "'10.40'" }
+          let(:default_value_sql) { "'10.40'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is in a monetary format' do
+          let(:default_value) { "'$1,430.00'" }
+          let(:default_value_sql) { "'1430.00'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is in Japanese yen' do
+          let(:default_value) { "'143,000 yen'" }
+          let(:default_value_sql) { "'143000'" }
+          it_behaves_like *examples
+        end
+      end
+      context 'with unsupported column def' do
+        before do
+          column[:column] = "value_unsupported"
+          column[:type] = "_unsupported"
+        end
+        let(:type_sql) { %Q|"value_unsupported" varchar(max)| }
+        let(:not_null_default_sql) { " DEFAULT ''" }
+
+        context 'when default_value is normal' do
+          let(:default_value) { "'happy'" }
+          let(:default_value_sql) { "'happy'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default value has a PostgreSQL-style type cast' do
+          let(:default_value) { "'ok'::mood" }
+          let(:default_value_sql) { "'ok'" }
+          it_behaves_like *examples
+        end
+      end
     end
 
     context 'for create table' do
@@ -684,6 +856,16 @@ EOS
         let(:value) { '2012-1-23 10:40:5.243 +0230' }
         it { is_expected.to eq('2012-01-23 08:10:05.243000') }
       end
+
+      context 'with colon separated timezone value' do
+        let(:value) { '2012-01-23 19:20:15 +01:00' }
+        it { is_expected.to eq('2012-01-23 18:20:15.000000') }
+      end
+
+      context 'without space before imezone value' do
+        let(:value) { '2012-01-23 19:20:15+01:00' }
+        it { is_expected.to eq('2012-01-23 18:20:15.000000') }
+      end
     end
 
     context 'with unix epoch time' do
@@ -779,6 +961,21 @@ EOS
           it { is_expected.to eq('0000-12-31 20:58:35.876543') } # floating-point arithmetic(only negative value)
         end
       end
+
+      context 'when value has timezone' do
+        context 'with no-colon timezone value' do
+          let(:value) { '13:01:24 +0800' }
+          it { is_expected.to eq('0001-01-01 05:01:24.000000') }
+        end
+        context 'with colon separated timezone value' do
+          let(:value) { '13:01:24 -08:00' }
+          it { is_expected.to eq('0001-01-01 21:01:24.000000') }
+        end
+        context 'with a timezone whose corresponding UTC goes negative' do
+          let(:value) { '03:01:24 +0800' }
+          it { is_expected.to eq('0000-12-31 19:01:24.000000') }
+        end
+      end
     end
 
     context 'with nil value' do

data/lib/flydata.rb

@@ -21,6 +21,7 @@ require 'flydata/heroku'
 module Flydata
   FLYDATA_DEBUG = !!(ENV['FLYDATA_DEBUG'])
   FLYDATA_HOME = ENV['FLYDATA_HOME'] || "#{ENV['HOME']}/.flydata"
+  FLYDATA_TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
   FLYDATA_GEM_HOME = File.expand_path(File.join(File.dirname(__FILE__), '..'))
   FLYDATA_GEM_BIN = File.join(FLYDATA_GEM_HOME, 'bin')
   FLYDATA_TMPL_DIR = File.join(FLYDATA_GEM_HOME, 'tmpl')

data/lib/flydata/command/sender.rb

@@ -66,7 +66,7 @@ module Flydata
         start_fluentd.call unless fluentd_started
         if options[:show_final_message] && !options[:quiet]
           data_port = flydata.data_port.get
-          log_info_stdout("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")
+          log_info_stdout("Go to your Dashboard! #{flydata.flydata_api_host}/dashboard")
           log_info_stdout <<EOF
 Please Note: Records and Total Size are updated every 10-20 minutes.
 EOF
@@ -83,7 +83,7 @@ EOF
           log_info_stdout('Done.') unless options[:quiet]
           return true
         end
-        raise 'Something has gone wrong..'
+        raise 'Failed to stop sender process.'
       end
       def flush_client_buffer(options = {})
         force_flush = options.has_key?(:force) ? options[:force] : true
@@ -103,7 +103,7 @@ EOF
           Kernel.sleep 5
         end
 
-        raise 'Something is wrong! Unable to flush client buffer'
+        raise 'Unable to flush client buffer'
       end
       def restart(options = {})
         if process_exist?
@@ -111,7 +111,7 @@ EOF
           begin
             stop(options)
           rescue => e
-            log_warn_stderr "Something has gone wrong. Force stopping the FlyData process. error:#{e.to_s}"
+            log_warn_stderr "Force stopping the FlyData process. error:#{e.to_s}"
             kill_all(options)
           end
         else
@@ -136,7 +136,7 @@ EOF
           log_info_stdout("Done.") unless options[:quiet]
           return true
         else
-          raise 'Something has gone wrong...'
+          raise 'Could not terminate FlyData process.'
         end
       end
       # call the method only when no legit process is running.
@@ -217,7 +217,7 @@ EOF
           log_info_stdout("Waiting logs uploading... (#{i}/#{retry_count})") unless options[:quiet]
           Kernel.sleep 30
         end
-        raise 'Cannot confirm that your logs exist on the FlyData server. Something has gone wrong..'
+        raise 'Cannot confirm that your logs exist on the FlyData server.'
       end
 
       def server_ready?
@@ -238,8 +238,11 @@ EOF
       def client_buffer_empty?(options = {})
         client_buffer = File.join(FLYDATA_HOME, 'buffer')
         log_info_stdout("Checking the client buffer #{client_buffer}") unless options[:quiet]
-        Dir.glob("#{client_buffer}/*").empty?
+        buffer_files = Dir.glob("#{client_buffer}/*")
+        #there can be empty buffer files
+        buffer_files.empty? || buffer_files.all? {|f| File.stat(f).size == 0 }
       end
+
       def pid_file
         "#{FLYDATA_HOME}/flydata.pid"
       end

data/lib/flydata/command/sync.rb

@@ -241,6 +241,7 @@ EOS
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
           on 's', 'skip-primary-key-check', 'Skip primary key check when generating DDL'
           on 'all-tables', 'Generate all table schema'
+          on 'drop-append-only', 'Include queries to drop append-only tables'
           #no 'force-run' option. because stdout is often redirected to a file.
         end
       end
@@ -1169,7 +1170,9 @@ EOM
         create_flydata_ctl_table = true
         append_only = tables & @append_only_tables
         flydata_tabledefs.each do |flydata_tabledef|
-          puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?, skip_drop_table: append_only.include?(flydata_tabledef[:table_name]))
+          skip_drop_table = opts.drop_append_only? ? false : append_only.include?(flydata_tabledef[:table_name])
+          puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef,
+            flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?, skip_drop_table: skip_drop_table)
           create_flydata_ctl_table = false
         end
         table_validity_hash = {}

data/lib/flydata/fluent-plugins/flydata_plugin_ext/base.rb

@@ -2,4 +2,5 @@
 lib = File.expand_path(File.join(File.dirname(__FILE__), '../../../'))
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
+require_relative 'fluent_log_ext'
 require 'flydata'

data/lib/flydata/fluent-plugins/flydata_plugin_ext/fluent_log_ext.rb

@@ -0,0 +1,73 @@
+module Fluent
+
+  # 1. patch to avoid writing the configuration dump
+  #   until suppress config dump option will be supported.
+  #   https://github.com/fluent/fluentd/pull/186
+  # 2. patch to support custom depth_offset
+  class Log
+    # This is used for checking if patched
+    # flydata-core/logger.rb uses this
+    attr_accessor :flydata_patched
+
+    def trace(*args, &block)
+      return if @level > LEVEL_TRACE
+      args << block.call if block
+      depth_offset = extract_depth_offset(args)
+      time, msg = event(:trace, args)
+      puts [@color_trace, caller_line(time, depth_offset, LEVEL_TRACE), msg, @color_reset].join
+    rescue
+      # logger should not raise an exception. This rescue prevents unexpected behaviour.
+    end
+
+    def debug(*args, &block)
+      return if @level > LEVEL_DEBUG
+      args << block.call if block
+      depth_offset = extract_depth_offset(args)
+      time, msg = event(:debug, args)
+      puts [@color_debug, caller_line(time, depth_offset, LEVEL_DEBUG), msg, @color_reset].join
+    rescue
+    end
+
+    def warn(*args, &block)
+      return if @level > LEVEL_WARN
+      args << block.call if block
+      depth_offset = extract_depth_offset(args)
+      time, msg = event(:warn, args)
+      if msg and m = /^parameter '(?<param>[^']+)' in/.match(msg.strip)
+        msg = "parameter '#{m[:param]}' is not used."
+      end
+      puts [@color_warn, caller_line(time, depth_offset, LEVEL_WARN), msg, @color_reset].join
+    rescue
+    end
+
+    def info(*args, &block)
+      return if @level > LEVEL_INFO
+      args << block.call if block
+      depth_offset = extract_depth_offset(args)
+      time, msg = event(:info, args)
+      puts [@color_info, caller_line(time, depth_offset, LEVEL_INFO), msg, @color_reset].join
+    rescue
+    end
+
+    def error(*args, &block)
+      return if @level > LEVEL_ERROR
+      args << block.call if block
+      depth_offset = extract_depth_offset(args)
+      time, msg = event(:error, args)
+      puts [@color_error, caller_line(time, depth_offset, LEVEL_ERROR), msg, @color_reset].join
+    rescue
+    end
+
+    private
+
+    def extract_depth_offset(args)
+      opt = args.last
+      if opt.kind_of?(Hash) and opt[:depth_offset].kind_of?(Integer)
+        return opt.delete(:depth_offset)
+      end
+      @depth_offset
+    end
+  end
+
+end
+