flydata 0.3.9 → 0.3.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: abffe0873e3fe1143a5053d9f02b5c61eb1e90a3
-  data.tar.gz: c7f54222297282bcc2c13ffa1ca18ddc45bb10ef
+  metadata.gz: 478d29c0767a534808dea15674528d5b5442bbed
+  data.tar.gz: c6536fb66fedffbdcc52069cb28f32f74fe3465d
 SHA512:
-  metadata.gz: 3b81782a6f4ac5bf2c28485b56ea1912e6f8490d375cb6ffd5f70e070776691fe4d20f74cfb444672192a0440a651b73835765054fc9da617e36182e4c021655
-  data.tar.gz: 7ef6babb2f76e4c1e75ff700a377c1b8a9504f022fe1bbe0fe09dc30f54e629277dafe2d55cfd905967edeac55d70d0019181fc1a90000c77172f9f7325961b8
+  metadata.gz: 6b97f3acc0e763fcff4f2d4a18dcc4d9625d736db229b00554aee36da0ff8e68a189324a46abb3d985dfd1e2d4964ec6963a9299d475170e759485cb1e07faec
+  data.tar.gz: fdee03e1c1bf32699f8c5157aa51f48ecf6f1c023d2160a63b996ca7ac0599fa952de02b55045715b4a66283083b204860201a31a157b9cbf92d2413aebc77d5

data/VERSION

@@ -1 +1 @@
-0.3.9
+0.3.10

data/flydata-core/lib/flydata-core/errors.rb

@@ -51,6 +51,7 @@ end
 # RetryableError
 
 class RetryableError < StandardError
+  NO_RETRY_LIMIT = :no_retry_limit
   def initialize(original_exception, retry_alert_limit = nil)
     @original_exception = original_exception
     @retry_alert_limit = retry_alert_limit
@@ -59,10 +60,20 @@ class RetryableError < StandardError
   attr_reader :retry_alert_limit
 end
 
-# DataDeliveryError
+# UserMaintenanceError
+
+class UserMaintenanceError < StandardError
+end
+
+# An error that indicates that sync record was received for a table whose sync is broken
+# These records cannot be retried, should be ignored
+class BrokenTableSyncError < StandardError
+end
+
+# FlyDataError
 # data_node_id, data_entry_id, chunk_identifier, table_name, err_code, err_reason, err_level
 
-class DataDeliveryError < StandardError
+class FlydataError < StandardError
   def initialize(error_reason, error_content = {})
     error_content = if error_reason.kind_of?(String)
                       { err_reason: error_reason}.merge(error_content)
@@ -125,6 +136,14 @@ class DataDeliveryError < StandardError
   end
 end
 
+# DataDeliveryError
+#  An error which results in data clog or loss of information.
+#  This type of error gets reported to customer as data delivery error.
+# data_node_id, data_entry_id, chunk_identifier, table_name, err_code, err_reason, err_level
+
+class DataDeliveryError < FlydataError
+end
+
 # + RecordDeliveryError
 # data_node_id, data_entry_id, chunk_identifier, table_name, err_code, err_reason, err_level, record_no, raw_record
 
@@ -174,7 +193,8 @@ class TableRevisionError < RecordDeliveryError
 end
 
 # Table's revision is lower than the record's revision.  Should catch up soon.
-class StaleTableRevisionError < TableRevisionError
+class StaleTableRevisionError < FlydataError
+  def err_code; ErrorCode::TABLE_REVISION_ERROR; end
 end
 
 # Unsupported ALTER TABLE which breaks the tables sync consistency.

data/flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb

@@ -0,0 +1,8 @@
+module Fluent
+  module MultiBufferedOutputConstant
+    # '-' was chosen because it's valid in URL but invalid in a SQL table name.
+    # Thus, it can be passed as an URL parameter yet it never conflicts with
+    # a table name.
+    TABLE_TOTAL = "-total"
+  end
+end

data/flydata-core/lib/flydata-core/logger.rb

@@ -108,33 +108,63 @@ module FlydataCore
     def log_common(level, message, log_params, options)
       msg = build_log_message(level, message, log_params, options)
       before_logging(level, message, msg, log_params, options)
-      get_logger(options).send(level, msg)
+      logger = get_logger(options)
+      set_mutex(logger)
+      logger.mutex.synchronize do
+        original_depth_offset = nil
+        if logger.instance_variable_defined? :@depth_offset
+          original_depth_offset = logger.instance_variable_get(:@depth_offset)
+          depth_offset = 3  # log_common + synchronize + synchronized block 
+          depth_offset += options[:depth_offset] if options.has_key?(:depth_offset)
+          logger.instance_variable_set(:@depth_offset, original_depth_offset + depth_offset)
+        end
+        logger.send(level, msg)
+        if original_depth_offset
+          logger.instance_variable_set(:@depth_offset, original_depth_offset)
+        end
+      end
     end
 
     def log_debug(message, log_params = {}, options = {})
-      log_common(:debug, message, log_params, options)
+      opt = { depth_offset: 1}.merge(options)
+      log_common(:debug, message, log_params, opt)
     end
 
     def log_info(message, log_params = {}, options = {})
-      log_common(:info, message, log_params, options)
+      opt = { depth_offset: 1}.merge(options)
+      log_common(:info, message, log_params, opt)
     end
 
     def log_warn(message, log_params = {}, options = {})
-      log_common(:warn, message, log_params, options)
+      opt = { depth_offset: 1}.merge(options)
+      log_common(:warn, message, log_params, opt)
     end
 
     def log_error(message, log_params = {}, options = {})
-      log_common(:error, message, log_params, options)
+      opt = { depth_offset: 1}.merge(options)
+      log_common(:error, message, log_params, opt)
     end
 
     def log_error_with_backtrace(message, log_params = {}, options = {})
-      log_common(:error, message, log_params, options.merge(backtrace: true))
+      opt = { depth_offset: 1}.merge(options)
+      log_common(:error, message, log_params, opt.merge(backtrace: true))
     end
 
     def get_logger(options = {})
       options[:logger] || logger || log_context_logger || $log
     end
 
+    # set mutex if logger doesn't have one
+    def set_mutex(logger)
+      return if logger.respond_to?(:mutex)
+
+      mtx = Mutex.new
+      logger.instance_variable_set(:@mutex, mtx)
+      class << logger
+        attr_reader :mutex
+      end
+    end
+
     def build_log_message(level, raw_message, log_params = {}, options = {})
       message = raw_message.dup
       # check keys
@@ -209,7 +239,9 @@ module FlydataCore
         exception = exception.original_exception unless exception.original_exception.nil?
       end
 
-      unless is_retryable_error && retry_count < retry_alert_limit
+      unless is_retryable_error &&
+              (retry_alert_limit == FlydataCore::RetryableError::NO_RETRY_LIMIT ||
+               retry_count < retry_alert_limit)
         if exception.kind_of?(FlydataCore::DataDeliveryError)
           # Do not log a DataDeliveryError as ERROR.  It's a user error which
           # should not cause a system alert

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -62,6 +62,7 @@ class MysqlTableDef
     'tinytext' => {type: 'text'},
     'varbinary' => {type: 'varbinary', override: PROC_override_varbinary},
     'varchar' => {type: 'varchar', override: PROC_override_varchar},
+    'year' => {type: 'year'},
   }
 
   def self.convert_to_flydata_type(type)

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -33,6 +33,7 @@ class RedshiftTableDef
     'time' => {type: 'timestamp', default_value: '0000-01-01'},
     'varbinary' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
     'varchar' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
+    'year' => {type: 'date', default_value: '0001-01-01'},
   }
   def self.from_flydata_tabledef(flydata_tabledef, options = {})
     options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
@@ -56,7 +57,7 @@ class RedshiftTableDef
 CREATE SCHEMA "%s";
 EOS
   CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
-CREATE TABLE %s(
+CREATE TABLE IF NOT EXISTS %s(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -86,7 +87,7 @@ EOS
   end
 
   CREATE_TABLE_SQL = <<EOS
-DROP TABLE %s;
+DROP TABLE IF EXISTS %s;
 CREATE TABLE %s (
 %s
 )%s;
@@ -141,9 +142,10 @@ EOS
 
   NULL_STR = "NULL"
 
-  def self.replace_default_value(type, default_value)
+  def self.replace_default_value(redshift_type, default_value)
     return NULL_STR if default_value.nil?
-    case type
+
+    case redshift_type
     when 'timestamp'
       if default_value.upcase == "CURRENT_TIMESTAMP"
         'SYSDATE'
@@ -224,7 +226,7 @@ EOS
   def self.check_and_replace_max(params, max_size_a)
       final_params = []
       params.split(",").each_with_index do |param, i|
-        final_params << (/\d+/.match(param) && max_size_a[i] && param.to_i > max_size_a[i].to_i ? 
+        final_params << (/\d+/.match(param) && max_size_a[i] && param.to_i > max_size_a[i].to_i ?
                             max_size_a[i] : param)
       end
       final_params.join(",")
@@ -234,23 +236,24 @@ EOS
   TIME_REGEXP = Regexp.new('^(?<sign>-)?(?<hour>\d{2,3}):(?<minute>[0-5][0-9]):(?<second>[0-5][0-9](\.\d+)?)$')
 
   def self.parse_timestamp(value)
-    return nil if value.to_s.empty?
-    if value.kind_of?(Integer) or /^\d+$/ === value
+    value_str = value.to_s
+    return nil if value_str.empty?
+    if value.kind_of?(Integer) or /^\d+$/ === value_str
       # Unix epoch in UTC
-      t = DateTime.strptime(value.to_s, '%s')
-    elsif APACHE_TIMESTAMP_REGEXP.match(value)
+      t = DateTime.strptime(value_str, '%s')
+    elsif APACHE_TIMESTAMP_REGEXP.match(value_str)
       # apache time format
       t = DateTime.strptime(value, "[%d/%b/%Y:%H:%M:%S %Z]")
-    elsif time_match = TIME_REGEXP.match(value)
+    elsif time_match = TIME_REGEXP.match(value_str)
       t = convert_time_into_timestamp(time_match)
     else
-      t = DateTime.parse(value)
+      t = DateTime.parse(value_str)
     end
     t = t.new_offset(0) # Redshift Plug-in uses UTC
     t.strftime('%Y-%m-%d %H:%M:%S.%6N')
   rescue ArgumentError => ae
     # '0000-00-00 00:00:00' is valid for mysql datetime column
-    if value.start_with?('0000-00-00 00:00:00')
+    if value_str.start_with?('0000-00-00 00:00:00')
       return '0001-01-01 00:00:00.000000'
     else
       raise ae
@@ -268,15 +271,29 @@ EOS
   end
 
   def self.parse_date(value)
-    dt = Date.parse(value)
+    dt = Date.parse(convert_year_into_date(value))
     dt.strftime('%Y-%m-%d')
   rescue ArgumentError => ae
     # '0000-00-00' is valid for mysql date column
     return '0001-01-01' if value == '0000-00-00'
     raise ae
   end
+
+  def self.convert_year_into_date(value)
+    if value == '0' || value == '0000'
+      converted_value = '0001-01-01' # '0001-01-01' is the "base" date
+    else
+      case value
+      when /^\d{4}$/
+        converted_value = "#{value}-01-01"
+      when /^\d{2}$/
+        converted_value = "#{value.to_i >= 70 ? "19" : "20"}#{value}-01-01"
+      else
+        converted_value = value # Return the value as is
+      end
+    end
+  end
 end
 
 end
 end
-

data/flydata-core/spec/table_def/mysql_table_def_spec.rb

@@ -56,7 +56,10 @@ describe MysqlTableDef do
             {:column=>"col_tinyint", :type=>"int1(4)", :default=>nil},
             {:column=>"col_tinytext", :type=>"text"},
             {:column=>"col_varbinary", :type=>"varbinary(512)", :default=>nil},
-            {:column=>"col_varchar", :type=>"varchar(372)", :default=>nil}
+            {:column=>"col_varchar", :type=>"varchar(372)", :default=>nil},
+            {:column=>"col_year", :type=>"year", :default=>nil},
+            {:column=>"col_year_4", :type=>"year(4)", :default=>nil},
+            {:column=>"col_year_2", :type=>"year(2)", :default=>nil},
           ]
         )
       end
@@ -329,6 +332,10 @@ describe MysqlTableDef do
         let(:flydata_type) { 'text' }
         it_behaves_like "converting the mysql data type to the expected flydata type"
       end
+      context "year" do
+        let(:flydata_type) { 'year' }
+        it_behaves_like "converting the mysql data type to the expected flydata type"
+      end
     end
   end
 end

data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb

@@ -27,7 +27,7 @@ module FlydataCore
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -36,7 +36,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "test_table_all";
+DROP TABLE IF EXISTS "test_table_all";
 CREATE TABLE "test_table_all" (
   "id" int8 NOT NULL,
   "col_binary" varchar(202) DEFAULT NULL,
@@ -65,6 +65,9 @@ CREATE TABLE "test_table_all" (
   "col_tinytext" varchar(max),
   "col_varbinary" varchar(512) DEFAULT NULL,
   "col_varchar" varchar(372) DEFAULT NULL,
+  "col_year" date DEFAULT NULL,
+  "col_year_4" date DEFAULT NULL,
+  "col_year_2" date DEFAULT NULL,
   PRIMARY KEY (id)
 ) DISTKEY(id) SORTKEY(id);
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_all';
@@ -95,7 +98,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('test_table_all', 'col_tinyint', 'int1(4)', 24),
 ('test_table_all', 'col_tinytext', 'text', 25),
 ('test_table_all', 'col_varbinary', 'varbinary(512)', 26),
-('test_table_all', 'col_varchar', 'varchar(372)', 27);
+('test_table_all', 'col_varchar', 'varchar(372)', 27),
+('test_table_all', 'col_year', 'year', 28),
+('test_table_all', 'col_year_4', 'year(4)', 29),
+('test_table_all', 'col_year_2', 'year(2)', 30);
 EOT
         end
       end
@@ -105,7 +111,7 @@ EOT
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -114,7 +120,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "bit_test_def_1";
+DROP TABLE IF EXISTS "bit_test_def_1";
 CREATE TABLE "bit_test_def_1" (
   "id" int4 NOT NULL,
   "bit_value" bigint DEFAULT 1,
@@ -135,7 +141,7 @@ EOT
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -144,7 +150,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "product_order";
+DROP TABLE IF EXISTS "product_order";
 CREATE TABLE "product_order" (
   "no" int4 NOT NULL,
   "product_category" int4 NOT NULL,
@@ -167,7 +173,7 @@ EOT
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -176,7 +182,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "test_table_column_comment";
+DROP TABLE IF EXISTS "test_table_column_comment";
 CREATE TABLE "test_table_column_comment" (
   "id" int4 NOT NULL DEFAULT '0',
   "value" varchar(max),
@@ -197,7 +203,7 @@ EOT
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -206,7 +212,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "test_table_enum";
+DROP TABLE IF EXISTS "test_table_enum";
 CREATE TABLE "test_table_enum" (
   "id" int4 NOT NULL,
   "enum_1" varchar encode bytedict DEFAULT NULL,
@@ -229,7 +235,7 @@ EOT
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -238,7 +244,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "test_table_multi_pk";
+DROP TABLE IF EXISTS "test_table_multi_pk";
 CREATE TABLE "test_table_multi_pk" (
   "id1" int4 NOT NULL DEFAULT '0',
   "id2" int4 NOT NULL DEFAULT '0',
@@ -267,7 +273,7 @@ EOT
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -276,7 +282,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "sample1";
+DROP TABLE IF EXISTS "sample1";
 CREATE TABLE "sample1" (
   "id" int4 NOT NULL,
   "title" varchar(768) DEFAULT NULL,
@@ -299,7 +305,7 @@ EOT
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -308,7 +314,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "sample1";
+DROP TABLE IF EXISTS "sample1";
 CREATE TABLE "sample1" (
   "id" int4 NOT NULL,
   "title" varchar(768) DEFAULT NULL,
@@ -331,7 +337,7 @@ EOT
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -340,7 +346,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "invoice_items";
+DROP TABLE IF EXISTS "invoice_items";
 CREATE TABLE "invoice_items" (
   "id" int4 NOT NULL,
   "app_id" int4 NOT NULL,
@@ -391,7 +397,7 @@ EOT
 
         it 'should return ddl' do
           expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -400,7 +406,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "zerofill_table";
+DROP TABLE IF EXISTS "zerofill_table";
 CREATE TABLE "zerofill_table" (
   "id" int4 NOT NULL,
   "value_int" int8 DEFAULT NULL,

data/flydata-core/spec/table_def/mysqldump_test_table_all.dump

@@ -50,6 +50,9 @@ CREATE TABLE `test_table_all` (
   `col_tinytext` tinytext,
   `col_varbinary` varbinary(255) DEFAULT NULL,
   `col_varchar` varchar(124) DEFAULT NULL,
+  `col_year` year DEFAULT NULL,
+  `col_year_4` year(4) DEFAULT NULL,
+  `col_year_2` year(2) DEFAULT NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table includes all kind of format type';
 /*!40101 SET character_set_client = @saved_cs_client */;

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -22,7 +22,7 @@ describe RedshiftTableDef do
     context 'with simple flydatadef' do
       it 'should return ddl' do
         expect(subject).to eq( <<EOT.strip )
-DROP TABLE "test_table";
+DROP TABLE IF EXISTS "test_table";
 CREATE TABLE "test_table" (
   "id" int4 NOT NULL,
   "age" int8,
@@ -43,7 +43,7 @@ EOT
 
       it 'should return ddl including flydata_ctl_columns creation' do
         expect(subject).to eq( <<EOT.strip )
-CREATE TABLE "flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -52,7 +52,7 @@ CREATE TABLE "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "test_table";
+DROP TABLE IF EXISTS "test_table";
 CREATE TABLE "test_table" (
   "id" int4 NOT NULL,
   "age" int8,
@@ -80,7 +80,7 @@ EOT
 
       it 'should add comment creation ddl' do
         expect(subject).to eq( <<EOT.strip )
-DROP TABLE "test_table";
+DROP TABLE IF EXISTS "test_table";
 CREATE TABLE "test_table" (
   "id" int4 NOT NULL,
   "value" varchar(max),
@@ -112,7 +112,7 @@ EOT
       it 'should preappend schema name to table name' do
         expect(subject).to eq( <<EOT.strip )
 CREATE SCHEMA "test_schema";
-CREATE TABLE "test_schema"."flydata_ctl_columns"(
+CREATE TABLE IF NOT EXISTS "test_schema"."flydata_ctl_columns"(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -121,7 +121,7 @@ CREATE TABLE "test_schema"."flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
-DROP TABLE "test_schema"."test_table";
+DROP TABLE IF EXISTS "test_schema"."test_table";
 CREATE TABLE "test_schema"."test_table" (
   "id" int4 NOT NULL,
   "value" varchar(max),
@@ -409,6 +409,50 @@ EOT
           it_behaves_like *examples
         end
       end
+
+      context 'with year column def' do
+        before do
+          column[:column] = "value_year"
+          column[:type] = "year(4)"
+        end
+        let(:type_sql) { %Q|"value_year" date| }
+        let(:not_null_default_sql) { " DEFAULT '0001-01-01'" }
+
+        context 'when default_value is normal' do
+          let(:default_value) { "'2014'" }
+          let(:default_value_sql) { "'2014-01-01'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 69' do
+          let(:default_value) { "'69'" }
+          let(:default_value_sql) { "'2069-01-01'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 70' do
+          let(:default_value) { "'70'" }
+          let(:default_value_sql) { "'1970-01-01'" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 0' do
+          let(:default_value) { "'0'" }
+          let(:default_value_sql) { "'0001-01-01'" }
+          it_behaves_like *examples
+        end
+
+        context 'when the type has no width' do
+          before do
+            column[:type] = "year"
+          end
+          context 'when default_value is normal' do
+            let(:default_value) { "'2014'" }
+            let(:default_value_sql) { "'2014-01-01'" }
+            it_behaves_like *examples
+          end
+        end
+      end
     end
 
     context 'for create table' do
@@ -592,6 +636,63 @@ EOT
       let(:value) { 'abcd' }
       it { expect{subject}.to raise_error(ArgumentError) }
     end
+
+    context 'with boolean value' do
+      let(:value){ false }
+      it { expect{subject}.to raise_error(ArgumentError) }
+    end
+  end
+
+  describe '.convert_year_into_date' do
+    let(:value) { nil }
+    subject { described_class.convert_year_into_date(value) }
+
+    context 'with year values' do
+      context 'with value 0' do
+        let(:value){ '0' }
+        it { is_expected.to eq('0001-01-01') }
+      end
+
+      context 'with value 0000' do
+        let(:value){ '0000' }
+        it { is_expected.to eq('0001-01-01') }
+      end
+
+      context 'with year(4) >= 1970' do
+        let(:value){ '2000' }
+        it { is_expected.to eq('2000-01-01') }
+      end
+
+      context 'with year(4) < 1970' do
+        let(:value){ '1969' }
+        it { is_expected.to eq('1969-01-01') }
+      end
+
+      context 'with year(2) >= 70' do
+        let(:value){ '07' }
+        it { is_expected.to eq('2007-01-01') }
+      end
+
+      context 'with year(2) < 70' do
+        let(:value){ '69' }
+        it { is_expected.to eq('2069-01-01') }
+      end
+    end
+
+    context 'with date values' do
+      shared_examples "returning the input value as is" do
+        it { is_expected.to eq(value) }
+      end
+      context 'with valid date' do
+        let(:value){ '1920-01-01' }
+        it_behaves_like "returning the input value as is"
+      end
+
+      context 'with zero date' do
+        let(:value){ '0000-00-00' }
+        it_behaves_like "returning the input value as is"
+      end
+    end
   end
 end
 

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.9 ruby lib
+# stub: flydata 0.3.10 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.9"
+  s.version = "0.3.10"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-03-21"
+  s.date = "2015-03-31"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -44,6 +44,7 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/core_ext/object.rb",
     "flydata-core/lib/flydata-core/core_ext/object/prepend.rb",
     "flydata-core/lib/flydata-core/errors.rb",
+    "flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb",
     "flydata-core/lib/flydata-core/logger.rb",
     "flydata-core/lib/flydata-core/table_def.rb",
     "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
@@ -167,7 +168,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.2.2"
+  s.rubygems_version = "2.4.3"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/sync.rb

@@ -24,7 +24,7 @@ module Flydata
     class Sync < Base
       include Helpers
       INSERT_PROGRESS_INTERVAL = 1000
-      SERVER_DATA_PROCESSING_TIMEOUT = 600 # seconds
+      SERVER_DATA_PROCESSING_TIMEOUT = 3600 # seconds
 
       # for dump.pos file
       STATUS_PARSING = 'PARSING'
@@ -179,7 +179,10 @@ EOS
           sync_fm.table_rev_file_paths(*@input_tables),
           sync_fm.table_ddl_file_paths(*@input_tables)
         ]
-        delete_files << sync_fm.binlog_path if @input_tables.empty? or @full_tables.empty?
+        new_tables_after_reset = @new_tables + @input_tables
+        if @input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
+          delete_files << sync_fm.binlog_path
+        end
         delete_files.flatten.each do |path|
           FileUtils.rm(path) if File.exists?(path)
         end
@@ -737,13 +740,13 @@ Thank you for using FlyData!
       def flush_buffer_and_stop(tables = [])
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer
+        sender.stop(quiet: true)
         if opts.skip_flush?
           log_info_stdout("Skip waiting for server data processing.")
         else
           wait_for_server_data_processing(
             timeout: SERVER_DATA_PROCESSING_TIMEOUT, tables: tables)
         end
-        sender.stop(quiet: true)
       end
 
       # Utility methods

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.3.10
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-21 00:00:00.000000000 Z
+date: 2015-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -467,6 +467,7 @@ files:
 - flydata-core/lib/flydata-core/core_ext/object.rb
 - flydata-core/lib/flydata-core/core_ext/object/prepend.rb
 - flydata-core/lib/flydata-core/errors.rb
+- flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb
 - flydata-core/lib/flydata-core/logger.rb
 - flydata-core/lib/flydata-core/table_def.rb
 - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
@@ -607,7 +608,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent