flydata 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8d414dcb17fd8fac585af5e47bc79b26809a276c
-  data.tar.gz: e2ee59e6864c7c75e4a36bb439d4b172e307ba40
+  metadata.gz: 3b07bc6d96c6f880657b1c3bd04cdca32b6a7733
+  data.tar.gz: c229361c0f7a9a901c47d843575472ded9160580
 SHA512:
-  metadata.gz: a205d6dfb530938d493d39e264b2efe6f4c319cf2c94951e7a19a9be99e3f13ea0db0bbe9453643072e46adc4be35385fc1e54ec08510c2c43c8b092e03e8fc6
-  data.tar.gz: 4ea437373319cfaab4dd2a6a0ae28401df803767824c425e82b795dbe12e121a8cdab89d6f17a9cb1d00117b6053188b5b58df20ef497c828510dc8f8d63ad10
+  metadata.gz: 055ebeaa6935d909a447d38ecaabe75a99886fe4640bde07b58724274967a487d122da6cf6ae2948d0a1fec742df011cda5bdb3ec1ce46250f639826b0aa78c9
+  data.tar.gz: 86341f2728f2d142e9e4b44b9d10cabc583c3b3507b92dde437aab89d49a1fa6e303d0344a43a2cc2e82fb8bee4716975eb8892f55f06e888858f43efae1444e

data/Gemfile

@@ -13,16 +13,14 @@ gem "pg", '~> 0.18.4'
 gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
-gem "io-console", '~> 0.4.2', '>= 0.4.2'
+gem "io-console", '~> 0.4.6', '>= 0.4.6'
 gem "kodama", '~> 0.1', '>= 0.1.12'
 gem "serverengine", '~> 1.5'
 
 group :development do
   gem "jeweler", '~> 1.8', '>= 1.8.8'
   gem "rspec", '~> 3.0'
-  #gem "autotest"
-  #gem "autotest-standalone"
-  #gem "autotest-notification"
+  gem 'test-unit'
   gem 'timecop', '~> 0.7', '>= 0.7.1'
   gem "sqlite3", '~> 1.3', '>= 1.3.9'
   gem 'ruby-prof', '~> 0.15', '>= 0.15.1'
@@ -30,6 +28,7 @@ group :development do
   gem 'activerecord', '~> 4.0', '>= 4.0.0'
   gem 'protected_attributes', '~> 1.0', '>= 1.0.8'
   gem 'pry'
+  gem 'rake'
   #gem 'ruby-oci8'
   gem 'rake-compiler', '~> 0.9', '>= 0.9.5'
 end

data/Gemfile.lock

@@ -49,7 +49,7 @@ GEM
     highline (1.6.21)
     http_parser.rb (0.6.0)
     i18n (0.6.11)
-    io-console (0.4.2)
+    io-console (0.4.6)
     jeweler (1.8.8)
       builder
       bundler (~> 1.0)
@@ -81,6 +81,7 @@ GEM
       rack (~> 1.2)
     pg (0.18.4)
     polyglot (0.3.5)
+    power_assert (1.0.1)
     protected_attributes (1.0.8)
       activemodel (>= 4.0.1, < 5.0)
     pry (0.10.1)
@@ -117,6 +118,8 @@ GEM
     sqlite3 (1.3.9)
     sys-filesystem (1.1.3)
       ffi
+    test-unit (3.2.3)
+      power_assert
     thread_safe (0.3.4)
     timecop (0.7.1)
     treetop (1.5.3)
@@ -135,7 +138,7 @@ DEPENDENCIES
   fluentd (= 0.10.46)
   highline (~> 1.6, >= 1.6.19)
   i18n (~> 0.6, >= 0.6.5)
-  io-console (~> 0.4.2, >= 0.4.2)
+  io-console (~> 0.4.6, >= 0.4.6)
   jeweler (~> 1.8, >= 1.8.8)
   json (~> 1.8, >= 1.8.0)
   kodama (~> 0.1, >= 0.1.12)
@@ -143,6 +146,7 @@ DEPENDENCIES
   pg (~> 0.18.4)
   protected_attributes (~> 1.0, >= 1.0.8)
   pry
+  rake
   rake-compiler (~> 0.9, >= 0.9.5)
   rest-client (~> 1.6, >= 1.6.7)
   rspec (~> 3.0)
@@ -151,8 +155,9 @@ DEPENDENCIES
   slop (~> 3.4, >= 3.4.6)
   sqlite3 (~> 1.3, >= 1.3.9)
   sys-filesystem (~> 1.1, >= 1.1.3)
+  test-unit
   timecop (~> 0.7, >= 0.7.1)
   treetop (~> 1.5, >= 1.5.3)
 
 BUNDLED WITH
-   1.11.2
+   1.13.7

data/VERSION

@@ -1 +1 @@
-0.8.2
+0.8.3

data/circle.yml

@@ -1,6 +1,6 @@
 machine:
   ruby:
-    version: 2.0.0-p481
+    version: 2.2.6
 
 dependencies:
   pre:

data/flydata-core/Gemfile

@@ -10,6 +10,7 @@ group :development, :test do
    gem 'rubocop'
    gem 'mysql2'
    gem 'pg'
+   gem 'rake'
    gem 'rspec', '~> 3.1'
    gem 'simplecov', :require => false
 end

data/flydata-core/Gemfile.lock

@@ -20,6 +20,7 @@ GEM
     pg (0.18.4)
     powerpack (0.0.9)
     rainbow (2.0.0)
+    rake (10.4.2)
     rspec (3.1.0)
       rspec-core (~> 3.1.0)
       rspec-expectations (~> 3.1.0)
@@ -56,6 +57,7 @@ DEPENDENCIES
   aws-sdk (= 1.18.0)
   mysql2
   pg
+  rake
   rspec (~> 3.1)
   rubocop
   ruby-prof
@@ -63,4 +65,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   1.10.6
+   1.13.7

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -38,6 +38,7 @@ class MysqlTableDef < Base
     'bool' => {type: 'int1', def_width: '1'},
     'boolean' => {type: 'int1', def_width: '1'},
     'char' => {type: 'varchar', def_width: '1', override: PROC_override_varchar},
+    'char varying' => {type: 'varchar', override: PROC_override_varchar},
     'date' => {type: 'date'},
     'datetime' => {type: 'datetime'},
     'dec' => {type: 'numeric', def_width: '10,0'},
@@ -47,15 +48,26 @@ class MysqlTableDef < Base
     'enum' => {type: 'enum'},
     'fixed' => {type: 'numeric', def_width: '10,0'},
     'float' => {type: 'float4'},
+    'float4' => {type: 'float4'},
+    'float8' => {type: 'float8'},
     'int' => {type: 'int4', def_width: '11'},
+    'int1' => {type: 'int1', def_width: '1'},
+    'int2' => {type: 'int2', def_width: '6'},
+    'int3' => {type: 'int3', def_width: '9'},
+    'int8' => {type: 'int8', def_width: '20'},
     'integer' => {type: 'int4', def_width: '11'},
     'json' => {type: 'json'},
+    'long' => {type: 'text'},
+    'long varbinary' => {type: 'varbinary(16777215)'},
+    'long varchar' => {type: 'text'},
     'longblob' => {type: 'varbinary(4294967295)'},
     'longtext' => {type: 'text'},
     'mediumblob' => {type: 'varbinary(16777215)'},
     'mediumint' => {type: 'int3', def_width: '9'},
     'mediumtext' => {type: 'text'},
+    'middleint' => {type: 'int3', def_width: '9'},
     'numeric' => {type: 'numeric', def_width: '10,0'},
+    'real' => {type: 'float8'},
     'set' => {type: 'set'},
     'smallint' => {type: 'int2', def_width: '6'},
     'text' => {type: 'text'},

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -14,7 +14,7 @@ class RedshiftTableDef
     'boolean' => {type: 'boolean', default_value: false},
     'varbit' => {type: 'bigint', default_value: '0'},
     'bytea' => {type: 'varchar(max)', default_value: ''},
-    'char' => {type: 'char', use_params: true, default_value: ''},
+    'char' => {type: 'char', use_params: true, min_size: 1, default_value: ''},
     'date' => {type: 'date', default_value: '0000-01-01'},
     'datetime' => {type: 'timestamp', default_value: '0000-01-01'},
     'datetimetz' => {type: 'timestamp', default_value: '0000-01-01'},
@@ -42,7 +42,7 @@ class RedshiftTableDef
     'time' => {type: 'timestamp', default_value: '0000-01-01'},
     'timetz' => {type: 'timestamp', default_value: '0000-01-01'},
     'varbinary' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
-    'varchar' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
+    'varchar' => {type: 'varchar', use_params: true, min_size: 1, max_size: 65535, default_value: ''},
     'year' => {type: 'date', default_value: '0001-01-01'},
     '_unsupported' => {type: 'varchar(max)', default_value: ''},
   }
@@ -230,6 +230,7 @@ EOS
     rs_type = if type_info[:use_params]
                 if params
                   params = check_and_replace_max(params, Array(type_info[:max_size]), type) if type_info[:max_size]
+                  params = check_and_replace_min(params, type_info[:min_size], type) if type_info[:min_size]
                 else
                   #source data type has no parameter. use default parameters.
                   raise "No default pramameters for type:`#{column[:type]}`" unless type_info[:default_params]
@@ -434,6 +435,7 @@ EOS
 
   def self.check_and_replace_max(params, max_size_a, flydata_type)
       final_params = []
+
       case flydata_type
       when 'numeric', 'numeric unsigned'
         p = params.split(",")
@@ -459,6 +461,11 @@ EOS
       final_params.join(",")
   end
 
+  def self.check_and_replace_min(params, min_size, flydata_type)
+    return params unless params.to_s.match(/\d+/)
+    params.to_i < min_size ? min_size.to_s : params
+  end
+
   APACHE_TIMESTAMP_REGEXP = Regexp.new('^(?<apache_time_format>\[[0-3]\d\/\D{3}\/[1-2]\d{3}:[0-2]\d:[0-5]\d:[0-5]\d ?[\+\-]\d{2}:?\d{2}\])$')
   TIME_REGEXP = Regexp.new('^(?<sign>-)?(?<hour>\d{2,3}):(?<minute>[0-5][0-9]):(?<second>[0-5][0-9](\.\d+)?)\s*(?<tz>[+-][0-2][0-9](:?\d{2})?)?$')
 

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -166,6 +166,22 @@ EOT
         expect(subject).to eq "BEGIN;\n#{create_table_queries}\n#{flydata_ctl_update}\nCOMMIT;\n#{drop_backup_table_subquery}\n"
       end
     end
+
+    context 'with size 0 varchar' do
+      let(:value_column_hash) { { column: "value", type: "varchar(0)" } }
+
+      it 'should return ddl by replacing (0) with (1)' do
+        expect(subject).to eq "BEGIN;\n#{create_table_queries}\n#{flydata_ctl_update}\nCOMMIT;\n#{drop_backup_table_subquery}\n".gsub("(max)", "(1)").gsub("'text'", "'varchar(0)'")
+      end
+    end
+
+    context 'with size > 0 varchar' do
+      let(:value_column_hash) { { column: "value", type: "varchar(24)" } }
+
+      it 'should return ddl without replacement' do
+        expect(subject).to eq "BEGIN;\n#{create_table_queries}\n#{flydata_ctl_update}\nCOMMIT;\n#{drop_backup_table_subquery}\n".gsub("(max)", "(24)").gsub("'text'", "'varchar(24)'")
+      end
+    end
   end
 
   describe '.column_def_sql' do

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.8.2"
+  s.version = "0.8.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2017-02-11"
+  s.date = "2017-02-14"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]
@@ -447,11 +447,12 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
       s.add_runtime_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_runtime_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
-      s.add_runtime_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
+      s.add_runtime_dependency(%q<io-console>, [">= 0.4.6", "~> 0.4.6"])
       s.add_runtime_dependency(%q<kodama>, [">= 0.1.12", "~> 0.1"])
       s.add_runtime_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_development_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_development_dependency(%q<rspec>, ["~> 3.0"])
+      s.add_development_dependency(%q<test-unit>, [">= 0"])
       s.add_development_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
       s.add_development_dependency(%q<sqlite3>, [">= 1.3.9", "~> 1.3"])
       s.add_development_dependency(%q<ruby-prof>, [">= 0.15.1", "~> 0.15"])
@@ -459,6 +460,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<activerecord>, [">= 4.0.0", "~> 4.0"])
       s.add_development_dependency(%q<protected_attributes>, [">= 1.0.8", "~> 1.0"])
       s.add_development_dependency(%q<pry>, [">= 0"])
+      s.add_development_dependency(%q<rake>, [">= 0"])
       s.add_development_dependency(%q<rake-compiler>, [">= 0.9.5", "~> 0.9"])
     else
       s.add_dependency(%q<rest-client>, [">= 1.6.7", "~> 1.6"])
@@ -473,11 +475,12 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
       s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
-      s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
+      s.add_dependency(%q<io-console>, [">= 0.4.6", "~> 0.4.6"])
       s.add_dependency(%q<kodama>, [">= 0.1.12", "~> 0.1"])
       s.add_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_dependency(%q<rspec>, ["~> 3.0"])
+      s.add_dependency(%q<test-unit>, [">= 0"])
       s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
       s.add_dependency(%q<sqlite3>, [">= 1.3.9", "~> 1.3"])
       s.add_dependency(%q<ruby-prof>, [">= 0.15.1", "~> 0.15"])
@@ -485,6 +488,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<activerecord>, [">= 4.0.0", "~> 4.0"])
       s.add_dependency(%q<protected_attributes>, [">= 1.0.8", "~> 1.0"])
       s.add_dependency(%q<pry>, [">= 0"])
+      s.add_dependency(%q<rake>, [">= 0"])
       s.add_dependency(%q<rake-compiler>, [">= 0.9.5", "~> 0.9"])
     end
   else
@@ -500,11 +504,12 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
     s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
     s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
-    s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
+    s.add_dependency(%q<io-console>, [">= 0.4.6", "~> 0.4.6"])
     s.add_dependency(%q<kodama>, [">= 0.1.12", "~> 0.1"])
     s.add_dependency(%q<serverengine>, ["~> 1.5"])
     s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
     s.add_dependency(%q<rspec>, ["~> 3.0"])
+    s.add_dependency(%q<test-unit>, [">= 0"])
     s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
     s.add_dependency(%q<sqlite3>, [">= 1.3.9", "~> 1.3"])
     s.add_dependency(%q<ruby-prof>, [">= 0.15.1", "~> 0.15"])
@@ -512,6 +517,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<activerecord>, [">= 4.0.0", "~> 4.0"])
     s.add_dependency(%q<protected_attributes>, [">= 1.0.8", "~> 1.0"])
     s.add_dependency(%q<pry>, [">= 0"])
+    s.add_dependency(%q<rake>, [">= 0"])
     s.add_dependency(%q<rake-compiler>, [">= 0.9.5", "~> 0.9"])
   end
 end

data/spec/flydata/command/sender_spec.rb

@@ -23,7 +23,7 @@ module Flydata
           allow(data_port).to receive(:get).and_return("Wibble")
 
           allow_any_instance_of(Flydata::AgentCompatibilityCheck).to receive(:check).and_return(true)
-          Flydata::Command::Sync.any_instance.should_receive(:try_initial_sync).and_return("Wobble")
+          expect_any_instance_of(Flydata::Command::Sync).to receive(:try_initial_sync).and_return("Wobble")
         end
 
         context "as daemon" do

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -592,7 +592,7 @@ EOT
           # Need to make sure no event is sent... how do I do that
           let(:sync_fm) { double('sync_fm') }
           before do
-            Flydata::SyncFileManager.any_instance.should_receive(:get_new_table_list).with(TEST_TABLES.split(","), "pos").and_return([TEST_TABLE])
+            expect_any_instance_of(Flydata::SyncFileManager).to receive(:get_new_table_list).with(TEST_TABLES.split(","), "pos").and_return([TEST_TABLE])
 
             Test.configure_plugin(plugin, TEST_CONFIG)
             plugin.event_listener(rotate_event)

data/spec/spec_helper.rb

@@ -61,12 +61,12 @@ RSpec::Matchers.define :terminate do |code|
     @status_code = status_code
   end
 
-  failure_message_for_should do |block|
+  failure_message do |block|
     "expected block to call exit(#{status_code}) but exit" +
       (actual.nil? ? " not called" : "(#{actual}) was called")
   end
 
-  failure_message_for_should_not do |block|
+  failure_message_when_negated do |block|
     "expected block not to call exit(#{status_code})"
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-11 00:00:00.000000000 Z
+date: 2017-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -242,20 +242,20 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.6
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.6
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.6
 - !ruby/object:Gem::Dependency
   name: kodama
   requirement: !ruby/object:Gem::Requirement
@@ -324,6 +324,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -458,6 +472,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement