flydata 0.2.17 → 0.2.18

data/{spec/flydata → flydata-core/spec}/table_def/redshift_table_def_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
-require 'flydata/table_def'
+require 'flydata-core/table_def'
 
-module Flydata
+module FlydataCore
 module TableDef
 
 describe RedshiftTableDef do
@@ -233,7 +233,7 @@ EOT
           it_behaves_like *examples
         end
       end
-      
+
       context 'with medium blob column def' do
         context 'when size exceeds max size' do
           let(:default_value) { "something" }
@@ -246,9 +246,9 @@ EOT
           let(:type_sql) { %Q|"col_medium_blob" varchar(65535)| }
 
           it_behaves_like *examples
-        end    
+        end
       end
-      
+
       context 'with unsigned column def' do
         let(:default_value) { 4 }
         let(:default_value_sql) { "'#{default_value}'" }
@@ -312,8 +312,8 @@ EOT
 
       context 'with decimal column def' do
         context 'when precision exceeds max allowed' do
-          let(:default_value) { 4 }
-          let(:default_value_sql) { "'#{default_value}'" }
+          let(:default_value) { "'4'" }
+          let(:default_value_sql) { default_value }
           let(:not_null_default_sql) { " DEFAULT '0'" }
           before do
             column[:column] = "value"
@@ -324,8 +324,8 @@ EOT
           it_behaves_like *examples
         end
         context 'when scale exceeds max allowed' do #not possible
-          let(:default_value) { 4 }
-          let(:default_value_sql) { "'#{default_value}'" }
+          let(:default_value) { "'4'" }
+          let(:default_value_sql) { default_value }
           let(:not_null_default_sql) { " DEFAULT '0'" }
           before do
             column[:column] = "value"
@@ -337,8 +337,8 @@ EOT
         end
 
         context 'when precision and scale exceeds max allowed (for unsigned)' do
-          let(:default_value) { 4 }
-          let(:default_value_sql) { "'#{default_value}'" }
+          let(:default_value) { "'4'" }
+          let(:default_value_sql) { default_value }
           let(:not_null_default_sql) { " DEFAULT '0'" }
           before do
             column[:column] = "value"
@@ -350,8 +350,8 @@ EOT
         end
 
         context 'when precision and scale are within limit (for unsigned)' do
-          let(:default_value) { 4 }
-          let(:default_value_sql) { "'#{default_value}'" }
+          let(:default_value) { "'4'" }
+          let(:default_value_sql) { default_value }
           let(:not_null_default_sql) { " DEFAULT '0'" }
           before do
             column[:column] = "value"
@@ -362,6 +362,52 @@ EOT
           it_behaves_like *examples
         end
       end
+
+      context 'with bit column def' do
+        before do
+          column[:column] = "value_bit"
+          column[:type] = "bit(1)"
+        end
+        let(:type_sql) { %Q|"value_bit" bigint| }
+        let(:not_null_default_sql) { " DEFAULT '0'" }
+
+        context 'when default_value is small bit' do
+          let(:default_value) { "b'1'" }
+          let(:default_value_sql) { "1" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is large bit' do
+          let(:default_value) { "b'10110'" }
+          let(:default_value_sql) { "22" }
+          it_behaves_like *examples
+        end
+
+        # it's not real case
+        context "when default_value is x'' format" do
+          let(:default_value) { "x'12'" }
+          let(:default_value_sql) { "18" }
+          it_behaves_like *examples
+        end
+
+        context "when default_value is X'' format" do
+          let(:default_value) { "X'2a'" }
+          let(:default_value_sql) { "42" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 0x format' do
+          let(:default_value) { "0x0013" }
+          let(:default_value_sql) { "19" }
+          it_behaves_like *examples
+        end
+
+        context 'when default_value is 0b format' do
+          let(:default_value) { "0b1011" }
+          let(:default_value_sql) { "11" }
+          it_behaves_like *examples
+        end
+      end
     end
 
     context 'for create table' do
@@ -378,7 +424,7 @@ EOT
       context "without not null" do
         let(:expected_query) { %Q|  #{type_sql}#{default_sql}| }
         let(:no_default_sql) { "" }
-    
+
         it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
       end
       context "with not null false" do
@@ -387,16 +433,17 @@ EOT
           column[:not_null] = false
         end
         let(:no_default_sql) { "" }
-    
+
         it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
       end
+
       context "with not null true" do
         let(:expected_query) { %Q|  #{type_sql} NOT NULL#{default_sql}| }
         before do
           column[:not_null] = true
         end
         let(:no_default_sql) { not_null_default_sql }
-    
+
         it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
       end
     end

data/flydata.gemspec

@@ -5,16 +5,15 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.17"
+  s.version = "0.2.18"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2014-11-24"
+  s.date = "2014-11-27"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
   s.files = [
-    ".gitignore",
     ".rspec",
     "Gemfile",
     "Gemfile.lock",
@@ -28,6 +27,38 @@ Gem::Specification.new do |s|
     "bin/fdmysqldump",
     "bin/flydata",
     "bin/serverinfo",
+    "flydata-core/.gitignore",
+    "flydata-core/.rspec",
+    "flydata-core/.travis.yml",
+    "flydata-core/Gemfile",
+    "flydata-core/Gemfile.lock",
+    "flydata-core/lib/flydata-core.rb",
+    "flydata-core/lib/flydata-core/core_ext.rb",
+    "flydata-core/lib/flydata-core/core_ext/module.rb",
+    "flydata-core/lib/flydata-core/core_ext/module/include.rb",
+    "flydata-core/lib/flydata-core/core_ext/object.rb",
+    "flydata-core/lib/flydata-core/core_ext/object/prepend.rb",
+    "flydata-core/lib/flydata-core/errors.rb",
+    "flydata-core/lib/flydata-core/logger.rb",
+    "flydata-core/lib/flydata-core/table_def.rb",
+    "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
+    "flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
+    "flydata-core/lib/flydata-core/thread_context.rb",
+    "flydata-core/spec/spec_helper.rb",
+    "flydata-core/spec/table_def/mysql_table_def_spec.rb",
+    "flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb",
+    "flydata-core/spec/table_def/mysqldump_test_bit_table.dump",
+    "flydata-core/spec/table_def/mysqldump_test_foreign_key.dump",
+    "flydata-core/spec/table_def/mysqldump_test_table_all.dump",
+    "flydata-core/spec/table_def/mysqldump_test_table_column_comment.dump",
+    "flydata-core/spec/table_def/mysqldump_test_table_enum.dump",
+    "flydata-core/spec/table_def/mysqldump_test_table_multi_pk.dump",
+    "flydata-core/spec/table_def/mysqldump_test_table_no_pk.dump",
+    "flydata-core/spec/table_def/mysqldump_test_unique_key.dump",
+    "flydata-core/spec/table_def/mysqldump_test_unique_key2.dump",
+    "flydata-core/spec/table_def/mysqldump_test_unique_key3.dump",
+    "flydata-core/spec/table_def/mysqldump_test_unsigned.dump",
+    "flydata-core/spec/table_def/redshift_table_def_spec.rb",
     "flydata.gemspec",
     "lib/fly_data_model.rb",
     "lib/flydata.rb",
@@ -83,9 +114,6 @@ Gem::Specification.new do |s|
     "lib/flydata/preference/data_entry_preference.rb",
     "lib/flydata/proxy.rb",
     "lib/flydata/sync_file_manager.rb",
-    "lib/flydata/table_def.rb",
-    "lib/flydata/table_def/mysql_table_def.rb",
-    "lib/flydata/table_def/redshift_table_def.rb",
     "lib/flydata/util/encryptor.rb",
     "spec/fluent_plugins_spec_helper.rb",
     "spec/fly_data_model_spec.rb",
@@ -103,18 +131,6 @@ Gem::Specification.new do |s|
     "spec/flydata/parser/mysql/alter_table_parser_spec.rb",
     "spec/flydata/parser/mysql/dump_parser_spec.rb",
     "spec/flydata/sync_file_manager_spec.rb",
-    "spec/flydata/table_def/mysql_table_def_spec.rb",
-    "spec/flydata/table_def/mysqldump_test_foreign_key.dump",
-    "spec/flydata/table_def/mysqldump_test_table_all.dump",
-    "spec/flydata/table_def/mysqldump_test_table_column_comment.dump",
-    "spec/flydata/table_def/mysqldump_test_table_enum.dump",
-    "spec/flydata/table_def/mysqldump_test_table_multi_pk.dump",
-    "spec/flydata/table_def/mysqldump_test_table_no_pk.dump",
-    "spec/flydata/table_def/mysqldump_test_unique_key.dump",
-    "spec/flydata/table_def/mysqldump_test_unique_key2.dump",
-    "spec/flydata/table_def/mysqldump_test_unique_key3.dump",
-    "spec/flydata/table_def/mysqldump_test_unsigned.dump",
-    "spec/flydata/table_def/redshift_table_def_spec.rb",
     "spec/flydata/util/encryptor_spec.rb",
     "spec/flydata_spec.rb",
     "spec/spec_helper.rb",

data/lib/flydata/command/sync.rb

@@ -5,9 +5,9 @@ require 'mysql2'
 require 'flydata/sync_file_manager'
 require 'flydata/agent_errors'
 require 'flydata/compatibility_check'
-require 'flydata/table_def'
 require 'flydata/output/forwarder'
 require 'flydata/parser/mysql/dump_parser'
+require 'flydata-core/table_def'
 #require 'ruby-prof'
 
 module Flydata
@@ -26,6 +26,7 @@ module Flydata
           on 'c', 'skip-cleanup', 'Skip server cleanup'
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
           on 'd', 'dump-file', 'Dump mysqldump into a file. Use this for debugging after making sure the free space.'
+          on 'n', 'no-flydata-start', 'Don\'t start the flydata agent after initial sync.'
         end
       end
 
@@ -150,6 +151,7 @@ module Flydata
         Slop.new do
           on 'c', 'ctl-only', 'Only generate FlyData Control definitions'
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
+          on 's', 'skip-primary-key-check', 'Skip primary key check when generating DDL'
         end
       end
 
@@ -241,14 +243,14 @@ module Flydata
           create_flydata_ctl_table = mp['initial_sync']
           while !stdout.eof?
             begin
-              mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(stdout)
+              mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, skip_primary_key_check: opts.skip_primary_key_check?)
               if mysql_tabledef.nil?
                 # stream had no more create table definition
                 break
               end
               flydata_tabledef = mysql_tabledef.to_flydata_tabledef
-              puts Flydata::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
-            rescue TableDefError=> e
+              puts FlydataCore::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
+            rescue FlydataCore::TableDefError=> e
               error_list << e.err_hash
               next
             end
@@ -515,16 +517,17 @@ Thank you for using FlyData!
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         info = sync_fm.load_dump_pos
         if info[:status] == STATUS_COMPLETE
-          puts "Starting FlyData Agent..."
           if de['mysql_data_entry_preference']['initial_sync']
             sync_fm.save_binlog(info[:binlog_pos])
           end
           sync_fm.move_table_binlog_files(de['mysql_data_entry_preference']['tables'].split(','))
           sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'].split(','))
           sync_fm.backup_dump_dir
-          Flydata::Command::Sender.new.start(quiet: true)
-          puts "  -> Done"
-
+          unless opts.no_flydata_start?
+            puts "Starting FlyData Agent..."
+            Flydata::Command::Sender.new.start(quiet: true)
+            puts "  -> Done"
+          end
           dashboard_url = "#{flydata.flydata_api_host}/dashboard"
           redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
           last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]

data/lib/flydata/parser/mysql/mysql_alter_table.treetop

@@ -10,7 +10,7 @@
 #   - ADD COLUMN (col_name, col_def, ...)
 #   - DROP COLUMN col_name
 
-require 'flydata/table_def/mysql_table_def'
+require 'flydata-core/table_def/mysql_table_def'
 
 grammar MysqlAlterTable
 
@@ -182,6 +182,20 @@ grammar MysqlAlterTable
         }
       end
     }
+    / alter_sym opt_column sp field_ident sp set_sym sp default_sym sp signed_literal {
+      def action
+        { action: :alter_column,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+    / alter_sym opt_column sp field_ident sp drop_sym sp default_sym {
+      def action
+        { action: :alter_column,
+          support_level: :nonbreaking,
+        }
+      end
+    }
     / rename_sym opt_to nsp table_ident {
       def action
         { action: :rename_table }
@@ -202,13 +216,27 @@ grammar MysqlAlterTable
         { action: :default_charset }
       end
     }
-    /force_sym {
+    / force_sym {
       def action
         { action: :force,
           support_level: :nonbreaking,
         }
       end
     }
+    / alter_algorithm_option {
+      def action
+        { action: :algorithm,
+          support_level: :nonbreaking
+        }
+      end
+    }
+    / alter_lock_option {
+      def action
+        { action: :lock,
+          support_level: :nonbreaking
+        }
+      end
+    }
     / [^,]+ {
       def action
         raise "unsupported ALTER TABLE query.  Contact FlyData Support"
@@ -309,7 +337,20 @@ grammar MysqlAlterTable
         end
       end
     }
-    / opt_constraint 'foreign'i key_sym opt_ident nsp '(' key_list ')' sp references
+    / constraint_sym sp foreign_sym sp key_sym opt_ident nsp '(' key_list ')' sp references {
+      def action
+        { action: :add_foreign_key_constraint,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+    / opt_constraint foreign_sym sp key_sym opt_ident nsp '(' key_list ')' sp references {
+      def action
+        { action: :add_foreign_key_constraint,
+          support_level: :nonbreaking,
+        }
+      end
+    }
     / constraint_sym sp check_constraint {
       def action
         { action: :add_check_constraint,
@@ -335,7 +376,7 @@ grammar MysqlAlterTable
   end
 
   rule references
-    'references'i sp table_ident opt_ref_list opt_match_clause opt_on_update_delete
+    references_sym sp table_ident opt_ref_list opt_match_clause opt_on_update_delete
   end
 
   rule table_ident
@@ -354,25 +395,25 @@ grammar MysqlAlterTable
   end
 
   rule opt_match_clause
-    ( sp ( 'match'i sp 'full'i / 'match'i sp 'partial'i / 'match'i sp 'simple'i ) )?
+    ( sp ( match_sym sp full_sym / match_sym sp partial_sym / match_sym sp simple_sym ) )?
   end
 
   rule opt_on_update_delete
     ( sp (
-        'on'i sp 'update'i delete_option sp 'on'i sp 'delete'i sp delete_option
-        / 'on'i sp 'delete'i delete_option sp 'on'i sp 'update'i sp delete_option
-        / 'on'i sp 'update'i sp delete_option
-        / 'on'i sp 'delete'i sp delete_option
+        on_sym sp update_sym sp delete_option sp on_sym sp delete_sym sp delete_option
+        / on_sym sp delete_sym sp delete_option sp on_sym sp update_sym sp delete_option
+        / on_sym sp update_sym sp delete_option
+        / on_sym sp delete_sym sp delete_option
       )
     )?
   end
 
   rule delete_option
-    'restrict'i
-    / 'cascade'i
-    / 'set'i sp 'null'i
-    / 'no'i sp 'action'i
-    / 'set'i sp 'default'i
+    restrict_sym
+    / cascade_sym
+    / set_sym sp null_sym
+    / no_sym sp action_sym
+    / set_sym sp default_sym
   end
 
   rule opt_constraint
@@ -384,7 +425,7 @@ grammar MysqlAlterTable
   end
 
   rule constraint_sym
-    'constraint'i
+    'constraint'i ![A-Za-z0-9_]
   end
 
   rule constraint_key_type
@@ -407,7 +448,7 @@ grammar MysqlAlterTable
   end
 
   rule key_sym
-    'key'i
+    'key'i ![A-Za-z0-9_]
   end
 
   rule keys_sym
@@ -431,7 +472,55 @@ grammar MysqlAlterTable
   end
 
   rule foreign_sym
-    'foreign'i
+    'foreign'i ![A-Za-z0-9_]
+  end
+
+  rule references_sym
+    'references'i ![A-Za-z0-9_]
+  end
+
+  rule on_sym
+    'on'i ![A-Za-z0-9_]
+  end
+
+  rule match_sym
+    'match'i ![A-Za-z0-9_]
+  end
+
+  rule full_sym
+    'full'i ![A-Za-z0-9_]
+  end
+
+  rule partial_sym
+    'partial'i ![A-Za-z0-9_]
+  end
+
+  rule simple_sym
+    'simple'i ![A-Za-z0-9_]
+  end
+
+  rule update_sym
+    'update'i ![A-Za-z0-9_]
+  end
+
+  rule delete_sym
+    'delete'i ![A-Za-z0-9_]
+  end
+
+  rule restrict_sym
+    'restrict'i ![A-Za-z0-9_]
+  end
+
+  rule cascade_sym
+    'cascade'i ![A-Za-z0-9_]
+  end
+
+  rule no_sym
+    'no'i ![A-Za-z0-9_]
+  end
+
+  rule action_sym
+    'action'i ![A-Za-z0-9_]
   end
 
   rule fulltext
@@ -649,7 +738,7 @@ grammar MysqlAlterTable
       def data_type
         meta = (meta_text.text_value.size > 1) ? meta_text.text_value : ''
         type = data_type_name.text_value.downcase + meta
-        type = Flydata::TableDef::MysqlTableDef.convert_to_flydata_type(type)
+        type = FlydataCore::TableDef::MysqlTableDef.convert_to_flydata_type(type)
         type << " unsigned" if !unsigned.terminal?
         ret = { type: type }
         ret[:zefofill] = true if !zerofill.terminal?
@@ -1357,4 +1446,25 @@ grammar MysqlAlterTable
     ( sp column_sym )?
   end
 
+  rule alter_algorithm_option
+    algorithm_sym opt_equal nsp default_sym
+    / algorithm_sym opt_equal nsp ident
+  end
+
+  rule algorithm_sym
+    'algorithm'i ![A-Za-z0-9_]
+  end
+
+  rule alter_sym
+    'alter'i ![A-Za-z0-9_]
+  end
+
+  rule lock_sym
+    'lock'i ![A-Za-z0-9_]
+  end
+
+  rule alter_lock_option
+    lock_sym opt_equal nsp default_sym
+    / lock_sym opt_equal nsp ident
+  end
 end

data/lib/flydata/parser_provider.rb

@@ -1,6 +1,6 @@
 require 'thread'
 require 'treetop'
-require 'flydata/table_def'
+require 'flydata-core/table_def'
 
 module Mysql
   class ParserProvider

data/lib/flydata.rb

@@ -1,4 +1,14 @@
-# common
+# load_path settings
+def append_load_path_if_not_exist(new_lib_path)
+  absolute_path = File.realpath(new_lib_path)
+  $LOAD_PATH.unshift absolute_path unless $LOAD_PATH.include? absolute_path
+end
+flydata_top_dir = File.realpath('../../', __FILE__)
+flydata_core_dir = File.join(flydata_top_dir, 'flydata-core')
+flydata_core_lib_dir = File.realpath('lib', flydata_core_dir)
+append_load_path_if_not_exist(flydata_core_lib_dir)
+
+# require common libs
 require 'active_support/core_ext/string'
 require 'active_support/dependencies'
 require 'active_support/time'