flydata 0.2.8 → 0.2.9

data/lib/flydata/parser/mysql/mysql_alter_table.treetop

@@ -16,12 +16,12 @@ grammar MysqlAlterTable
 
   rule alter_table
     alter_key sp online_option ignore_option table_key sp tbl_name
-    sp alter_specs (sp partition_opts)? {
+    sp alter_commands {
       def tree
         value = {
           type: :alter_table,
           table_name: tbl_name.table_part.value,
-          actions: alter_specs.actions
+          actions: alter_commands.actions
         }
         value[:schema_name] = tbl_name.schema_part.value if tbl_name.has_schema?
         value
@@ -37,6 +37,31 @@ grammar MysqlAlterTable
     ( 'ignore'i sp )?
   end
 
+  rule alter_commands
+    discard_sym sp tablespace_sym {
+      def actions
+        [{
+          action: :discard_tablespace,
+          query: text_value
+         }]
+      end
+    }
+    / import_sym sp tablespace_sym {
+      def actions
+        [{
+          action: :import_tablespace,
+          query: text_value
+         }]
+      end
+    }
+    /
+    alter_specs (sp partition_opts)? {
+      def actions
+        alter_specs.actions
+      end
+    }
+  end
+
   rule alter_specs
     alter_spec ( comma alter_spec )* {
       def actions
@@ -119,6 +144,40 @@ grammar MysqlAlterTable
         }
       end
     }
+    / disable_sym sp keys_sym {
+      def action
+        { action: :disable_keys,
+          support_level: :nonbreaking,
+        }
+      end
+    }
+    / rename_sym opt_to nsp table_ident {
+      def action
+        { action: :rename_table }
+      end
+    }
+    / alter_order_clause {
+      def action
+        { action: :order_table }
+      end
+    }
+    / convert_sym sp to_sym sp charset sp charset_name_or_default opt_collate {
+      def action
+        { action: :convert_charset }
+      end
+    }
+    / create_table_options_space_separated {
+      def action
+        { action: :default_charset }
+      end
+    }
+    /force_sym {
+      def action
+        { action: :force,
+          support_level: :nonbreaking,
+        }
+      end
+    }
     / [^,]+ {
       def action
         raise "unsupported ALTER TABLE query.  Contact FlyData Support"
@@ -336,6 +395,10 @@ grammar MysqlAlterTable
     'enable'i
   end
 
+  rule disable_sym
+    'disable'i
+  end
+
   rule foreign_sym
     'foreign'i
   end
@@ -700,6 +763,155 @@ grammar MysqlAlterTable
     }
   end
 
+  rule rename_sym
+    'rename'i ![A-Za-z0-9_]
+  end
+
+  rule to_sym
+    'to'i ![A-Za-z0-9_]
+  end
+
+  rule eq_sym
+    '='
+  end
+
+  rule as_sym
+    'as'i
+  end
+
+  rule opt_to
+    (nsp (to_sym / eq_sym / as_sym) )?
+  end
+
+  rule order_sym
+    'order'i
+  end
+
+  rule by_sym
+    'by'i
+  end
+
+  rule alter_order_clause
+    order_sym sp by_sym sp alter_order_list
+  end
+
+  rule alter_order_list
+    alter_order_item comma alter_order_list
+    / alter_order_item
+  end
+
+  rule alter_order_item
+    nsp simple_ident_nospvar order_dir
+  end
+
+  rule simple_ident_nospvar
+    ident '.' ident '.' ident
+    / ident '.' ident
+    / '.' ident '.' ident
+    / ident
+  end
+
+  rule convert_sym
+    'convert'i ![A-Za-z0-9_]
+  end
+
+  rule character_sym
+    'character'i ![A-Za-z0-9_]
+  end
+
+  rule set_sym
+    'set'i ![A-Za-z0-9_]
+  end
+
+  rule charset_sym
+    'charset'i ![A-Za-z0-9_]
+  end
+
+  rule default_sym
+    'default'i ![A-Za-z0-9_]
+  end
+
+  rule binary_sym
+    'binary'i ![A-Za-z0-9_]
+  end
+
+  rule collate_sym
+    'collate'i ![A-Za-z0-9_]
+  end
+
+  rule force_sym
+    'force'i ![A-Za-z0-9_]
+  end
+
+  rule discard_sym
+    'discard'i ![A-Za-z0-9_]
+  end
+
+  rule import_sym
+    'import'i ![A-Za-z0-9_]
+  end
+
+  rule tablespace_sym
+    'tablespace'i ![A-Za-z0-9_]
+  end
+
+  rule ident_or_text
+    ident
+    # TEXT_STRING # TODO - To be implemented when required
+    # LEX_HOSTNAME # TODO
+  end
+
+  rule charset_name
+    binary_sym
+    / ident_or_text
+  end
+
+  rule charset
+    character_sym sp set_sym
+    / charset_sym
+  end
+
+  rule charset_name_or_default
+    default_sym
+    / charset_name
+  end
+
+  rule collation_name
+    ident_or_text
+  end
+
+  rule collation_name_or_default
+    default_sym
+    / collation_name
+  end
+
+  rule opt_collate
+    (sp collate_sym sp collation_name_or_default)?
+  end
+
+  rule create_table_options_space_separated
+    create_table_option sp create_table_options_space_separated
+    / create_table_option
+  end
+
+  rule create_table_option
+    default_charset
+    / default_collation
+    #TODO - There are other rules which need to be implemented when required
+  end
+
+  rule default_charset
+    opt_default nsp charset opt_equal nsp charset_name_or_default
+  end
+
+  rule opt_default
+    ( nsp default_sym )?
+  end
+
+  rule default_collation
+    opt_default nsp collate_sym opt_equal nsp collation_name_or_default
+  end
+
   ######## keys
 
   rule alter_key 'alter'i end

data/lib/flydata/sync_file_manager.rb

@@ -25,7 +25,7 @@ module Flydata
 
       def load_dump_pos
         path = dump_pos_path
-        return nil unless File.exists?(path)
+        return {} unless File.exists?(path)
         items = File.open(path, 'r').readline.split("\t")
         raise "Invalid dump.pos file: #{path}" unless items.length >= 5 && items.length <= 7
         mysql_table = load_mysql_table_marshal_dump

data/lib/flydata/table_def/mysql_table_def.rb

@@ -2,57 +2,11 @@ module Flydata
 module TableDef
 
 class MysqlTableDef
-  TYPE_MAP_M2F = {
-    'bigint' => 'int8',
-    'binary' => 'binary',
-    'blob' => 'varbinary(65535)',
-    'bool' => 'int1',
-    'boolean' => 'int1',
-    'char' => 'varchar',
-    'date' => 'date',
-    'datetime' => 'datetime',
-    'dec' => 'numeric',
-    'decimal' => 'numeric',
-    'double' => 'float8',
-    'double precision' => 'float8',
-    'enum' => 'enum',
-    'fixed' => 'numeric',
-    'float' => 'float4',
-    'int' => 'int4',
-    'integer' => 'int4',
-    'longblob' => 'varbinary(4294967295)',
-    'longtext' => 'text',
-    'mediumblob' => 'varbinary(16777215)',
-    'mediumint' => 'int3',
-    'mediumtext' => 'text',
-    'numeric' => 'numeric',
-    'smallint' => 'int2',
-    'text' => 'text',
-    'time' => 'time',
-    'timestamp' => 'datetime',
-    'tinyblob' => 'varbinary(255)',
-    'tinyint' => 'int1',
-    'tinytext' => 'text',
-    'varbinary' => 'varbinary',
-    'varchar' => 'varchar',
-  }
-
-  def self.convert_to_flydata_type(type)
-    TYPE_MAP_M2F.each do |mysql_type, flydata_type|
-      if /^#{mysql_type}\(|^#{mysql_type}$/.match(type)
-        ret_type = type.gsub(/^#{mysql_type}/, flydata_type)
-        ret_type = check_and_set_varchar_length(ret_type, mysql_type, flydata_type)
-        return ret_type
-      end
-    end
-    nil
-  end
-
   # Check and set the varchar(char) size which is converted from
   # length to byte size.
   # On Mysql the record size of varchar(char) is a length of characters.
   # ex) varchar(6) on mysql -> varchar(18) on flydata
-  def self.check_and_set_varchar_length(type, mysql_type, flydata_type)
+  PROC_override_varchar = ->(type, mysql_type, flydata_type) do
     return type unless %w(char varchar).include?(mysql_type)
     if type =~ /\((\d+)\)/
       # expect 3 byte UTF-8 character
@@ -62,6 +16,66 @@ class MysqlTableDef
     end
   end
 
+  PROC_override_varbinary = ->(type, mysql_type, flydata_type) do
+    return type unless %w(binary varbinary).include?(mysql_type)
+    if type =~ /\((\d+)\)/
+      # expect 2 bytes for each original byte + 2 bytes for the prefix
+      # ex) 4E5DFF => "0x4e5dff"
+      "#{flydata_type}(#{$1.to_i * 2 + 2})"
+    else
+      raise "Invalid varbinary type. It must be a bug... type:#{type}"
+    end
+  end
+
+  TYPE_MAP_M2F = {
+    'bigint' => {type: 'int8'},
+    'binary' => {type: 'binary', override: PROC_override_varbinary},
+    'blob' => {type: 'varbinary(65535)'},
+    'bool' => {type: 'int1'},
+    'boolean' => {type: 'int1'},
+    'char' => {type: 'varchar', override: PROC_override_varchar},
+    'date' => {type: 'date'},
+    'datetime' => {type: 'datetime'},
+    'dec' => {type: 'numeric'},
+    'decimal' => {type: 'numeric'},
+    'double' => {type: 'float8'},
+    'double precision' => {type: 'float8'},
+    'enum' => {type: 'enum'},
+    'fixed' => {type: 'numeric'},
+    'float' => {type: 'float4'},
+    'int' => {type: 'int4'},
+    'integer' => {type: 'int4'},
+    'longblob' => {type: 'varbinary(4294967295)'},
+    'longtext' => {type: 'text'},
+    'mediumblob' => {type: 'varbinary(16777215)'},
+    'mediumint' => {type: 'int3'},
+    'mediumtext' => {type: 'text'},
+    'numeric' => {type: 'numeric'},
+    'smallint' => {type: 'int2'},
+    'text' => {type: 'text'},
+    'time' => {type: 'time'},
+    'timestamp' => {type: 'datetime'},
+    'tinyblob' => {type: 'varbinary(255)'},
+    'tinyint' => {type: 'int1'},
+    'tinytext' => {type: 'text'},
+    'varbinary' => {type: 'varbinary', override: PROC_override_varbinary},
+    'varchar' => {type: 'varchar', override: PROC_override_varchar},
+  }
+
+  def self.convert_to_flydata_type(type)
+    TYPE_MAP_M2F.each do |mysql_type, type_hash|
+      flydata_type = type_hash[:type]
+      if /^#{mysql_type}\(|^#{mysql_type}$/.match(type)
+        ret_type = type.gsub(/^#{mysql_type}/, flydata_type)
+        if type_hash[:override]
+          ret_type = type_hash[:override].call(ret_type, mysql_type, flydata_type)
+        end
+        return ret_type
+      end
+    end
+    nil
+  end
+
   def self.create(io)
     params = _create(io)
     params ? self.new(*params) : nil

data/lib/flydata/table_def/redshift_table_def.rb

@@ -3,31 +3,31 @@ module TableDef
 
 class RedshiftTableDef
   TYPE_MAP_F2R = {
-    'binary' => {type: 'varchar', use_params: true},
-    'char' => {type: 'char', use_params: true},
-    'date' => {type: 'date'},
-    'datetime' => {type: 'timestamp'},
-    'enum' => {type: 'varchar encode bytedict'},
-    'float4' => {type: 'float4'},
-    'float4 unsigned' => {type: 'float4'},
-    'float8' => {type: 'float8'},
-    'float8 unsigned' => {type: 'float8'},
-    'int1' => {type: 'int2'},
-    'int1 unsigned' => {type: 'int2', unsigned: true},
-    'int2' => {type: 'int2'},
-    'int2 unsigned' => {type: 'int4', unsigned: true},
-    'int3' => {type: 'int4'},
-    'int3 unsigned' => {type: 'int4', unsigned: true},
-    'int4' => {type: 'int4'},
-    'int4 unsigned' => {type: 'int8', unsigned: true},
-    'int8' => {type: 'int8'},
-    'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true},
-    'numeric' => {type: 'numeric', use_params: true, max_size: [38,37]},
-    'numeric unsigned' => {type: 'numeric', use_params: true, max_size: [38,37]},
-    'text' => {type: 'varchar(max)'},
-    'time' => {type: 'timestamp'},
-    'varbinary' => {type: 'varchar', use_params: true, max_size: 65535},
-    'varchar' => {type: 'varchar', use_params: true, max_size: 65535},
+    'binary' => {type: 'varchar', use_params: true, default_value: ''},
+    'char' => {type: 'char', use_params: true, default_value: ''},
+    'date' => {type: 'date', default_value: '0000-01-01'},
+    'datetime' => {type: 'timestamp', default_value: '0000-01-01'},
+    'enum' => {type: 'varchar encode bytedict', default_value: ''},
+    'float4' => {type: 'float4', default_value: '0'},
+    'float4 unsigned' => {type: 'float4', default_value: '0'},
+    'float8' => {type: 'float8', default_value: '0'},
+    'float8 unsigned' => {type: 'float8', default_value: '0'},
+    'int1' => {type: 'int2', default_value: '0'},
+    'int1 unsigned' => {type: 'int2', unsigned: true, default_value: '0'},
+    'int2' => {type: 'int2', default_value: '0'},
+    'int2 unsigned' => {type: 'int4', unsigned: true, default_value: '0'},
+    'int3' => {type: 'int4', default_value: '0'},
+    'int3 unsigned' => {type: 'int4', unsigned: true, default_value: '0'},
+    'int4' => {type: 'int4', default_value: '0'},
+    'int4 unsigned' => {type: 'int8', unsigned: true, default_value: '0'},
+    'int8' => {type: 'int8', default_value: '0'},
+    'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true, default_value: '0'},
+    'numeric' => {type: 'numeric', use_params: true, max_size: [38,37], default_value: '0'},
+    'numeric unsigned' => {type: 'numeric', use_params: true, max_size: [38,37], default_value: '0'},
+    'text' => {type: 'varchar(max)', default_value: ''},
+    'time' => {type: 'timestamp', default_value: '0000-01-01'},
+    'varbinary' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
+    'varchar' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
   }
   def self.from_flydata_tabledef(flydata_tabledef, options = {})
     options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
@@ -86,7 +86,7 @@ EOS
     CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents]
   end
 
-  def self.column_def_sql(column)
+  def self.column_def_sql(column, opt = {})
     type = column[:type]
     if type =~ /\((.*?)\)/
       type = $` + $'
@@ -107,6 +107,10 @@ EOS
     if (column.has_key?(:default))
       val = replace_default_value(type_info[:type], column[:default])
       line += " DEFAULT #{val}"
+    elsif column[:not_null] && opt[:for] == :alter_table
+      # Redshift doesn't allow adding a not null column without default value
+      # Add a defalt value
+      line += " DEFAULT '#{type_info[:default_value]}'"
     end
 # Commented out because no IDENTITY column must be used for a replicated table.
 # Values come from the master.