flydata 0.2.8 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,12 +16,12 @@ grammar MysqlAlterTable
16
16
 
17
17
  rule alter_table
18
18
  alter_key sp online_option ignore_option table_key sp tbl_name
19
- sp alter_specs (sp partition_opts)? {
19
+ sp alter_commands {
20
20
  def tree
21
21
  value = {
22
22
  type: :alter_table,
23
23
  table_name: tbl_name.table_part.value,
24
- actions: alter_specs.actions
24
+ actions: alter_commands.actions
25
25
  }
26
26
  value[:schema_name] = tbl_name.schema_part.value if tbl_name.has_schema?
27
27
  value
@@ -37,6 +37,31 @@ grammar MysqlAlterTable
37
37
  ( 'ignore'i sp )?
38
38
  end
39
39
 
40
+ rule alter_commands
41
+ discard_sym sp tablespace_sym {
42
+ def actions
43
+ [{
44
+ action: :discard_tablespace,
45
+ query: text_value
46
+ }]
47
+ end
48
+ }
49
+ / import_sym sp tablespace_sym {
50
+ def actions
51
+ [{
52
+ action: :import_tablespace,
53
+ query: text_value
54
+ }]
55
+ end
56
+ }
57
+ /
58
+ alter_specs (sp partition_opts)? {
59
+ def actions
60
+ alter_specs.actions
61
+ end
62
+ }
63
+ end
64
+
40
65
  rule alter_specs
41
66
  alter_spec ( comma alter_spec )* {
42
67
  def actions
@@ -119,6 +144,40 @@ grammar MysqlAlterTable
119
144
  }
120
145
  end
121
146
  }
147
+ / disable_sym sp keys_sym {
148
+ def action
149
+ { action: :disable_keys,
150
+ support_level: :nonbreaking,
151
+ }
152
+ end
153
+ }
154
+ / rename_sym opt_to nsp table_ident {
155
+ def action
156
+ { action: :rename_table }
157
+ end
158
+ }
159
+ / alter_order_clause {
160
+ def action
161
+ { action: :order_table }
162
+ end
163
+ }
164
+ / convert_sym sp to_sym sp charset sp charset_name_or_default opt_collate {
165
+ def action
166
+ { action: :convert_charset }
167
+ end
168
+ }
169
+ / create_table_options_space_separated {
170
+ def action
171
+ { action: :default_charset }
172
+ end
173
+ }
174
+ /force_sym {
175
+ def action
176
+ { action: :force,
177
+ support_level: :nonbreaking,
178
+ }
179
+ end
180
+ }
122
181
  / [^,]+ {
123
182
  def action
124
183
  raise "unsupported ALTER TABLE query. Contact FlyData Support"
@@ -336,6 +395,10 @@ grammar MysqlAlterTable
336
395
  'enable'i
337
396
  end
338
397
 
398
+ rule disable_sym
399
+ 'disable'i
400
+ end
401
+
339
402
  rule foreign_sym
340
403
  'foreign'i
341
404
  end
@@ -700,6 +763,155 @@ grammar MysqlAlterTable
700
763
  }
701
764
  end
702
765
 
766
+ rule rename_sym
767
+ 'rename'i ![A-Za-z0-9_]
768
+ end
769
+
770
+ rule to_sym
771
+ 'to'i ![A-Za-z0-9_]
772
+ end
773
+
774
+ rule eq_sym
775
+ '='
776
+ end
777
+
778
+ rule as_sym
779
+ 'as'i
780
+ end
781
+
782
+ rule opt_to
783
+ (nsp (to_sym / eq_sym / as_sym) )?
784
+ end
785
+
786
+ rule order_sym
787
+ 'order'i
788
+ end
789
+
790
+ rule by_sym
791
+ 'by'i
792
+ end
793
+
794
+ rule alter_order_clause
795
+ order_sym sp by_sym sp alter_order_list
796
+ end
797
+
798
+ rule alter_order_list
799
+ alter_order_item comma alter_order_list
800
+ / alter_order_item
801
+ end
802
+
803
+ rule alter_order_item
804
+ nsp simple_ident_nospvar order_dir
805
+ end
806
+
807
+ rule simple_ident_nospvar
808
+ ident '.' ident '.' ident
809
+ / ident '.' ident
810
+ / '.' ident '.' ident
811
+ / ident
812
+ end
813
+
814
+ rule convert_sym
815
+ 'convert'i ![A-Za-z0-9_]
816
+ end
817
+
818
+ rule character_sym
819
+ 'character'i ![A-Za-z0-9_]
820
+ end
821
+
822
+ rule set_sym
823
+ 'set'i ![A-Za-z0-9_]
824
+ end
825
+
826
+ rule charset_sym
827
+ 'charset'i ![A-Za-z0-9_]
828
+ end
829
+
830
+ rule default_sym
831
+ 'default'i ![A-Za-z0-9_]
832
+ end
833
+
834
+ rule binary_sym
835
+ 'binary'i ![A-Za-z0-9_]
836
+ end
837
+
838
+ rule collate_sym
839
+ 'collate'i ![A-Za-z0-9_]
840
+ end
841
+
842
+ rule force_sym
843
+ 'force'i ![A-Za-z0-9_]
844
+ end
845
+
846
+ rule discard_sym
847
+ 'discard'i ![A-Za-z0-9_]
848
+ end
849
+
850
+ rule import_sym
851
+ 'import'i ![A-Za-z0-9_]
852
+ end
853
+
854
+ rule tablespace_sym
855
+ 'tablespace'i ![A-Za-z0-9_]
856
+ end
857
+
858
+ rule ident_or_text
859
+ ident
860
+ # TEXT_STRING # TODO - To be implemented when required
861
+ # LEX_HOSTNAME # TODO
862
+ end
863
+
864
+ rule charset_name
865
+ binary_sym
866
+ / ident_or_text
867
+ end
868
+
869
+ rule charset
870
+ character_sym sp set_sym
871
+ / charset_sym
872
+ end
873
+
874
+ rule charset_name_or_default
875
+ default_sym
876
+ / charset_name
877
+ end
878
+
879
+ rule collation_name
880
+ ident_or_text
881
+ end
882
+
883
+ rule collation_name_or_default
884
+ default_sym
885
+ / collation_name
886
+ end
887
+
888
+ rule opt_collate
889
+ (sp collate_sym sp collation_name_or_default)?
890
+ end
891
+
892
+ rule create_table_options_space_separated
893
+ create_table_option sp create_table_options_space_separated
894
+ / create_table_option
895
+ end
896
+
897
+ rule create_table_option
898
+ default_charset
899
+ / default_collation
900
+ #TODO - There are other rules which need to be implemented when required
901
+ end
902
+
903
+ rule default_charset
904
+ opt_default nsp charset opt_equal nsp charset_name_or_default
905
+ end
906
+
907
+ rule opt_default
908
+ ( nsp default_sym )?
909
+ end
910
+
911
+ rule default_collation
912
+ opt_default nsp collate_sym opt_equal nsp collation_name_or_default
913
+ end
914
+
703
915
  ######## keys
704
916
 
705
917
  rule alter_key 'alter'i end
@@ -25,7 +25,7 @@ module Flydata
25
25
 
26
26
  def load_dump_pos
27
27
  path = dump_pos_path
28
- return nil unless File.exists?(path)
28
+ return {} unless File.exists?(path)
29
29
  items = File.open(path, 'r').readline.split("\t")
30
30
  raise "Invalid dump.pos file: #{path}" unless items.length >= 5 && items.length <= 7
31
31
  mysql_table = load_mysql_table_marshal_dump
@@ -2,57 +2,11 @@ module Flydata
2
2
  module TableDef
3
3
 
4
4
  class MysqlTableDef
5
- TYPE_MAP_M2F = {
6
- 'bigint' => 'int8',
7
- 'binary' => 'binary',
8
- 'blob' => 'varbinary(65535)',
9
- 'bool' => 'int1',
10
- 'boolean' => 'int1',
11
- 'char' => 'varchar',
12
- 'date' => 'date',
13
- 'datetime' => 'datetime',
14
- 'dec' => 'numeric',
15
- 'decimal' => 'numeric',
16
- 'double' => 'float8',
17
- 'double precision' => 'float8',
18
- 'enum' => 'enum',
19
- 'fixed' => 'numeric',
20
- 'float' => 'float4',
21
- 'int' => 'int4',
22
- 'integer' => 'int4',
23
- 'longblob' => 'varbinary(4294967295)',
24
- 'longtext' => 'text',
25
- 'mediumblob' => 'varbinary(16777215)',
26
- 'mediumint' => 'int3',
27
- 'mediumtext' => 'text',
28
- 'numeric' => 'numeric',
29
- 'smallint' => 'int2',
30
- 'text' => 'text',
31
- 'time' => 'time',
32
- 'timestamp' => 'datetime',
33
- 'tinyblob' => 'varbinary(255)',
34
- 'tinyint' => 'int1',
35
- 'tinytext' => 'text',
36
- 'varbinary' => 'varbinary',
37
- 'varchar' => 'varchar',
38
- }
39
-
40
- def self.convert_to_flydata_type(type)
41
- TYPE_MAP_M2F.each do |mysql_type, flydata_type|
42
- if /^#{mysql_type}\(|^#{mysql_type}$/.match(type)
43
- ret_type = type.gsub(/^#{mysql_type}/, flydata_type)
44
- ret_type = check_and_set_varchar_length(ret_type, mysql_type, flydata_type)
45
- return ret_type
46
- end
47
- end
48
- nil
49
- end
50
-
51
5
  # Check and set the varchar(char) size which is converted from
52
6
  # length to byte size.
53
7
  # On Mysql the record size of varchar(char) is a length of characters.
54
8
  # ex) varchar(6) on mysql -> varchar(18) on flydata
55
- def self.check_and_set_varchar_length(type, mysql_type, flydata_type)
9
+ PROC_override_varchar = ->(type, mysql_type, flydata_type) do
56
10
  return type unless %w(char varchar).include?(mysql_type)
57
11
  if type =~ /\((\d+)\)/
58
12
  # expect 3 byte UTF-8 character
@@ -62,6 +16,66 @@ class MysqlTableDef
62
16
  end
63
17
  end
64
18
 
19
+ PROC_override_varbinary = ->(type, mysql_type, flydata_type) do
20
+ return type unless %w(binary varbinary).include?(mysql_type)
21
+ if type =~ /\((\d+)\)/
22
+ # expect 2 bytes for each original byte + 2 bytes for the prefix
23
+ # ex) 4E5DFF => "0x4e5dff"
24
+ "#{flydata_type}(#{$1.to_i * 2 + 2})"
25
+ else
26
+ raise "Invalid varbinary type. It must be a bug... type:#{type}"
27
+ end
28
+ end
29
+
30
+ TYPE_MAP_M2F = {
31
+ 'bigint' => {type: 'int8'},
32
+ 'binary' => {type: 'binary', override: PROC_override_varbinary},
33
+ 'blob' => {type: 'varbinary(65535)'},
34
+ 'bool' => {type: 'int1'},
35
+ 'boolean' => {type: 'int1'},
36
+ 'char' => {type: 'varchar', override: PROC_override_varchar},
37
+ 'date' => {type: 'date'},
38
+ 'datetime' => {type: 'datetime'},
39
+ 'dec' => {type: 'numeric'},
40
+ 'decimal' => {type: 'numeric'},
41
+ 'double' => {type: 'float8'},
42
+ 'double precision' => {type: 'float8'},
43
+ 'enum' => {type: 'enum'},
44
+ 'fixed' => {type: 'numeric'},
45
+ 'float' => {type: 'float4'},
46
+ 'int' => {type: 'int4'},
47
+ 'integer' => {type: 'int4'},
48
+ 'longblob' => {type: 'varbinary(4294967295)'},
49
+ 'longtext' => {type: 'text'},
50
+ 'mediumblob' => {type: 'varbinary(16777215)'},
51
+ 'mediumint' => {type: 'int3'},
52
+ 'mediumtext' => {type: 'text'},
53
+ 'numeric' => {type: 'numeric'},
54
+ 'smallint' => {type: 'int2'},
55
+ 'text' => {type: 'text'},
56
+ 'time' => {type: 'time'},
57
+ 'timestamp' => {type: 'datetime'},
58
+ 'tinyblob' => {type: 'varbinary(255)'},
59
+ 'tinyint' => {type: 'int1'},
60
+ 'tinytext' => {type: 'text'},
61
+ 'varbinary' => {type: 'varbinary', override: PROC_override_varbinary},
62
+ 'varchar' => {type: 'varchar', override: PROC_override_varchar},
63
+ }
64
+
65
+ def self.convert_to_flydata_type(type)
66
+ TYPE_MAP_M2F.each do |mysql_type, type_hash|
67
+ flydata_type = type_hash[:type]
68
+ if /^#{mysql_type}\(|^#{mysql_type}$/.match(type)
69
+ ret_type = type.gsub(/^#{mysql_type}/, flydata_type)
70
+ if type_hash[:override]
71
+ ret_type = type_hash[:override].call(ret_type, mysql_type, flydata_type)
72
+ end
73
+ return ret_type
74
+ end
75
+ end
76
+ nil
77
+ end
78
+
65
79
  def self.create(io)
66
80
  params = _create(io)
67
81
  params ? self.new(*params) : nil
@@ -3,31 +3,31 @@ module TableDef
3
3
 
4
4
  class RedshiftTableDef
5
5
  TYPE_MAP_F2R = {
6
- 'binary' => {type: 'varchar', use_params: true},
7
- 'char' => {type: 'char', use_params: true},
8
- 'date' => {type: 'date'},
9
- 'datetime' => {type: 'timestamp'},
10
- 'enum' => {type: 'varchar encode bytedict'},
11
- 'float4' => {type: 'float4'},
12
- 'float4 unsigned' => {type: 'float4'},
13
- 'float8' => {type: 'float8'},
14
- 'float8 unsigned' => {type: 'float8'},
15
- 'int1' => {type: 'int2'},
16
- 'int1 unsigned' => {type: 'int2', unsigned: true},
17
- 'int2' => {type: 'int2'},
18
- 'int2 unsigned' => {type: 'int4', unsigned: true},
19
- 'int3' => {type: 'int4'},
20
- 'int3 unsigned' => {type: 'int4', unsigned: true},
21
- 'int4' => {type: 'int4'},
22
- 'int4 unsigned' => {type: 'int8', unsigned: true},
23
- 'int8' => {type: 'int8'},
24
- 'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true},
25
- 'numeric' => {type: 'numeric', use_params: true, max_size: [38,37]},
26
- 'numeric unsigned' => {type: 'numeric', use_params: true, max_size: [38,37]},
27
- 'text' => {type: 'varchar(max)'},
28
- 'time' => {type: 'timestamp'},
29
- 'varbinary' => {type: 'varchar', use_params: true, max_size: 65535},
30
- 'varchar' => {type: 'varchar', use_params: true, max_size: 65535},
6
+ 'binary' => {type: 'varchar', use_params: true, default_value: ''},
7
+ 'char' => {type: 'char', use_params: true, default_value: ''},
8
+ 'date' => {type: 'date', default_value: '0000-01-01'},
9
+ 'datetime' => {type: 'timestamp', default_value: '0000-01-01'},
10
+ 'enum' => {type: 'varchar encode bytedict', default_value: ''},
11
+ 'float4' => {type: 'float4', default_value: '0'},
12
+ 'float4 unsigned' => {type: 'float4', default_value: '0'},
13
+ 'float8' => {type: 'float8', default_value: '0'},
14
+ 'float8 unsigned' => {type: 'float8', default_value: '0'},
15
+ 'int1' => {type: 'int2', default_value: '0'},
16
+ 'int1 unsigned' => {type: 'int2', unsigned: true, default_value: '0'},
17
+ 'int2' => {type: 'int2', default_value: '0'},
18
+ 'int2 unsigned' => {type: 'int4', unsigned: true, default_value: '0'},
19
+ 'int3' => {type: 'int4', default_value: '0'},
20
+ 'int3 unsigned' => {type: 'int4', unsigned: true, default_value: '0'},
21
+ 'int4' => {type: 'int4', default_value: '0'},
22
+ 'int4 unsigned' => {type: 'int8', unsigned: true, default_value: '0'},
23
+ 'int8' => {type: 'int8', default_value: '0'},
24
+ 'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true, default_value: '0'},
25
+ 'numeric' => {type: 'numeric', use_params: true, max_size: [38,37], default_value: '0'},
26
+ 'numeric unsigned' => {type: 'numeric', use_params: true, max_size: [38,37], default_value: '0'},
27
+ 'text' => {type: 'varchar(max)', default_value: ''},
28
+ 'time' => {type: 'timestamp', default_value: '0000-01-01'},
29
+ 'varbinary' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
30
+ 'varchar' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
31
31
  }
32
32
  def self.from_flydata_tabledef(flydata_tabledef, options = {})
33
33
  options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
@@ -86,7 +86,7 @@ EOS
86
86
  CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents]
87
87
  end
88
88
 
89
- def self.column_def_sql(column)
89
+ def self.column_def_sql(column, opt = {})
90
90
  type = column[:type]
91
91
  if type =~ /\((.*?)\)/
92
92
  type = $` + $'
@@ -107,6 +107,10 @@ EOS
107
107
  if (column.has_key?(:default))
108
108
  val = replace_default_value(type_info[:type], column[:default])
109
109
  line += " DEFAULT #{val}"
110
+ elsif column[:not_null] && opt[:for] == :alter_table
111
+ # Redshift doesn't allow adding a not null column without default value
112
+ # Add a defalt value
113
+ line += " DEFAULT '#{type_info[:default_value]}'"
110
114
  end
111
115
  # Commented out because no IDENTITY column must be used for a replicated table.
112
116
  # Values come from the master.