flydata 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +9 -5
- data/lib/flydata/command/sync.rb +89 -988
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +6 -1
- data/lib/flydata/helpers.rb +11 -0
- data/lib/flydata/output/forwarder.rb +166 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +729 -0
- data/lib/flydata/parser/mysql/mysql_alter_table.treetop +214 -2
- data/lib/flydata/sync_file_manager.rb +1 -1
- data/lib/flydata/table_def/mysql_table_def.rb +61 -47
- data/lib/flydata/table_def/redshift_table_def.rb +30 -26
- data/spec/flydata/command/sync_spec.rb +0 -1160
- data/spec/flydata/output/forwarder_spec.rb +105 -0
- data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +224 -23
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +900 -0
- data/spec/flydata/sync_file_manager_spec.rb +159 -0
- data/spec/flydata/table_def/mysql_table_def_spec.rb +2 -2
- data/spec/flydata/table_def/redshift_table_def_spec.rb +199 -44
- metadata +8 -3
@@ -16,12 +16,12 @@ grammar MysqlAlterTable
|
|
16
16
|
|
17
17
|
rule alter_table
|
18
18
|
alter_key sp online_option ignore_option table_key sp tbl_name
|
19
|
-
sp
|
19
|
+
sp alter_commands {
|
20
20
|
def tree
|
21
21
|
value = {
|
22
22
|
type: :alter_table,
|
23
23
|
table_name: tbl_name.table_part.value,
|
24
|
-
actions:
|
24
|
+
actions: alter_commands.actions
|
25
25
|
}
|
26
26
|
value[:schema_name] = tbl_name.schema_part.value if tbl_name.has_schema?
|
27
27
|
value
|
@@ -37,6 +37,31 @@ grammar MysqlAlterTable
|
|
37
37
|
( 'ignore'i sp )?
|
38
38
|
end
|
39
39
|
|
40
|
+
rule alter_commands
|
41
|
+
discard_sym sp tablespace_sym {
|
42
|
+
def actions
|
43
|
+
[{
|
44
|
+
action: :discard_tablespace,
|
45
|
+
query: text_value
|
46
|
+
}]
|
47
|
+
end
|
48
|
+
}
|
49
|
+
/ import_sym sp tablespace_sym {
|
50
|
+
def actions
|
51
|
+
[{
|
52
|
+
action: :import_tablespace,
|
53
|
+
query: text_value
|
54
|
+
}]
|
55
|
+
end
|
56
|
+
}
|
57
|
+
/
|
58
|
+
alter_specs (sp partition_opts)? {
|
59
|
+
def actions
|
60
|
+
alter_specs.actions
|
61
|
+
end
|
62
|
+
}
|
63
|
+
end
|
64
|
+
|
40
65
|
rule alter_specs
|
41
66
|
alter_spec ( comma alter_spec )* {
|
42
67
|
def actions
|
@@ -119,6 +144,40 @@ grammar MysqlAlterTable
|
|
119
144
|
}
|
120
145
|
end
|
121
146
|
}
|
147
|
+
/ disable_sym sp keys_sym {
|
148
|
+
def action
|
149
|
+
{ action: :disable_keys,
|
150
|
+
support_level: :nonbreaking,
|
151
|
+
}
|
152
|
+
end
|
153
|
+
}
|
154
|
+
/ rename_sym opt_to nsp table_ident {
|
155
|
+
def action
|
156
|
+
{ action: :rename_table }
|
157
|
+
end
|
158
|
+
}
|
159
|
+
/ alter_order_clause {
|
160
|
+
def action
|
161
|
+
{ action: :order_table }
|
162
|
+
end
|
163
|
+
}
|
164
|
+
/ convert_sym sp to_sym sp charset sp charset_name_or_default opt_collate {
|
165
|
+
def action
|
166
|
+
{ action: :convert_charset }
|
167
|
+
end
|
168
|
+
}
|
169
|
+
/ create_table_options_space_separated {
|
170
|
+
def action
|
171
|
+
{ action: :default_charset }
|
172
|
+
end
|
173
|
+
}
|
174
|
+
/force_sym {
|
175
|
+
def action
|
176
|
+
{ action: :force,
|
177
|
+
support_level: :nonbreaking,
|
178
|
+
}
|
179
|
+
end
|
180
|
+
}
|
122
181
|
/ [^,]+ {
|
123
182
|
def action
|
124
183
|
raise "unsupported ALTER TABLE query. Contact FlyData Support"
|
@@ -336,6 +395,10 @@ grammar MysqlAlterTable
|
|
336
395
|
'enable'i
|
337
396
|
end
|
338
397
|
|
398
|
+
rule disable_sym
|
399
|
+
'disable'i
|
400
|
+
end
|
401
|
+
|
339
402
|
rule foreign_sym
|
340
403
|
'foreign'i
|
341
404
|
end
|
@@ -700,6 +763,155 @@ grammar MysqlAlterTable
|
|
700
763
|
}
|
701
764
|
end
|
702
765
|
|
766
|
+
rule rename_sym
|
767
|
+
'rename'i ![A-Za-z0-9_]
|
768
|
+
end
|
769
|
+
|
770
|
+
rule to_sym
|
771
|
+
'to'i ![A-Za-z0-9_]
|
772
|
+
end
|
773
|
+
|
774
|
+
rule eq_sym
|
775
|
+
'='
|
776
|
+
end
|
777
|
+
|
778
|
+
rule as_sym
|
779
|
+
'as'i
|
780
|
+
end
|
781
|
+
|
782
|
+
rule opt_to
|
783
|
+
(nsp (to_sym / eq_sym / as_sym) )?
|
784
|
+
end
|
785
|
+
|
786
|
+
rule order_sym
|
787
|
+
'order'i
|
788
|
+
end
|
789
|
+
|
790
|
+
rule by_sym
|
791
|
+
'by'i
|
792
|
+
end
|
793
|
+
|
794
|
+
rule alter_order_clause
|
795
|
+
order_sym sp by_sym sp alter_order_list
|
796
|
+
end
|
797
|
+
|
798
|
+
rule alter_order_list
|
799
|
+
alter_order_item comma alter_order_list
|
800
|
+
/ alter_order_item
|
801
|
+
end
|
802
|
+
|
803
|
+
rule alter_order_item
|
804
|
+
nsp simple_ident_nospvar order_dir
|
805
|
+
end
|
806
|
+
|
807
|
+
rule simple_ident_nospvar
|
808
|
+
ident '.' ident '.' ident
|
809
|
+
/ ident '.' ident
|
810
|
+
/ '.' ident '.' ident
|
811
|
+
/ ident
|
812
|
+
end
|
813
|
+
|
814
|
+
rule convert_sym
|
815
|
+
'convert'i ![A-Za-z0-9_]
|
816
|
+
end
|
817
|
+
|
818
|
+
rule character_sym
|
819
|
+
'character'i ![A-Za-z0-9_]
|
820
|
+
end
|
821
|
+
|
822
|
+
rule set_sym
|
823
|
+
'set'i ![A-Za-z0-9_]
|
824
|
+
end
|
825
|
+
|
826
|
+
rule charset_sym
|
827
|
+
'charset'i ![A-Za-z0-9_]
|
828
|
+
end
|
829
|
+
|
830
|
+
rule default_sym
|
831
|
+
'default'i ![A-Za-z0-9_]
|
832
|
+
end
|
833
|
+
|
834
|
+
rule binary_sym
|
835
|
+
'binary'i ![A-Za-z0-9_]
|
836
|
+
end
|
837
|
+
|
838
|
+
rule collate_sym
|
839
|
+
'collate'i ![A-Za-z0-9_]
|
840
|
+
end
|
841
|
+
|
842
|
+
rule force_sym
|
843
|
+
'force'i ![A-Za-z0-9_]
|
844
|
+
end
|
845
|
+
|
846
|
+
rule discard_sym
|
847
|
+
'discard'i ![A-Za-z0-9_]
|
848
|
+
end
|
849
|
+
|
850
|
+
rule import_sym
|
851
|
+
'import'i ![A-Za-z0-9_]
|
852
|
+
end
|
853
|
+
|
854
|
+
rule tablespace_sym
|
855
|
+
'tablespace'i ![A-Za-z0-9_]
|
856
|
+
end
|
857
|
+
|
858
|
+
rule ident_or_text
|
859
|
+
ident
|
860
|
+
# TEXT_STRING # TODO - To be implemented when required
|
861
|
+
# LEX_HOSTNAME # TODO
|
862
|
+
end
|
863
|
+
|
864
|
+
rule charset_name
|
865
|
+
binary_sym
|
866
|
+
/ ident_or_text
|
867
|
+
end
|
868
|
+
|
869
|
+
rule charset
|
870
|
+
character_sym sp set_sym
|
871
|
+
/ charset_sym
|
872
|
+
end
|
873
|
+
|
874
|
+
rule charset_name_or_default
|
875
|
+
default_sym
|
876
|
+
/ charset_name
|
877
|
+
end
|
878
|
+
|
879
|
+
rule collation_name
|
880
|
+
ident_or_text
|
881
|
+
end
|
882
|
+
|
883
|
+
rule collation_name_or_default
|
884
|
+
default_sym
|
885
|
+
/ collation_name
|
886
|
+
end
|
887
|
+
|
888
|
+
rule opt_collate
|
889
|
+
(sp collate_sym sp collation_name_or_default)?
|
890
|
+
end
|
891
|
+
|
892
|
+
rule create_table_options_space_separated
|
893
|
+
create_table_option sp create_table_options_space_separated
|
894
|
+
/ create_table_option
|
895
|
+
end
|
896
|
+
|
897
|
+
rule create_table_option
|
898
|
+
default_charset
|
899
|
+
/ default_collation
|
900
|
+
#TODO - There are other rules which need to be implemented when required
|
901
|
+
end
|
902
|
+
|
903
|
+
rule default_charset
|
904
|
+
opt_default nsp charset opt_equal nsp charset_name_or_default
|
905
|
+
end
|
906
|
+
|
907
|
+
rule opt_default
|
908
|
+
( nsp default_sym )?
|
909
|
+
end
|
910
|
+
|
911
|
+
rule default_collation
|
912
|
+
opt_default nsp collate_sym opt_equal nsp collation_name_or_default
|
913
|
+
end
|
914
|
+
|
703
915
|
######## keys
|
704
916
|
|
705
917
|
rule alter_key 'alter'i end
|
@@ -25,7 +25,7 @@ module Flydata
|
|
25
25
|
|
26
26
|
def load_dump_pos
|
27
27
|
path = dump_pos_path
|
28
|
-
return
|
28
|
+
return {} unless File.exists?(path)
|
29
29
|
items = File.open(path, 'r').readline.split("\t")
|
30
30
|
raise "Invalid dump.pos file: #{path}" unless items.length >= 5 && items.length <= 7
|
31
31
|
mysql_table = load_mysql_table_marshal_dump
|
@@ -2,57 +2,11 @@ module Flydata
|
|
2
2
|
module TableDef
|
3
3
|
|
4
4
|
class MysqlTableDef
|
5
|
-
TYPE_MAP_M2F = {
|
6
|
-
'bigint' => 'int8',
|
7
|
-
'binary' => 'binary',
|
8
|
-
'blob' => 'varbinary(65535)',
|
9
|
-
'bool' => 'int1',
|
10
|
-
'boolean' => 'int1',
|
11
|
-
'char' => 'varchar',
|
12
|
-
'date' => 'date',
|
13
|
-
'datetime' => 'datetime',
|
14
|
-
'dec' => 'numeric',
|
15
|
-
'decimal' => 'numeric',
|
16
|
-
'double' => 'float8',
|
17
|
-
'double precision' => 'float8',
|
18
|
-
'enum' => 'enum',
|
19
|
-
'fixed' => 'numeric',
|
20
|
-
'float' => 'float4',
|
21
|
-
'int' => 'int4',
|
22
|
-
'integer' => 'int4',
|
23
|
-
'longblob' => 'varbinary(4294967295)',
|
24
|
-
'longtext' => 'text',
|
25
|
-
'mediumblob' => 'varbinary(16777215)',
|
26
|
-
'mediumint' => 'int3',
|
27
|
-
'mediumtext' => 'text',
|
28
|
-
'numeric' => 'numeric',
|
29
|
-
'smallint' => 'int2',
|
30
|
-
'text' => 'text',
|
31
|
-
'time' => 'time',
|
32
|
-
'timestamp' => 'datetime',
|
33
|
-
'tinyblob' => 'varbinary(255)',
|
34
|
-
'tinyint' => 'int1',
|
35
|
-
'tinytext' => 'text',
|
36
|
-
'varbinary' => 'varbinary',
|
37
|
-
'varchar' => 'varchar',
|
38
|
-
}
|
39
|
-
|
40
|
-
def self.convert_to_flydata_type(type)
|
41
|
-
TYPE_MAP_M2F.each do |mysql_type, flydata_type|
|
42
|
-
if /^#{mysql_type}\(|^#{mysql_type}$/.match(type)
|
43
|
-
ret_type = type.gsub(/^#{mysql_type}/, flydata_type)
|
44
|
-
ret_type = check_and_set_varchar_length(ret_type, mysql_type, flydata_type)
|
45
|
-
return ret_type
|
46
|
-
end
|
47
|
-
end
|
48
|
-
nil
|
49
|
-
end
|
50
|
-
|
51
5
|
# Check and set the varchar(char) size which is converted from
|
52
6
|
# length to byte size.
|
53
7
|
# On Mysql the record size of varchar(char) is a length of characters.
|
54
8
|
# ex) varchar(6) on mysql -> varchar(18) on flydata
|
55
|
-
|
9
|
+
PROC_override_varchar = ->(type, mysql_type, flydata_type) do
|
56
10
|
return type unless %w(char varchar).include?(mysql_type)
|
57
11
|
if type =~ /\((\d+)\)/
|
58
12
|
# expect 3 byte UTF-8 character
|
@@ -62,6 +16,66 @@ class MysqlTableDef
|
|
62
16
|
end
|
63
17
|
end
|
64
18
|
|
19
|
+
PROC_override_varbinary = ->(type, mysql_type, flydata_type) do
|
20
|
+
return type unless %w(binary varbinary).include?(mysql_type)
|
21
|
+
if type =~ /\((\d+)\)/
|
22
|
+
# expect 2 bytes for each original byte + 2 bytes for the prefix
|
23
|
+
# ex) 4E5DFF => "0x4e5dff"
|
24
|
+
"#{flydata_type}(#{$1.to_i * 2 + 2})"
|
25
|
+
else
|
26
|
+
raise "Invalid varbinary type. It must be a bug... type:#{type}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
TYPE_MAP_M2F = {
|
31
|
+
'bigint' => {type: 'int8'},
|
32
|
+
'binary' => {type: 'binary', override: PROC_override_varbinary},
|
33
|
+
'blob' => {type: 'varbinary(65535)'},
|
34
|
+
'bool' => {type: 'int1'},
|
35
|
+
'boolean' => {type: 'int1'},
|
36
|
+
'char' => {type: 'varchar', override: PROC_override_varchar},
|
37
|
+
'date' => {type: 'date'},
|
38
|
+
'datetime' => {type: 'datetime'},
|
39
|
+
'dec' => {type: 'numeric'},
|
40
|
+
'decimal' => {type: 'numeric'},
|
41
|
+
'double' => {type: 'float8'},
|
42
|
+
'double precision' => {type: 'float8'},
|
43
|
+
'enum' => {type: 'enum'},
|
44
|
+
'fixed' => {type: 'numeric'},
|
45
|
+
'float' => {type: 'float4'},
|
46
|
+
'int' => {type: 'int4'},
|
47
|
+
'integer' => {type: 'int4'},
|
48
|
+
'longblob' => {type: 'varbinary(4294967295)'},
|
49
|
+
'longtext' => {type: 'text'},
|
50
|
+
'mediumblob' => {type: 'varbinary(16777215)'},
|
51
|
+
'mediumint' => {type: 'int3'},
|
52
|
+
'mediumtext' => {type: 'text'},
|
53
|
+
'numeric' => {type: 'numeric'},
|
54
|
+
'smallint' => {type: 'int2'},
|
55
|
+
'text' => {type: 'text'},
|
56
|
+
'time' => {type: 'time'},
|
57
|
+
'timestamp' => {type: 'datetime'},
|
58
|
+
'tinyblob' => {type: 'varbinary(255)'},
|
59
|
+
'tinyint' => {type: 'int1'},
|
60
|
+
'tinytext' => {type: 'text'},
|
61
|
+
'varbinary' => {type: 'varbinary', override: PROC_override_varbinary},
|
62
|
+
'varchar' => {type: 'varchar', override: PROC_override_varchar},
|
63
|
+
}
|
64
|
+
|
65
|
+
def self.convert_to_flydata_type(type)
|
66
|
+
TYPE_MAP_M2F.each do |mysql_type, type_hash|
|
67
|
+
flydata_type = type_hash[:type]
|
68
|
+
if /^#{mysql_type}\(|^#{mysql_type}$/.match(type)
|
69
|
+
ret_type = type.gsub(/^#{mysql_type}/, flydata_type)
|
70
|
+
if type_hash[:override]
|
71
|
+
ret_type = type_hash[:override].call(ret_type, mysql_type, flydata_type)
|
72
|
+
end
|
73
|
+
return ret_type
|
74
|
+
end
|
75
|
+
end
|
76
|
+
nil
|
77
|
+
end
|
78
|
+
|
65
79
|
def self.create(io)
|
66
80
|
params = _create(io)
|
67
81
|
params ? self.new(*params) : nil
|
@@ -3,31 +3,31 @@ module TableDef
|
|
3
3
|
|
4
4
|
class RedshiftTableDef
|
5
5
|
TYPE_MAP_F2R = {
|
6
|
-
'binary' => {type: 'varchar', use_params: true},
|
7
|
-
'char' => {type: 'char', use_params: true},
|
8
|
-
'date' => {type: 'date'},
|
9
|
-
'datetime' => {type: 'timestamp'},
|
10
|
-
'enum' => {type: 'varchar encode bytedict'},
|
11
|
-
'float4' => {type: 'float4'},
|
12
|
-
'float4 unsigned' => {type: 'float4'},
|
13
|
-
'float8' => {type: 'float8'},
|
14
|
-
'float8 unsigned' => {type: 'float8'},
|
15
|
-
'int1' => {type: 'int2'},
|
16
|
-
'int1 unsigned' => {type: 'int2', unsigned: true},
|
17
|
-
'int2' => {type: 'int2'},
|
18
|
-
'int2 unsigned' => {type: 'int4', unsigned: true},
|
19
|
-
'int3' => {type: 'int4'},
|
20
|
-
'int3 unsigned' => {type: 'int4', unsigned: true},
|
21
|
-
'int4' => {type: 'int4'},
|
22
|
-
'int4 unsigned' => {type: 'int8', unsigned: true},
|
23
|
-
'int8' => {type: 'int8'},
|
24
|
-
'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true},
|
25
|
-
'numeric' => {type: 'numeric', use_params: true, max_size: [38,37]},
|
26
|
-
'numeric unsigned' => {type: 'numeric', use_params: true, max_size: [38,37]},
|
27
|
-
'text' => {type: 'varchar(max)'},
|
28
|
-
'time' => {type: 'timestamp'},
|
29
|
-
'varbinary' => {type: 'varchar', use_params: true, max_size: 65535},
|
30
|
-
'varchar' => {type: 'varchar', use_params: true, max_size: 65535},
|
6
|
+
'binary' => {type: 'varchar', use_params: true, default_value: ''},
|
7
|
+
'char' => {type: 'char', use_params: true, default_value: ''},
|
8
|
+
'date' => {type: 'date', default_value: '0000-01-01'},
|
9
|
+
'datetime' => {type: 'timestamp', default_value: '0000-01-01'},
|
10
|
+
'enum' => {type: 'varchar encode bytedict', default_value: ''},
|
11
|
+
'float4' => {type: 'float4', default_value: '0'},
|
12
|
+
'float4 unsigned' => {type: 'float4', default_value: '0'},
|
13
|
+
'float8' => {type: 'float8', default_value: '0'},
|
14
|
+
'float8 unsigned' => {type: 'float8', default_value: '0'},
|
15
|
+
'int1' => {type: 'int2', default_value: '0'},
|
16
|
+
'int1 unsigned' => {type: 'int2', unsigned: true, default_value: '0'},
|
17
|
+
'int2' => {type: 'int2', default_value: '0'},
|
18
|
+
'int2 unsigned' => {type: 'int4', unsigned: true, default_value: '0'},
|
19
|
+
'int3' => {type: 'int4', default_value: '0'},
|
20
|
+
'int3 unsigned' => {type: 'int4', unsigned: true, default_value: '0'},
|
21
|
+
'int4' => {type: 'int4', default_value: '0'},
|
22
|
+
'int4 unsigned' => {type: 'int8', unsigned: true, default_value: '0'},
|
23
|
+
'int8' => {type: 'int8', default_value: '0'},
|
24
|
+
'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true, default_value: '0'},
|
25
|
+
'numeric' => {type: 'numeric', use_params: true, max_size: [38,37], default_value: '0'},
|
26
|
+
'numeric unsigned' => {type: 'numeric', use_params: true, max_size: [38,37], default_value: '0'},
|
27
|
+
'text' => {type: 'varchar(max)', default_value: ''},
|
28
|
+
'time' => {type: 'timestamp', default_value: '0000-01-01'},
|
29
|
+
'varbinary' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
|
30
|
+
'varchar' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
|
31
31
|
}
|
32
32
|
def self.from_flydata_tabledef(flydata_tabledef, options = {})
|
33
33
|
options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
|
@@ -86,7 +86,7 @@ EOS
|
|
86
86
|
CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents]
|
87
87
|
end
|
88
88
|
|
89
|
-
def self.column_def_sql(column)
|
89
|
+
def self.column_def_sql(column, opt = {})
|
90
90
|
type = column[:type]
|
91
91
|
if type =~ /\((.*?)\)/
|
92
92
|
type = $` + $'
|
@@ -107,6 +107,10 @@ EOS
|
|
107
107
|
if (column.has_key?(:default))
|
108
108
|
val = replace_default_value(type_info[:type], column[:default])
|
109
109
|
line += " DEFAULT #{val}"
|
110
|
+
elsif column[:not_null] && opt[:for] == :alter_table
|
111
|
+
# Redshift doesn't allow adding a not null column without default value
|
112
|
+
# Add a defalt value
|
113
|
+
line += " DEFAULT '#{type_info[:default_value]}'"
|
110
114
|
end
|
111
115
|
# Commented out because no IDENTITY column must be used for a replicated table.
|
112
116
|
# Values come from the master.
|