flydata 0.3.16 → 0.3.17
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/record/record.rb +13 -0
- data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +107 -5
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +62 -11
- data/flydata-core/spec/table_def/mysql_table_def_spec.rb +37 -1
- data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +120 -0
- data/flydata-core/spec/table_def/mysqldump_test_column_charset.dump +45 -0
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +70 -88
- data/flydata.gemspec +13 -8
- data/lib/flydata/command/setup.rb +4 -4
- data/lib/flydata/command/sync.rb +18 -29
- data/lib/flydata/compatibility_check.rb +2 -2
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +15 -10
- data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -3
- data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb +15 -8
- data/lib/flydata/fluent-plugins/mysql/table_meta.rb +6 -34
- data/lib/flydata/{fluent-plugins/mysql → mysql}/binlog_position.rb +2 -0
- data/lib/flydata/{util → mysql}/mysql_util.rb +34 -1
- data/lib/flydata/mysql/table_ddl.rb +118 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +5 -5
- data/lib/flydata/parser/mysql/mysql_alter_table.treetop +29 -5
- data/lib/flydata/sync_file_manager.rb +15 -2
- data/spec/flydata/command/sync_spec.rb +22 -1
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +7 -2
- data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +130 -0
- data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +1 -0
- data/spec/flydata/fluent-plugins/mysql/table_meta_spec.rb +14 -8
- data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +2 -1
- data/spec/flydata/{fluent-plugins/mysql → mysql}/binlog_position_spec.rb +3 -2
- data/spec/flydata/{util → mysql}/mysql_util_spec.rb +2 -2
- data/spec/flydata/mysql/table_ddl_spec.rb +193 -0
- data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +37 -9
- data/spec/flydata/sync_file_manager_spec.rb +102 -27
- metadata +12 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 15226b89faab2ec73d834294f47e4b2f8fc35b69
|
4
|
+
data.tar.gz: 19d207fcea4121d051cb73f536c04769c95cfbc0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f67502dcee392df1f1016346390266e5e977450359cec43ca0a20402707a28b84547b76ea06063ef9c50de510652a8cc362ae914087e37ace2fc9c2d54384581
|
7
|
+
data.tar.gz: a2dbad269e6e2e7eb0e2042c4b05a51441cb92b3f6cca58cc2fdd47b922a0a09eb702d22882e64557757dd93b809cd2552ecbc81f45c27673b75ee16317b3d19
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.17
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module FlydataCore
|
2
|
+
module Record
|
3
|
+
|
4
|
+
V1 = 1
|
5
|
+
# - Handles string/binary as UTF-8
|
6
|
+
# - No "v" peroperty
|
7
|
+
|
8
|
+
V2 = 2
|
9
|
+
# - Supports the common string/binary format
|
10
|
+
# - "v": 2
|
11
|
+
# - additions of "attrs" property. "enc":"b" is currently supported.
|
12
|
+
end
|
13
|
+
end
|
@@ -87,11 +87,14 @@ class MysqlTableDef
|
|
87
87
|
params ? self.new(*params) : nil
|
88
88
|
end
|
89
89
|
|
90
|
-
def initialize(table_def, table_name, columns, default_charset,
|
90
|
+
def initialize(table_def, table_name, columns, column_def, default_charset,
|
91
|
+
default_charset_mysql, comment)
|
91
92
|
@table_def = table_def
|
92
93
|
@table_name = table_name
|
93
94
|
@columns = columns
|
95
|
+
@column_def = column_def
|
94
96
|
@default_charset = default_charset
|
97
|
+
@default_charset_mysql = default_charset_mysql
|
95
98
|
@comment = comment
|
96
99
|
end
|
97
100
|
|
@@ -99,7 +102,9 @@ class MysqlTableDef
|
|
99
102
|
table_def = ''
|
100
103
|
table_name = nil
|
101
104
|
columns = []
|
105
|
+
column_def = {}
|
102
106
|
default_charset = nil
|
107
|
+
default_charset_mysql = nil
|
103
108
|
comment = nil
|
104
109
|
|
105
110
|
position = :before_create_table
|
@@ -120,13 +125,19 @@ class MysqlTableDef
|
|
120
125
|
stripped_line = line.strip
|
121
126
|
# `col_smallint` smallint(6) DEFAULT NULL,
|
122
127
|
if stripped_line.start_with?('`')
|
123
|
-
|
128
|
+
column = parse_one_column_def(line)
|
129
|
+
columns << column
|
130
|
+
coldef = line.strip.gsub(/,$/, '')
|
131
|
+
column_def[column[:column]] = coldef
|
124
132
|
# PRIMARY KEY (`id`)
|
125
133
|
elsif stripped_line.start_with?("PRIMARY KEY")
|
126
134
|
parse_key(line, columns)
|
127
135
|
#) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table';
|
128
136
|
elsif stripped_line.start_with?(')')
|
129
|
-
|
137
|
+
if line =~ /DEFAULT CHARSET\s*=\s*([a-z0-9]+)/
|
138
|
+
default_charset_mysql = $1
|
139
|
+
default_charset = flydata_charset(default_charset_mysql)
|
140
|
+
end
|
130
141
|
comment = $1 if /COMMENT='((?:\\'|[^'])*)'/.match(line)
|
131
142
|
position = :after_create_table
|
132
143
|
elsif stripped_line.start_with?("KEY")
|
@@ -146,9 +157,9 @@ class MysqlTableDef
|
|
146
157
|
break
|
147
158
|
end
|
148
159
|
end
|
149
|
-
position == :after_create_table ? [table_def, table_name, columns, default_charset, comment] : nil
|
160
|
+
position == :after_create_table ? [table_def, table_name, columns, column_def, default_charset, default_charset_mysql, comment] : nil
|
150
161
|
end
|
151
|
-
attr_reader :columns, :table_name
|
162
|
+
attr_reader :columns, :column_def, :table_name, :default_charset_mysql
|
152
163
|
|
153
164
|
def to_flydata_tabledef
|
154
165
|
tabledef = { table_name: @table_name,
|
@@ -207,6 +218,9 @@ class MysqlTableDef
|
|
207
218
|
if /COMMENT\s+'(((?:\\'|[^'])*))'/i.match(line)
|
208
219
|
column[:comment] = $1
|
209
220
|
end
|
221
|
+
if /CHARACTER SET\s+([a-z0-9]+)/i.match(line)
|
222
|
+
column[:charset] = flydata_charset($1)
|
223
|
+
end
|
210
224
|
if block_given?
|
211
225
|
column = yield(column, query, pos)
|
212
226
|
end
|
@@ -218,6 +232,93 @@ class MysqlTableDef
|
|
218
232
|
column
|
219
233
|
end
|
220
234
|
|
235
|
+
# 'binary' is a fixed length column type. mysqldump exports a binary column
|
236
|
+
# value at its full length by adding trailing 0's to the size of the column
|
237
|
+
# (ex. '0xAB00920000' for type 'binary(5)')
|
238
|
+
# FlyData Transport Format does not allow such paddings so they need to be
|
239
|
+
# removed. On the target database, padding may be added according to the
|
240
|
+
# destination DB's convention.
|
241
|
+
MYSQL_BINARY_PADDING_CHAR = '00'
|
242
|
+
PROC_convert_binary_value = -> (value) do
|
243
|
+
return nil unless value
|
244
|
+
while value.end_with?(MYSQL_BINARY_PADDING_CHAR)
|
245
|
+
value = value[0...-MYSQL_BINARY_PADDING_CHAR.size]
|
246
|
+
end
|
247
|
+
value
|
248
|
+
end
|
249
|
+
|
250
|
+
VALUE_CONVERTERS = {
|
251
|
+
'binary' => PROC_convert_binary_value,
|
252
|
+
}
|
253
|
+
|
254
|
+
def self.convert_to_flydata_value(value, type)
|
255
|
+
if converter = VALUE_CONVERTERS[type]
|
256
|
+
value = converter.call(value)
|
257
|
+
end
|
258
|
+
value
|
259
|
+
end
|
260
|
+
|
261
|
+
# Charset naming conversion rule. mysql => ruby
|
262
|
+
#
|
263
|
+
# mysql
|
264
|
+
# http://dev.mysql.com/doc/refman/5.6/en/charset-charsets.html
|
265
|
+
# mysql(supported CJK character sets)
|
266
|
+
# http://dev.mysql.com/doc/refman/5.6/en/faqs-cjk.html#qandaitem-A-11-1-1
|
267
|
+
# For ruby, you can see encoding list with "Encoding.list"
|
268
|
+
CHARSET_ENCODE_RULE = {
|
269
|
+
"armscii8" => nil,
|
270
|
+
"ascii" => Encoding::US_ASCII,
|
271
|
+
"big5" => Encoding::Big5,
|
272
|
+
"binary" => Encoding::ASCII_8BIT,
|
273
|
+
"cp1250" => Encoding::Windows_1250,
|
274
|
+
"cp1251" => Encoding::Windows_1251,
|
275
|
+
"cp1256" => Encoding::Windows_1256,
|
276
|
+
"cp1257" => Encoding::Windows_1257,
|
277
|
+
"cp850" => Encoding::CP850,
|
278
|
+
"cp852" => Encoding::CP852,
|
279
|
+
"cp866" => Encoding::IBM866,
|
280
|
+
"cp932" => Encoding::Windows_31J,
|
281
|
+
"dec8" => nil,
|
282
|
+
"eucjpms" => Encoding::EucJP_ms,
|
283
|
+
"euckr" => Encoding::EUC_KR,
|
284
|
+
"gb2312" => Encoding::EUC_CN,
|
285
|
+
"gbk" => Encoding::GBK,
|
286
|
+
"geostd8" => nil,
|
287
|
+
"greek" => Encoding::ISO_8859_7,
|
288
|
+
"hebrew" => Encoding::ISO_8859_8,
|
289
|
+
"hp8" => nil,
|
290
|
+
"keybcs2" => nil,
|
291
|
+
"koi8r" => Encoding::KOI8_R,
|
292
|
+
"koi8u" => Encoding::KOI8_U,
|
293
|
+
"latin1" => Encoding::ISO_8859_1,
|
294
|
+
"latin2" => Encoding::ISO_8859_2,
|
295
|
+
"latin5" => Encoding::ISO_8859_9,
|
296
|
+
"latin7" => Encoding::ISO_8859_13,
|
297
|
+
"macce" => Encoding::MacCentEuro,
|
298
|
+
"macroman" => Encoding::MacRoman,
|
299
|
+
"sjis" => Encoding::SHIFT_JIS,
|
300
|
+
"swe7" => nil,
|
301
|
+
"tis620" => Encoding::TIS_620,
|
302
|
+
"ucs2" => Encoding::UTF_16BE,
|
303
|
+
"ujis" => Encoding::EUC_JP,
|
304
|
+
"utf16" => Encoding::UTF_16,
|
305
|
+
"utf16le" => Encoding::UTF_16LE,
|
306
|
+
"utf32" => Encoding::UTF_32,
|
307
|
+
"utf8" => Encoding::UTF_8,
|
308
|
+
"utf8mb4" => Encoding::UTF_8,
|
309
|
+
}
|
310
|
+
|
311
|
+
def self.ruby_encoding(mysql_charset)
|
312
|
+
return nil if mysql_charset.to_s.empty?
|
313
|
+
raise "Unsupported charset:#{mysql_charset}." unless CHARSET_ENCODE_RULE.has_key?(mysql_charset)
|
314
|
+
charset = CHARSET_ENCODE_RULE[mysql_charset]
|
315
|
+
return charset
|
316
|
+
end
|
317
|
+
|
318
|
+
def self.flydata_charset(mysql_charset)
|
319
|
+
ruby_encoding(mysql_charset).to_s.upcase.gsub('-', '_')
|
320
|
+
end
|
321
|
+
|
221
322
|
private
|
222
323
|
|
223
324
|
def self.parse_key(line, columns, type = :primary_key)
|
@@ -232,6 +333,7 @@ class MysqlTableDef
|
|
232
333
|
column[type] = true
|
233
334
|
end
|
234
335
|
end
|
336
|
+
|
235
337
|
end
|
236
338
|
|
237
339
|
end
|
@@ -53,10 +53,11 @@ class RedshiftTableDef
|
|
53
53
|
end
|
54
54
|
|
55
55
|
FLYDATA_CTL_COLUMNS_TABLE = "flydata_ctl_columns"
|
56
|
+
FLYDATA_CTL_TABLES_TABLE = "flydata_ctl_tables"
|
56
57
|
CREATE_SCHEMA_SQL = <<EOS
|
57
|
-
CREATE SCHEMA "%s";
|
58
|
+
CREATE SCHEMA IF NOT EXISTS "%s";
|
58
59
|
EOS
|
59
|
-
|
60
|
+
CREATE_FLYDATA_CTL_COLUMNS_SQL = <<EOS
|
60
61
|
CREATE TABLE IF NOT EXISTS %s(
|
61
62
|
id integer NOT NULL IDENTITY(1,1),
|
62
63
|
table_name varchar(128) NOT NULL,
|
@@ -67,13 +68,29 @@ CREATE TABLE IF NOT EXISTS %s(
|
|
67
68
|
PRIMARY KEY(id)
|
68
69
|
) DISTKEY(table_name) SORTKEY(table_name);
|
69
70
|
EOS
|
71
|
+
CREATE_FLYDATA_CTL_TABLES_SQL = <<EOS
|
72
|
+
CREATE TABLE IF NOT EXISTS %s(
|
73
|
+
id integer NOT NULL IDENTITY(1,1),
|
74
|
+
table_name varchar(128) NOT NULL,
|
75
|
+
attribute varchar(128) NOT NULL,
|
76
|
+
value varchar(max),
|
77
|
+
created_at timestamp DEFAULT SYSDATE,
|
78
|
+
PRIMARY KEY(id)
|
79
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
80
|
+
EOS
|
81
|
+
CREATE_FLYDATA_CTL_TABLE_SQL = "#{CREATE_FLYDATA_CTL_COLUMNS_SQL}#{CREATE_FLYDATA_CTL_TABLES_SQL}"
|
82
|
+
|
70
83
|
|
71
84
|
def self.table_name_for_ddl(table_name, schema_name)
|
72
85
|
schema_name.to_s.empty? ? "\"#{table_name}\"" : "\"#{schema_name}\".\"#{table_name}\""
|
73
86
|
end
|
74
87
|
|
75
|
-
def self.flydata_ctl_table_for_ddl(schema_name)
|
76
|
-
|
88
|
+
def self.flydata_ctl_table_for_ddl(schema_name, ctl_table_type = :columns)
|
89
|
+
table_name = case ctl_table_type
|
90
|
+
when :columns; FLYDATA_CTL_COLUMNS_TABLE
|
91
|
+
when :tables; FLYDATA_CTL_TABLES_TABLE
|
92
|
+
end
|
93
|
+
table_name_for_ddl(table_name, schema_name)
|
77
94
|
end
|
78
95
|
|
79
96
|
def self.create_schema_sql(schema_name)
|
@@ -82,8 +99,14 @@ EOS
|
|
82
99
|
|
83
100
|
def self.create_flydata_ctl_table_sql(schema_name)
|
84
101
|
# No drop table here intentionally because losing the data is fatal.
|
85
|
-
|
86
|
-
|
102
|
+
columns_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
|
103
|
+
tables_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
|
104
|
+
CREATE_FLYDATA_CTL_TABLE_SQL % [columns_tbl, tables_tbl]
|
105
|
+
end
|
106
|
+
|
107
|
+
def self.create_flydata_ctl_tables_sql(schema_name)
|
108
|
+
tables_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
|
109
|
+
CREATE_FLYDATA_CTL_TABLES_SQL % [tables_tbl]
|
87
110
|
end
|
88
111
|
|
89
112
|
CREATE_TABLE_SQL = <<EOS
|
@@ -126,7 +149,7 @@ EOS
|
|
126
149
|
line = %Q| "#{column[:column]}" #{rs_type}|
|
127
150
|
line += " NOT NULL" if column[:not_null]
|
128
151
|
if (column.has_key?(:default))
|
129
|
-
val = replace_default_value(type_info[:type], column[:default])
|
152
|
+
val = replace_default_value(type, type_info[:type], column[:default])
|
130
153
|
line += " DEFAULT #{val}"
|
131
154
|
elsif column[:not_null] && opt[:for] == :alter_table
|
132
155
|
# Redshift doesn't allow adding a not null column without default value
|
@@ -142,8 +165,16 @@ EOS
|
|
142
165
|
|
143
166
|
NULL_STR = "NULL"
|
144
167
|
|
145
|
-
def self.replace_default_value(redshift_type, default_value)
|
168
|
+
def self.replace_default_value(flydata_type, redshift_type, default_value)
|
146
169
|
return NULL_STR if default_value.nil?
|
170
|
+
if flydata_type.start_with?('year')
|
171
|
+
value = convert_year_into_date(remove_single_quote(default_value))
|
172
|
+
begin
|
173
|
+
Date.parse(value)
|
174
|
+
rescue
|
175
|
+
raise "default value of YEAR type must be 2 or 4-digit, value:'#{default_value}'"
|
176
|
+
end
|
177
|
+
end
|
147
178
|
|
148
179
|
case redshift_type
|
149
180
|
when 'timestamp'
|
@@ -204,21 +235,41 @@ EOS
|
|
204
235
|
sql
|
205
236
|
end
|
206
237
|
|
238
|
+
def self.flydata_ctl_sql(flydata_tabledef, schema_name)
|
239
|
+
flydata_ctl_columns_sql(flydata_tabledef, schema_name) + "\n" +
|
240
|
+
flydata_ctl_tables_sql(flydata_tabledef, schema_name)
|
241
|
+
end
|
207
242
|
FLYDATA_CTL_COLUMNS_SQL = <<EOS
|
208
243
|
DELETE FROM %s WHERE table_name = '%s';
|
209
244
|
INSERT INTO %s (table_name, column_name, src_data_type, ordinal_position) VALUES
|
210
245
|
EOS
|
211
|
-
def self.
|
212
|
-
flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name)
|
246
|
+
def self.flydata_ctl_columns_sql(flydata_tabledef, schema_name)
|
247
|
+
flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
|
213
248
|
sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
|
214
249
|
values = []
|
215
250
|
flydata_tabledef[:columns].each.with_index(1) do |col, i|
|
216
|
-
|
251
|
+
charset = col[:charset] ? " cs:#{col[:charset]}" : ""
|
252
|
+
values << "('#{flydata_tabledef[:table_name]}', '#{col[:column]}', '#{escape(col[:type])}#{charset}', #{i})"
|
217
253
|
end
|
218
254
|
sql += values.join(",\n") + ';'
|
219
255
|
sql
|
220
256
|
end
|
221
257
|
|
258
|
+
FLYDATA_CTL_TABLES_SQL = <<EOS
|
259
|
+
DELETE FROM %s WHERE table_name = '%s';
|
260
|
+
INSERT INTO %s (table_name, attribute, value) VALUES
|
261
|
+
EOS
|
262
|
+
|
263
|
+
def self.flydata_ctl_tables_sql(flydata_tabledef, schema_name)
|
264
|
+
flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
|
265
|
+
sql = FLYDATA_CTL_TABLES_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
|
266
|
+
values = []
|
267
|
+
values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
|
268
|
+
values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
|
269
|
+
sql += values.join(",\n") + ';'
|
270
|
+
sql
|
271
|
+
end
|
272
|
+
|
222
273
|
def self.escape(text)
|
223
274
|
text.gsub("'", "\\\\'")
|
224
275
|
end
|
@@ -65,7 +65,7 @@ describe MysqlTableDef do
|
|
65
65
|
end
|
66
66
|
|
67
67
|
it 'default_charset should be set' do
|
68
|
-
expect(subject[:default_charset]).to eq('
|
68
|
+
expect(subject[:default_charset]).to eq('UTF_8')
|
69
69
|
end
|
70
70
|
|
71
71
|
it 'comment should be set' do
|
@@ -201,6 +201,13 @@ describe MysqlTableDef do
|
|
201
201
|
expect(subject[:columns][2][:default]).to eq("x'10'")
|
202
202
|
end
|
203
203
|
end
|
204
|
+
|
205
|
+
context 'when a column has a character set' do
|
206
|
+
let(:dump_file_io) { file_io('mysqldump_test_column_charset.dump')}
|
207
|
+
it 'should extract charset value for the columns' do
|
208
|
+
expect(subject[:columns][1][:charset]).to eq("ISO_8859_1")
|
209
|
+
end
|
210
|
+
end
|
204
211
|
end
|
205
212
|
|
206
213
|
describe '.convert_to_flydata_type' do
|
@@ -338,6 +345,35 @@ describe MysqlTableDef do
|
|
338
345
|
end
|
339
346
|
end
|
340
347
|
end
|
348
|
+
describe '.convert_to_flydata_value' do
|
349
|
+
subject { MysqlTableDef.convert_to_flydata_value(value, type) }
|
350
|
+
|
351
|
+
let(:value_body) { '0xFE0040' }
|
352
|
+
context 'binary type' do
|
353
|
+
let(:type) { 'binary' }
|
354
|
+
context 'value with padding' do
|
355
|
+
let(:value) { "#{value_body}0000000000" }
|
356
|
+
it 'truncates padding' do
|
357
|
+
is_expected.to eq value_body
|
358
|
+
end
|
359
|
+
end
|
360
|
+
context 'nil value' do
|
361
|
+
let(:value) { nil }
|
362
|
+
it 'returns nil' do
|
363
|
+
is_expected.to eq nil
|
364
|
+
end
|
365
|
+
end
|
366
|
+
end
|
367
|
+
context 'varbinary type' do
|
368
|
+
let(:type) { 'varbinary' }
|
369
|
+
context 'value_with_padding' do
|
370
|
+
let(:value) { "#{value_body}0000000000" }
|
371
|
+
it 'returns the value as is' do
|
372
|
+
is_expected.to eq value
|
373
|
+
end
|
374
|
+
end
|
375
|
+
end
|
376
|
+
end
|
341
377
|
end
|
342
378
|
|
343
379
|
end
|
@@ -36,6 +36,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
36
36
|
ordinal_position int NOT NULL,
|
37
37
|
PRIMARY KEY(id)
|
38
38
|
) DISTKEY(table_name) SORTKEY(table_name);
|
39
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
40
|
+
id integer NOT NULL IDENTITY(1,1),
|
41
|
+
table_name varchar(128) NOT NULL,
|
42
|
+
attribute varchar(128) NOT NULL,
|
43
|
+
value varchar(max),
|
44
|
+
created_at timestamp DEFAULT SYSDATE,
|
45
|
+
PRIMARY KEY(id)
|
46
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
39
47
|
DROP TABLE IF EXISTS "test_table_all";
|
40
48
|
CREATE TABLE "test_table_all" (
|
41
49
|
"id" int8 NOT NULL,
|
@@ -102,6 +110,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
102
110
|
('test_table_all', 'col_year', 'year', 28),
|
103
111
|
('test_table_all', 'col_year_4', 'year(4)', 29),
|
104
112
|
('test_table_all', 'col_year_2', 'year(2)', 30);
|
113
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_all';
|
114
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
115
|
+
('test_table_all', 'cs', 'UTF_8'),
|
116
|
+
('test_table_all', 'revision', 1);
|
105
117
|
EOT
|
106
118
|
end
|
107
119
|
end
|
@@ -120,6 +132,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
120
132
|
ordinal_position int NOT NULL,
|
121
133
|
PRIMARY KEY(id)
|
122
134
|
) DISTKEY(table_name) SORTKEY(table_name);
|
135
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
136
|
+
id integer NOT NULL IDENTITY(1,1),
|
137
|
+
table_name varchar(128) NOT NULL,
|
138
|
+
attribute varchar(128) NOT NULL,
|
139
|
+
value varchar(max),
|
140
|
+
created_at timestamp DEFAULT SYSDATE,
|
141
|
+
PRIMARY KEY(id)
|
142
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
123
143
|
DROP TABLE IF EXISTS "bit_test_def_1";
|
124
144
|
CREATE TABLE "bit_test_def_1" (
|
125
145
|
"id" int4 NOT NULL,
|
@@ -132,6 +152,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
132
152
|
('bit_test_def_1', 'id', 'int4(11)', 1),
|
133
153
|
('bit_test_def_1', 'bit_value', 'bit(1)', 2),
|
134
154
|
('bit_test_def_1', 'int_value', 'int4(11)', 3);
|
155
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'bit_test_def_1';
|
156
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
157
|
+
('bit_test_def_1', 'cs', 'UTF_8'),
|
158
|
+
('bit_test_def_1', 'revision', 1);
|
135
159
|
EOT
|
136
160
|
end
|
137
161
|
end
|
@@ -150,6 +174,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
150
174
|
ordinal_position int NOT NULL,
|
151
175
|
PRIMARY KEY(id)
|
152
176
|
) DISTKEY(table_name) SORTKEY(table_name);
|
177
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
178
|
+
id integer NOT NULL IDENTITY(1,1),
|
179
|
+
table_name varchar(128) NOT NULL,
|
180
|
+
attribute varchar(128) NOT NULL,
|
181
|
+
value varchar(max),
|
182
|
+
created_at timestamp DEFAULT SYSDATE,
|
183
|
+
PRIMARY KEY(id)
|
184
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
153
185
|
DROP TABLE IF EXISTS "product_order";
|
154
186
|
CREATE TABLE "product_order" (
|
155
187
|
"no" int4 NOT NULL,
|
@@ -164,6 +196,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
164
196
|
('product_order', 'product_category', 'int4(11)', 2),
|
165
197
|
('product_order', 'product_id', 'int4(11)', 3),
|
166
198
|
('product_order', 'customer_id', 'int4(11)', 4);
|
199
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'product_order';
|
200
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
201
|
+
('product_order', 'cs', 'UTF_8'),
|
202
|
+
('product_order', 'revision', 1);
|
167
203
|
EOT
|
168
204
|
end
|
169
205
|
end
|
@@ -182,6 +218,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
182
218
|
ordinal_position int NOT NULL,
|
183
219
|
PRIMARY KEY(id)
|
184
220
|
) DISTKEY(table_name) SORTKEY(table_name);
|
221
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
222
|
+
id integer NOT NULL IDENTITY(1,1),
|
223
|
+
table_name varchar(128) NOT NULL,
|
224
|
+
attribute varchar(128) NOT NULL,
|
225
|
+
value varchar(max),
|
226
|
+
created_at timestamp DEFAULT SYSDATE,
|
227
|
+
PRIMARY KEY(id)
|
228
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
185
229
|
DROP TABLE IF EXISTS "test_table_column_comment";
|
186
230
|
CREATE TABLE "test_table_column_comment" (
|
187
231
|
"id" int4 NOT NULL DEFAULT '0',
|
@@ -194,6 +238,10 @@ DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_column_comment'
|
|
194
238
|
INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
|
195
239
|
('test_table_column_comment', 'id', 'int4(11)', 1),
|
196
240
|
('test_table_column_comment', 'value', 'text', 2);
|
241
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_column_comment';
|
242
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
243
|
+
('test_table_column_comment', 'cs', 'UTF_8'),
|
244
|
+
('test_table_column_comment', 'revision', 1);
|
197
245
|
EOT
|
198
246
|
end
|
199
247
|
end
|
@@ -212,6 +260,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
212
260
|
ordinal_position int NOT NULL,
|
213
261
|
PRIMARY KEY(id)
|
214
262
|
) DISTKEY(table_name) SORTKEY(table_name);
|
263
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
264
|
+
id integer NOT NULL IDENTITY(1,1),
|
265
|
+
table_name varchar(128) NOT NULL,
|
266
|
+
attribute varchar(128) NOT NULL,
|
267
|
+
value varchar(max),
|
268
|
+
created_at timestamp DEFAULT SYSDATE,
|
269
|
+
PRIMARY KEY(id)
|
270
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
215
271
|
DROP TABLE IF EXISTS "test_table_enum";
|
216
272
|
CREATE TABLE "test_table_enum" (
|
217
273
|
"id" int4 NOT NULL,
|
@@ -226,6 +282,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
226
282
|
('test_table_enum', 'enum_1', 'enum(\\'apple\\',\\'orange\\',\\'banana\\')', 2),
|
227
283
|
('test_table_enum', 'enum_2', 'enum(\\'a\\',\\'b\\',\\'c\\')', 3),
|
228
284
|
('test_table_enum', 'enum_3', 'enum(\\'e\\',\\'f\\',\\'g\\')', 4);
|
285
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_enum';
|
286
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
287
|
+
('test_table_enum', 'cs', 'UTF_8'),
|
288
|
+
('test_table_enum', 'revision', 1);
|
229
289
|
EOT
|
230
290
|
end
|
231
291
|
end
|
@@ -244,6 +304,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
244
304
|
ordinal_position int NOT NULL,
|
245
305
|
PRIMARY KEY(id)
|
246
306
|
) DISTKEY(table_name) SORTKEY(table_name);
|
307
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
308
|
+
id integer NOT NULL IDENTITY(1,1),
|
309
|
+
table_name varchar(128) NOT NULL,
|
310
|
+
attribute varchar(128) NOT NULL,
|
311
|
+
value varchar(max),
|
312
|
+
created_at timestamp DEFAULT SYSDATE,
|
313
|
+
PRIMARY KEY(id)
|
314
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
247
315
|
DROP TABLE IF EXISTS "test_table_multi_pk";
|
248
316
|
CREATE TABLE "test_table_multi_pk" (
|
249
317
|
"id1" int4 NOT NULL DEFAULT '0',
|
@@ -256,6 +324,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
256
324
|
('test_table_multi_pk', 'id1', 'int4(11)', 1),
|
257
325
|
('test_table_multi_pk', 'id2', 'int4(11)', 2),
|
258
326
|
('test_table_multi_pk', 'value', 'text', 3);
|
327
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_multi_pk';
|
328
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
329
|
+
('test_table_multi_pk', 'cs', 'UTF_8'),
|
330
|
+
('test_table_multi_pk', 'revision', 1);
|
259
331
|
EOT
|
260
332
|
end
|
261
333
|
end
|
@@ -282,6 +354,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
282
354
|
ordinal_position int NOT NULL,
|
283
355
|
PRIMARY KEY(id)
|
284
356
|
) DISTKEY(table_name) SORTKEY(table_name);
|
357
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
358
|
+
id integer NOT NULL IDENTITY(1,1),
|
359
|
+
table_name varchar(128) NOT NULL,
|
360
|
+
attribute varchar(128) NOT NULL,
|
361
|
+
value varchar(max),
|
362
|
+
created_at timestamp DEFAULT SYSDATE,
|
363
|
+
PRIMARY KEY(id)
|
364
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
285
365
|
DROP TABLE IF EXISTS "sample1";
|
286
366
|
CREATE TABLE "sample1" (
|
287
367
|
"id" int4 NOT NULL,
|
@@ -296,6 +376,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
296
376
|
('sample1', 'title', 'varchar(768)', 2),
|
297
377
|
('sample1', 'name', 'text', 3),
|
298
378
|
('sample1', 'num', 'int4(11)', 4);
|
379
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
|
380
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
381
|
+
('sample1', 'cs', 'ISO_8859_1'),
|
382
|
+
('sample1', 'revision', 1);
|
299
383
|
EOT
|
300
384
|
end
|
301
385
|
end
|
@@ -314,6 +398,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
314
398
|
ordinal_position int NOT NULL,
|
315
399
|
PRIMARY KEY(id)
|
316
400
|
) DISTKEY(table_name) SORTKEY(table_name);
|
401
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
402
|
+
id integer NOT NULL IDENTITY(1,1),
|
403
|
+
table_name varchar(128) NOT NULL,
|
404
|
+
attribute varchar(128) NOT NULL,
|
405
|
+
value varchar(max),
|
406
|
+
created_at timestamp DEFAULT SYSDATE,
|
407
|
+
PRIMARY KEY(id)
|
408
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
317
409
|
DROP TABLE IF EXISTS "sample1";
|
318
410
|
CREATE TABLE "sample1" (
|
319
411
|
"id" int4 NOT NULL,
|
@@ -328,6 +420,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
328
420
|
('sample1', 'title', 'varchar(768)', 2),
|
329
421
|
('sample1', 'name', 'text', 3),
|
330
422
|
('sample1', 'num', 'int4(11)', 4);
|
423
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
|
424
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
425
|
+
('sample1', 'cs', 'ISO_8859_1'),
|
426
|
+
('sample1', 'revision', 1);
|
331
427
|
EOT
|
332
428
|
end
|
333
429
|
end
|
@@ -346,6 +442,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
346
442
|
ordinal_position int NOT NULL,
|
347
443
|
PRIMARY KEY(id)
|
348
444
|
) DISTKEY(table_name) SORTKEY(table_name);
|
445
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
446
|
+
id integer NOT NULL IDENTITY(1,1),
|
447
|
+
table_name varchar(128) NOT NULL,
|
448
|
+
attribute varchar(128) NOT NULL,
|
449
|
+
value varchar(max),
|
450
|
+
created_at timestamp DEFAULT SYSDATE,
|
451
|
+
PRIMARY KEY(id)
|
452
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
349
453
|
DROP TABLE IF EXISTS "invoice_items";
|
350
454
|
CREATE TABLE "invoice_items" (
|
351
455
|
"id" int4 NOT NULL,
|
@@ -388,6 +492,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
388
492
|
('invoice_items', 'bill_ahead_resolved_at', 'datetime', 16),
|
389
493
|
('invoice_items', 'stripe_invoice_id', 'varchar(765)', 17),
|
390
494
|
('invoice_items', 'is_vat', 'int1(1)', 18);
|
495
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'invoice_items';
|
496
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
497
|
+
('invoice_items', 'cs', 'UTF_8'),
|
498
|
+
('invoice_items', 'revision', 1);
|
391
499
|
EOT
|
392
500
|
end
|
393
501
|
end
|
@@ -406,6 +514,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
406
514
|
ordinal_position int NOT NULL,
|
407
515
|
PRIMARY KEY(id)
|
408
516
|
) DISTKEY(table_name) SORTKEY(table_name);
|
517
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
518
|
+
id integer NOT NULL IDENTITY(1,1),
|
519
|
+
table_name varchar(128) NOT NULL,
|
520
|
+
attribute varchar(128) NOT NULL,
|
521
|
+
value varchar(max),
|
522
|
+
created_at timestamp DEFAULT SYSDATE,
|
523
|
+
PRIMARY KEY(id)
|
524
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
409
525
|
DROP TABLE IF EXISTS "zerofill_table";
|
410
526
|
CREATE TABLE "zerofill_table" (
|
411
527
|
"id" int4 NOT NULL,
|
@@ -426,6 +542,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
426
542
|
('zerofill_table', 'value_double', 'float8 unsigned', 5),
|
427
543
|
('zerofill_table', 'name', 'varchar(768)', 6),
|
428
544
|
('zerofill_table', 'value_small_int', 'int2(5) unsigned', 7);
|
545
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'zerofill_table';
|
546
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
547
|
+
('zerofill_table', 'cs', 'ISO_8859_1'),
|
548
|
+
('zerofill_table', 'revision', 1);
|
429
549
|
EOT
|
430
550
|
end
|
431
551
|
end
|