flydata 0.3.16 → 0.3.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/record/record.rb +13 -0
- data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +107 -5
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +62 -11
- data/flydata-core/spec/table_def/mysql_table_def_spec.rb +37 -1
- data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +120 -0
- data/flydata-core/spec/table_def/mysqldump_test_column_charset.dump +45 -0
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +70 -88
- data/flydata.gemspec +13 -8
- data/lib/flydata/command/setup.rb +4 -4
- data/lib/flydata/command/sync.rb +18 -29
- data/lib/flydata/compatibility_check.rb +2 -2
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +15 -10
- data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -3
- data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb +15 -8
- data/lib/flydata/fluent-plugins/mysql/table_meta.rb +6 -34
- data/lib/flydata/{fluent-plugins/mysql → mysql}/binlog_position.rb +2 -0
- data/lib/flydata/{util → mysql}/mysql_util.rb +34 -1
- data/lib/flydata/mysql/table_ddl.rb +118 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +5 -5
- data/lib/flydata/parser/mysql/mysql_alter_table.treetop +29 -5
- data/lib/flydata/sync_file_manager.rb +15 -2
- data/spec/flydata/command/sync_spec.rb +22 -1
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +7 -2
- data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +130 -0
- data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +1 -0
- data/spec/flydata/fluent-plugins/mysql/table_meta_spec.rb +14 -8
- data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +2 -1
- data/spec/flydata/{fluent-plugins/mysql → mysql}/binlog_position_spec.rb +3 -2
- data/spec/flydata/{util → mysql}/mysql_util_spec.rb +2 -2
- data/spec/flydata/mysql/table_ddl_spec.rb +193 -0
- data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +37 -9
- data/spec/flydata/sync_file_manager_spec.rb +102 -27
- metadata +12 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 15226b89faab2ec73d834294f47e4b2f8fc35b69
|
4
|
+
data.tar.gz: 19d207fcea4121d051cb73f536c04769c95cfbc0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f67502dcee392df1f1016346390266e5e977450359cec43ca0a20402707a28b84547b76ea06063ef9c50de510652a8cc362ae914087e37ace2fc9c2d54384581
|
7
|
+
data.tar.gz: a2dbad269e6e2e7eb0e2042c4b05a51441cb92b3f6cca58cc2fdd47b922a0a09eb702d22882e64557757dd93b809cd2552ecbc81f45c27673b75ee16317b3d19
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.17
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module FlydataCore
|
2
|
+
module Record
|
3
|
+
|
4
|
+
V1 = 1
|
5
|
+
# - Handles string/binary as UTF-8
|
6
|
+
# - No "v" peroperty
|
7
|
+
|
8
|
+
V2 = 2
|
9
|
+
# - Supports the common string/binary format
|
10
|
+
# - "v": 2
|
11
|
+
# - additions of "attrs" property. "enc":"b" is currently supported.
|
12
|
+
end
|
13
|
+
end
|
@@ -87,11 +87,14 @@ class MysqlTableDef
|
|
87
87
|
params ? self.new(*params) : nil
|
88
88
|
end
|
89
89
|
|
90
|
-
def initialize(table_def, table_name, columns, default_charset,
|
90
|
+
def initialize(table_def, table_name, columns, column_def, default_charset,
|
91
|
+
default_charset_mysql, comment)
|
91
92
|
@table_def = table_def
|
92
93
|
@table_name = table_name
|
93
94
|
@columns = columns
|
95
|
+
@column_def = column_def
|
94
96
|
@default_charset = default_charset
|
97
|
+
@default_charset_mysql = default_charset_mysql
|
95
98
|
@comment = comment
|
96
99
|
end
|
97
100
|
|
@@ -99,7 +102,9 @@ class MysqlTableDef
|
|
99
102
|
table_def = ''
|
100
103
|
table_name = nil
|
101
104
|
columns = []
|
105
|
+
column_def = {}
|
102
106
|
default_charset = nil
|
107
|
+
default_charset_mysql = nil
|
103
108
|
comment = nil
|
104
109
|
|
105
110
|
position = :before_create_table
|
@@ -120,13 +125,19 @@ class MysqlTableDef
|
|
120
125
|
stripped_line = line.strip
|
121
126
|
# `col_smallint` smallint(6) DEFAULT NULL,
|
122
127
|
if stripped_line.start_with?('`')
|
123
|
-
|
128
|
+
column = parse_one_column_def(line)
|
129
|
+
columns << column
|
130
|
+
coldef = line.strip.gsub(/,$/, '')
|
131
|
+
column_def[column[:column]] = coldef
|
124
132
|
# PRIMARY KEY (`id`)
|
125
133
|
elsif stripped_line.start_with?("PRIMARY KEY")
|
126
134
|
parse_key(line, columns)
|
127
135
|
#) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table';
|
128
136
|
elsif stripped_line.start_with?(')')
|
129
|
-
|
137
|
+
if line =~ /DEFAULT CHARSET\s*=\s*([a-z0-9]+)/
|
138
|
+
default_charset_mysql = $1
|
139
|
+
default_charset = flydata_charset(default_charset_mysql)
|
140
|
+
end
|
130
141
|
comment = $1 if /COMMENT='((?:\\'|[^'])*)'/.match(line)
|
131
142
|
position = :after_create_table
|
132
143
|
elsif stripped_line.start_with?("KEY")
|
@@ -146,9 +157,9 @@ class MysqlTableDef
|
|
146
157
|
break
|
147
158
|
end
|
148
159
|
end
|
149
|
-
position == :after_create_table ? [table_def, table_name, columns, default_charset, comment] : nil
|
160
|
+
position == :after_create_table ? [table_def, table_name, columns, column_def, default_charset, default_charset_mysql, comment] : nil
|
150
161
|
end
|
151
|
-
attr_reader :columns, :table_name
|
162
|
+
attr_reader :columns, :column_def, :table_name, :default_charset_mysql
|
152
163
|
|
153
164
|
def to_flydata_tabledef
|
154
165
|
tabledef = { table_name: @table_name,
|
@@ -207,6 +218,9 @@ class MysqlTableDef
|
|
207
218
|
if /COMMENT\s+'(((?:\\'|[^'])*))'/i.match(line)
|
208
219
|
column[:comment] = $1
|
209
220
|
end
|
221
|
+
if /CHARACTER SET\s+([a-z0-9]+)/i.match(line)
|
222
|
+
column[:charset] = flydata_charset($1)
|
223
|
+
end
|
210
224
|
if block_given?
|
211
225
|
column = yield(column, query, pos)
|
212
226
|
end
|
@@ -218,6 +232,93 @@ class MysqlTableDef
|
|
218
232
|
column
|
219
233
|
end
|
220
234
|
|
235
|
+
# 'binary' is a fixed length column type. mysqldump exports a binary column
|
236
|
+
# value at its full length by adding trailing 0's to the size of the column
|
237
|
+
# (ex. '0xAB00920000' for type 'binary(5)')
|
238
|
+
# FlyData Transport Format does not allow such paddings so they need to be
|
239
|
+
# removed. On the target database, padding may be added according to the
|
240
|
+
# destination DB's convention.
|
241
|
+
MYSQL_BINARY_PADDING_CHAR = '00'
|
242
|
+
PROC_convert_binary_value = -> (value) do
|
243
|
+
return nil unless value
|
244
|
+
while value.end_with?(MYSQL_BINARY_PADDING_CHAR)
|
245
|
+
value = value[0...-MYSQL_BINARY_PADDING_CHAR.size]
|
246
|
+
end
|
247
|
+
value
|
248
|
+
end
|
249
|
+
|
250
|
+
VALUE_CONVERTERS = {
|
251
|
+
'binary' => PROC_convert_binary_value,
|
252
|
+
}
|
253
|
+
|
254
|
+
def self.convert_to_flydata_value(value, type)
|
255
|
+
if converter = VALUE_CONVERTERS[type]
|
256
|
+
value = converter.call(value)
|
257
|
+
end
|
258
|
+
value
|
259
|
+
end
|
260
|
+
|
261
|
+
# Charset naming conversion rule. mysql => ruby
|
262
|
+
#
|
263
|
+
# mysql
|
264
|
+
# http://dev.mysql.com/doc/refman/5.6/en/charset-charsets.html
|
265
|
+
# mysql(supported CJK character sets)
|
266
|
+
# http://dev.mysql.com/doc/refman/5.6/en/faqs-cjk.html#qandaitem-A-11-1-1
|
267
|
+
# For ruby, you can see encoding list with "Encoding.list"
|
268
|
+
CHARSET_ENCODE_RULE = {
|
269
|
+
"armscii8" => nil,
|
270
|
+
"ascii" => Encoding::US_ASCII,
|
271
|
+
"big5" => Encoding::Big5,
|
272
|
+
"binary" => Encoding::ASCII_8BIT,
|
273
|
+
"cp1250" => Encoding::Windows_1250,
|
274
|
+
"cp1251" => Encoding::Windows_1251,
|
275
|
+
"cp1256" => Encoding::Windows_1256,
|
276
|
+
"cp1257" => Encoding::Windows_1257,
|
277
|
+
"cp850" => Encoding::CP850,
|
278
|
+
"cp852" => Encoding::CP852,
|
279
|
+
"cp866" => Encoding::IBM866,
|
280
|
+
"cp932" => Encoding::Windows_31J,
|
281
|
+
"dec8" => nil,
|
282
|
+
"eucjpms" => Encoding::EucJP_ms,
|
283
|
+
"euckr" => Encoding::EUC_KR,
|
284
|
+
"gb2312" => Encoding::EUC_CN,
|
285
|
+
"gbk" => Encoding::GBK,
|
286
|
+
"geostd8" => nil,
|
287
|
+
"greek" => Encoding::ISO_8859_7,
|
288
|
+
"hebrew" => Encoding::ISO_8859_8,
|
289
|
+
"hp8" => nil,
|
290
|
+
"keybcs2" => nil,
|
291
|
+
"koi8r" => Encoding::KOI8_R,
|
292
|
+
"koi8u" => Encoding::KOI8_U,
|
293
|
+
"latin1" => Encoding::ISO_8859_1,
|
294
|
+
"latin2" => Encoding::ISO_8859_2,
|
295
|
+
"latin5" => Encoding::ISO_8859_9,
|
296
|
+
"latin7" => Encoding::ISO_8859_13,
|
297
|
+
"macce" => Encoding::MacCentEuro,
|
298
|
+
"macroman" => Encoding::MacRoman,
|
299
|
+
"sjis" => Encoding::SHIFT_JIS,
|
300
|
+
"swe7" => nil,
|
301
|
+
"tis620" => Encoding::TIS_620,
|
302
|
+
"ucs2" => Encoding::UTF_16BE,
|
303
|
+
"ujis" => Encoding::EUC_JP,
|
304
|
+
"utf16" => Encoding::UTF_16,
|
305
|
+
"utf16le" => Encoding::UTF_16LE,
|
306
|
+
"utf32" => Encoding::UTF_32,
|
307
|
+
"utf8" => Encoding::UTF_8,
|
308
|
+
"utf8mb4" => Encoding::UTF_8,
|
309
|
+
}
|
310
|
+
|
311
|
+
def self.ruby_encoding(mysql_charset)
|
312
|
+
return nil if mysql_charset.to_s.empty?
|
313
|
+
raise "Unsupported charset:#{mysql_charset}." unless CHARSET_ENCODE_RULE.has_key?(mysql_charset)
|
314
|
+
charset = CHARSET_ENCODE_RULE[mysql_charset]
|
315
|
+
return charset
|
316
|
+
end
|
317
|
+
|
318
|
+
def self.flydata_charset(mysql_charset)
|
319
|
+
ruby_encoding(mysql_charset).to_s.upcase.gsub('-', '_')
|
320
|
+
end
|
321
|
+
|
221
322
|
private
|
222
323
|
|
223
324
|
def self.parse_key(line, columns, type = :primary_key)
|
@@ -232,6 +333,7 @@ class MysqlTableDef
|
|
232
333
|
column[type] = true
|
233
334
|
end
|
234
335
|
end
|
336
|
+
|
235
337
|
end
|
236
338
|
|
237
339
|
end
|
@@ -53,10 +53,11 @@ class RedshiftTableDef
|
|
53
53
|
end
|
54
54
|
|
55
55
|
FLYDATA_CTL_COLUMNS_TABLE = "flydata_ctl_columns"
|
56
|
+
FLYDATA_CTL_TABLES_TABLE = "flydata_ctl_tables"
|
56
57
|
CREATE_SCHEMA_SQL = <<EOS
|
57
|
-
CREATE SCHEMA "%s";
|
58
|
+
CREATE SCHEMA IF NOT EXISTS "%s";
|
58
59
|
EOS
|
59
|
-
|
60
|
+
CREATE_FLYDATA_CTL_COLUMNS_SQL = <<EOS
|
60
61
|
CREATE TABLE IF NOT EXISTS %s(
|
61
62
|
id integer NOT NULL IDENTITY(1,1),
|
62
63
|
table_name varchar(128) NOT NULL,
|
@@ -67,13 +68,29 @@ CREATE TABLE IF NOT EXISTS %s(
|
|
67
68
|
PRIMARY KEY(id)
|
68
69
|
) DISTKEY(table_name) SORTKEY(table_name);
|
69
70
|
EOS
|
71
|
+
CREATE_FLYDATA_CTL_TABLES_SQL = <<EOS
|
72
|
+
CREATE TABLE IF NOT EXISTS %s(
|
73
|
+
id integer NOT NULL IDENTITY(1,1),
|
74
|
+
table_name varchar(128) NOT NULL,
|
75
|
+
attribute varchar(128) NOT NULL,
|
76
|
+
value varchar(max),
|
77
|
+
created_at timestamp DEFAULT SYSDATE,
|
78
|
+
PRIMARY KEY(id)
|
79
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
80
|
+
EOS
|
81
|
+
CREATE_FLYDATA_CTL_TABLE_SQL = "#{CREATE_FLYDATA_CTL_COLUMNS_SQL}#{CREATE_FLYDATA_CTL_TABLES_SQL}"
|
82
|
+
|
70
83
|
|
71
84
|
def self.table_name_for_ddl(table_name, schema_name)
|
72
85
|
schema_name.to_s.empty? ? "\"#{table_name}\"" : "\"#{schema_name}\".\"#{table_name}\""
|
73
86
|
end
|
74
87
|
|
75
|
-
def self.flydata_ctl_table_for_ddl(schema_name)
|
76
|
-
|
88
|
+
def self.flydata_ctl_table_for_ddl(schema_name, ctl_table_type = :columns)
|
89
|
+
table_name = case ctl_table_type
|
90
|
+
when :columns; FLYDATA_CTL_COLUMNS_TABLE
|
91
|
+
when :tables; FLYDATA_CTL_TABLES_TABLE
|
92
|
+
end
|
93
|
+
table_name_for_ddl(table_name, schema_name)
|
77
94
|
end
|
78
95
|
|
79
96
|
def self.create_schema_sql(schema_name)
|
@@ -82,8 +99,14 @@ EOS
|
|
82
99
|
|
83
100
|
def self.create_flydata_ctl_table_sql(schema_name)
|
84
101
|
# No drop table here intentionally because losing the data is fatal.
|
85
|
-
|
86
|
-
|
102
|
+
columns_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
|
103
|
+
tables_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
|
104
|
+
CREATE_FLYDATA_CTL_TABLE_SQL % [columns_tbl, tables_tbl]
|
105
|
+
end
|
106
|
+
|
107
|
+
def self.create_flydata_ctl_tables_sql(schema_name)
|
108
|
+
tables_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
|
109
|
+
CREATE_FLYDATA_CTL_TABLES_SQL % [tables_tbl]
|
87
110
|
end
|
88
111
|
|
89
112
|
CREATE_TABLE_SQL = <<EOS
|
@@ -126,7 +149,7 @@ EOS
|
|
126
149
|
line = %Q| "#{column[:column]}" #{rs_type}|
|
127
150
|
line += " NOT NULL" if column[:not_null]
|
128
151
|
if (column.has_key?(:default))
|
129
|
-
val = replace_default_value(type_info[:type], column[:default])
|
152
|
+
val = replace_default_value(type, type_info[:type], column[:default])
|
130
153
|
line += " DEFAULT #{val}"
|
131
154
|
elsif column[:not_null] && opt[:for] == :alter_table
|
132
155
|
# Redshift doesn't allow adding a not null column without default value
|
@@ -142,8 +165,16 @@ EOS
|
|
142
165
|
|
143
166
|
NULL_STR = "NULL"
|
144
167
|
|
145
|
-
def self.replace_default_value(redshift_type, default_value)
|
168
|
+
def self.replace_default_value(flydata_type, redshift_type, default_value)
|
146
169
|
return NULL_STR if default_value.nil?
|
170
|
+
if flydata_type.start_with?('year')
|
171
|
+
value = convert_year_into_date(remove_single_quote(default_value))
|
172
|
+
begin
|
173
|
+
Date.parse(value)
|
174
|
+
rescue
|
175
|
+
raise "default value of YEAR type must be 2 or 4-digit, value:'#{default_value}'"
|
176
|
+
end
|
177
|
+
end
|
147
178
|
|
148
179
|
case redshift_type
|
149
180
|
when 'timestamp'
|
@@ -204,21 +235,41 @@ EOS
|
|
204
235
|
sql
|
205
236
|
end
|
206
237
|
|
238
|
+
def self.flydata_ctl_sql(flydata_tabledef, schema_name)
|
239
|
+
flydata_ctl_columns_sql(flydata_tabledef, schema_name) + "\n" +
|
240
|
+
flydata_ctl_tables_sql(flydata_tabledef, schema_name)
|
241
|
+
end
|
207
242
|
FLYDATA_CTL_COLUMNS_SQL = <<EOS
|
208
243
|
DELETE FROM %s WHERE table_name = '%s';
|
209
244
|
INSERT INTO %s (table_name, column_name, src_data_type, ordinal_position) VALUES
|
210
245
|
EOS
|
211
|
-
def self.
|
212
|
-
flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name)
|
246
|
+
def self.flydata_ctl_columns_sql(flydata_tabledef, schema_name)
|
247
|
+
flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
|
213
248
|
sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
|
214
249
|
values = []
|
215
250
|
flydata_tabledef[:columns].each.with_index(1) do |col, i|
|
216
|
-
|
251
|
+
charset = col[:charset] ? " cs:#{col[:charset]}" : ""
|
252
|
+
values << "('#{flydata_tabledef[:table_name]}', '#{col[:column]}', '#{escape(col[:type])}#{charset}', #{i})"
|
217
253
|
end
|
218
254
|
sql += values.join(",\n") + ';'
|
219
255
|
sql
|
220
256
|
end
|
221
257
|
|
258
|
+
FLYDATA_CTL_TABLES_SQL = <<EOS
|
259
|
+
DELETE FROM %s WHERE table_name = '%s';
|
260
|
+
INSERT INTO %s (table_name, attribute, value) VALUES
|
261
|
+
EOS
|
262
|
+
|
263
|
+
def self.flydata_ctl_tables_sql(flydata_tabledef, schema_name)
|
264
|
+
flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
|
265
|
+
sql = FLYDATA_CTL_TABLES_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
|
266
|
+
values = []
|
267
|
+
values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
|
268
|
+
values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
|
269
|
+
sql += values.join(",\n") + ';'
|
270
|
+
sql
|
271
|
+
end
|
272
|
+
|
222
273
|
def self.escape(text)
|
223
274
|
text.gsub("'", "\\\\'")
|
224
275
|
end
|
@@ -65,7 +65,7 @@ describe MysqlTableDef do
|
|
65
65
|
end
|
66
66
|
|
67
67
|
it 'default_charset should be set' do
|
68
|
-
expect(subject[:default_charset]).to eq('
|
68
|
+
expect(subject[:default_charset]).to eq('UTF_8')
|
69
69
|
end
|
70
70
|
|
71
71
|
it 'comment should be set' do
|
@@ -201,6 +201,13 @@ describe MysqlTableDef do
|
|
201
201
|
expect(subject[:columns][2][:default]).to eq("x'10'")
|
202
202
|
end
|
203
203
|
end
|
204
|
+
|
205
|
+
context 'when a column has a character set' do
|
206
|
+
let(:dump_file_io) { file_io('mysqldump_test_column_charset.dump')}
|
207
|
+
it 'should extract charset value for the columns' do
|
208
|
+
expect(subject[:columns][1][:charset]).to eq("ISO_8859_1")
|
209
|
+
end
|
210
|
+
end
|
204
211
|
end
|
205
212
|
|
206
213
|
describe '.convert_to_flydata_type' do
|
@@ -338,6 +345,35 @@ describe MysqlTableDef do
|
|
338
345
|
end
|
339
346
|
end
|
340
347
|
end
|
348
|
+
describe '.convert_to_flydata_value' do
|
349
|
+
subject { MysqlTableDef.convert_to_flydata_value(value, type) }
|
350
|
+
|
351
|
+
let(:value_body) { '0xFE0040' }
|
352
|
+
context 'binary type' do
|
353
|
+
let(:type) { 'binary' }
|
354
|
+
context 'value with padding' do
|
355
|
+
let(:value) { "#{value_body}0000000000" }
|
356
|
+
it 'truncates padding' do
|
357
|
+
is_expected.to eq value_body
|
358
|
+
end
|
359
|
+
end
|
360
|
+
context 'nil value' do
|
361
|
+
let(:value) { nil }
|
362
|
+
it 'returns nil' do
|
363
|
+
is_expected.to eq nil
|
364
|
+
end
|
365
|
+
end
|
366
|
+
end
|
367
|
+
context 'varbinary type' do
|
368
|
+
let(:type) { 'varbinary' }
|
369
|
+
context 'value_with_padding' do
|
370
|
+
let(:value) { "#{value_body}0000000000" }
|
371
|
+
it 'returns the value as is' do
|
372
|
+
is_expected.to eq value
|
373
|
+
end
|
374
|
+
end
|
375
|
+
end
|
376
|
+
end
|
341
377
|
end
|
342
378
|
|
343
379
|
end
|
@@ -36,6 +36,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
36
36
|
ordinal_position int NOT NULL,
|
37
37
|
PRIMARY KEY(id)
|
38
38
|
) DISTKEY(table_name) SORTKEY(table_name);
|
39
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
40
|
+
id integer NOT NULL IDENTITY(1,1),
|
41
|
+
table_name varchar(128) NOT NULL,
|
42
|
+
attribute varchar(128) NOT NULL,
|
43
|
+
value varchar(max),
|
44
|
+
created_at timestamp DEFAULT SYSDATE,
|
45
|
+
PRIMARY KEY(id)
|
46
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
39
47
|
DROP TABLE IF EXISTS "test_table_all";
|
40
48
|
CREATE TABLE "test_table_all" (
|
41
49
|
"id" int8 NOT NULL,
|
@@ -102,6 +110,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
102
110
|
('test_table_all', 'col_year', 'year', 28),
|
103
111
|
('test_table_all', 'col_year_4', 'year(4)', 29),
|
104
112
|
('test_table_all', 'col_year_2', 'year(2)', 30);
|
113
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_all';
|
114
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
115
|
+
('test_table_all', 'cs', 'UTF_8'),
|
116
|
+
('test_table_all', 'revision', 1);
|
105
117
|
EOT
|
106
118
|
end
|
107
119
|
end
|
@@ -120,6 +132,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
120
132
|
ordinal_position int NOT NULL,
|
121
133
|
PRIMARY KEY(id)
|
122
134
|
) DISTKEY(table_name) SORTKEY(table_name);
|
135
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
136
|
+
id integer NOT NULL IDENTITY(1,1),
|
137
|
+
table_name varchar(128) NOT NULL,
|
138
|
+
attribute varchar(128) NOT NULL,
|
139
|
+
value varchar(max),
|
140
|
+
created_at timestamp DEFAULT SYSDATE,
|
141
|
+
PRIMARY KEY(id)
|
142
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
123
143
|
DROP TABLE IF EXISTS "bit_test_def_1";
|
124
144
|
CREATE TABLE "bit_test_def_1" (
|
125
145
|
"id" int4 NOT NULL,
|
@@ -132,6 +152,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
132
152
|
('bit_test_def_1', 'id', 'int4(11)', 1),
|
133
153
|
('bit_test_def_1', 'bit_value', 'bit(1)', 2),
|
134
154
|
('bit_test_def_1', 'int_value', 'int4(11)', 3);
|
155
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'bit_test_def_1';
|
156
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
157
|
+
('bit_test_def_1', 'cs', 'UTF_8'),
|
158
|
+
('bit_test_def_1', 'revision', 1);
|
135
159
|
EOT
|
136
160
|
end
|
137
161
|
end
|
@@ -150,6 +174,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
150
174
|
ordinal_position int NOT NULL,
|
151
175
|
PRIMARY KEY(id)
|
152
176
|
) DISTKEY(table_name) SORTKEY(table_name);
|
177
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
178
|
+
id integer NOT NULL IDENTITY(1,1),
|
179
|
+
table_name varchar(128) NOT NULL,
|
180
|
+
attribute varchar(128) NOT NULL,
|
181
|
+
value varchar(max),
|
182
|
+
created_at timestamp DEFAULT SYSDATE,
|
183
|
+
PRIMARY KEY(id)
|
184
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
153
185
|
DROP TABLE IF EXISTS "product_order";
|
154
186
|
CREATE TABLE "product_order" (
|
155
187
|
"no" int4 NOT NULL,
|
@@ -164,6 +196,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
164
196
|
('product_order', 'product_category', 'int4(11)', 2),
|
165
197
|
('product_order', 'product_id', 'int4(11)', 3),
|
166
198
|
('product_order', 'customer_id', 'int4(11)', 4);
|
199
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'product_order';
|
200
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
201
|
+
('product_order', 'cs', 'UTF_8'),
|
202
|
+
('product_order', 'revision', 1);
|
167
203
|
EOT
|
168
204
|
end
|
169
205
|
end
|
@@ -182,6 +218,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
182
218
|
ordinal_position int NOT NULL,
|
183
219
|
PRIMARY KEY(id)
|
184
220
|
) DISTKEY(table_name) SORTKEY(table_name);
|
221
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
222
|
+
id integer NOT NULL IDENTITY(1,1),
|
223
|
+
table_name varchar(128) NOT NULL,
|
224
|
+
attribute varchar(128) NOT NULL,
|
225
|
+
value varchar(max),
|
226
|
+
created_at timestamp DEFAULT SYSDATE,
|
227
|
+
PRIMARY KEY(id)
|
228
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
185
229
|
DROP TABLE IF EXISTS "test_table_column_comment";
|
186
230
|
CREATE TABLE "test_table_column_comment" (
|
187
231
|
"id" int4 NOT NULL DEFAULT '0',
|
@@ -194,6 +238,10 @@ DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_column_comment'
|
|
194
238
|
INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
|
195
239
|
('test_table_column_comment', 'id', 'int4(11)', 1),
|
196
240
|
('test_table_column_comment', 'value', 'text', 2);
|
241
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_column_comment';
|
242
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
243
|
+
('test_table_column_comment', 'cs', 'UTF_8'),
|
244
|
+
('test_table_column_comment', 'revision', 1);
|
197
245
|
EOT
|
198
246
|
end
|
199
247
|
end
|
@@ -212,6 +260,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
212
260
|
ordinal_position int NOT NULL,
|
213
261
|
PRIMARY KEY(id)
|
214
262
|
) DISTKEY(table_name) SORTKEY(table_name);
|
263
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
264
|
+
id integer NOT NULL IDENTITY(1,1),
|
265
|
+
table_name varchar(128) NOT NULL,
|
266
|
+
attribute varchar(128) NOT NULL,
|
267
|
+
value varchar(max),
|
268
|
+
created_at timestamp DEFAULT SYSDATE,
|
269
|
+
PRIMARY KEY(id)
|
270
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
215
271
|
DROP TABLE IF EXISTS "test_table_enum";
|
216
272
|
CREATE TABLE "test_table_enum" (
|
217
273
|
"id" int4 NOT NULL,
|
@@ -226,6 +282,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
226
282
|
('test_table_enum', 'enum_1', 'enum(\\'apple\\',\\'orange\\',\\'banana\\')', 2),
|
227
283
|
('test_table_enum', 'enum_2', 'enum(\\'a\\',\\'b\\',\\'c\\')', 3),
|
228
284
|
('test_table_enum', 'enum_3', 'enum(\\'e\\',\\'f\\',\\'g\\')', 4);
|
285
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_enum';
|
286
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
287
|
+
('test_table_enum', 'cs', 'UTF_8'),
|
288
|
+
('test_table_enum', 'revision', 1);
|
229
289
|
EOT
|
230
290
|
end
|
231
291
|
end
|
@@ -244,6 +304,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
244
304
|
ordinal_position int NOT NULL,
|
245
305
|
PRIMARY KEY(id)
|
246
306
|
) DISTKEY(table_name) SORTKEY(table_name);
|
307
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
308
|
+
id integer NOT NULL IDENTITY(1,1),
|
309
|
+
table_name varchar(128) NOT NULL,
|
310
|
+
attribute varchar(128) NOT NULL,
|
311
|
+
value varchar(max),
|
312
|
+
created_at timestamp DEFAULT SYSDATE,
|
313
|
+
PRIMARY KEY(id)
|
314
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
247
315
|
DROP TABLE IF EXISTS "test_table_multi_pk";
|
248
316
|
CREATE TABLE "test_table_multi_pk" (
|
249
317
|
"id1" int4 NOT NULL DEFAULT '0',
|
@@ -256,6 +324,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
256
324
|
('test_table_multi_pk', 'id1', 'int4(11)', 1),
|
257
325
|
('test_table_multi_pk', 'id2', 'int4(11)', 2),
|
258
326
|
('test_table_multi_pk', 'value', 'text', 3);
|
327
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_multi_pk';
|
328
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
329
|
+
('test_table_multi_pk', 'cs', 'UTF_8'),
|
330
|
+
('test_table_multi_pk', 'revision', 1);
|
259
331
|
EOT
|
260
332
|
end
|
261
333
|
end
|
@@ -282,6 +354,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
282
354
|
ordinal_position int NOT NULL,
|
283
355
|
PRIMARY KEY(id)
|
284
356
|
) DISTKEY(table_name) SORTKEY(table_name);
|
357
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
358
|
+
id integer NOT NULL IDENTITY(1,1),
|
359
|
+
table_name varchar(128) NOT NULL,
|
360
|
+
attribute varchar(128) NOT NULL,
|
361
|
+
value varchar(max),
|
362
|
+
created_at timestamp DEFAULT SYSDATE,
|
363
|
+
PRIMARY KEY(id)
|
364
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
285
365
|
DROP TABLE IF EXISTS "sample1";
|
286
366
|
CREATE TABLE "sample1" (
|
287
367
|
"id" int4 NOT NULL,
|
@@ -296,6 +376,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
296
376
|
('sample1', 'title', 'varchar(768)', 2),
|
297
377
|
('sample1', 'name', 'text', 3),
|
298
378
|
('sample1', 'num', 'int4(11)', 4);
|
379
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
|
380
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
381
|
+
('sample1', 'cs', 'ISO_8859_1'),
|
382
|
+
('sample1', 'revision', 1);
|
299
383
|
EOT
|
300
384
|
end
|
301
385
|
end
|
@@ -314,6 +398,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
314
398
|
ordinal_position int NOT NULL,
|
315
399
|
PRIMARY KEY(id)
|
316
400
|
) DISTKEY(table_name) SORTKEY(table_name);
|
401
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
402
|
+
id integer NOT NULL IDENTITY(1,1),
|
403
|
+
table_name varchar(128) NOT NULL,
|
404
|
+
attribute varchar(128) NOT NULL,
|
405
|
+
value varchar(max),
|
406
|
+
created_at timestamp DEFAULT SYSDATE,
|
407
|
+
PRIMARY KEY(id)
|
408
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
317
409
|
DROP TABLE IF EXISTS "sample1";
|
318
410
|
CREATE TABLE "sample1" (
|
319
411
|
"id" int4 NOT NULL,
|
@@ -328,6 +420,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
328
420
|
('sample1', 'title', 'varchar(768)', 2),
|
329
421
|
('sample1', 'name', 'text', 3),
|
330
422
|
('sample1', 'num', 'int4(11)', 4);
|
423
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
|
424
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
425
|
+
('sample1', 'cs', 'ISO_8859_1'),
|
426
|
+
('sample1', 'revision', 1);
|
331
427
|
EOT
|
332
428
|
end
|
333
429
|
end
|
@@ -346,6 +442,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
346
442
|
ordinal_position int NOT NULL,
|
347
443
|
PRIMARY KEY(id)
|
348
444
|
) DISTKEY(table_name) SORTKEY(table_name);
|
445
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
446
|
+
id integer NOT NULL IDENTITY(1,1),
|
447
|
+
table_name varchar(128) NOT NULL,
|
448
|
+
attribute varchar(128) NOT NULL,
|
449
|
+
value varchar(max),
|
450
|
+
created_at timestamp DEFAULT SYSDATE,
|
451
|
+
PRIMARY KEY(id)
|
452
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
349
453
|
DROP TABLE IF EXISTS "invoice_items";
|
350
454
|
CREATE TABLE "invoice_items" (
|
351
455
|
"id" int4 NOT NULL,
|
@@ -388,6 +492,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
388
492
|
('invoice_items', 'bill_ahead_resolved_at', 'datetime', 16),
|
389
493
|
('invoice_items', 'stripe_invoice_id', 'varchar(765)', 17),
|
390
494
|
('invoice_items', 'is_vat', 'int1(1)', 18);
|
495
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'invoice_items';
|
496
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
497
|
+
('invoice_items', 'cs', 'UTF_8'),
|
498
|
+
('invoice_items', 'revision', 1);
|
391
499
|
EOT
|
392
500
|
end
|
393
501
|
end
|
@@ -406,6 +514,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
|
|
406
514
|
ordinal_position int NOT NULL,
|
407
515
|
PRIMARY KEY(id)
|
408
516
|
) DISTKEY(table_name) SORTKEY(table_name);
|
517
|
+
CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
518
|
+
id integer NOT NULL IDENTITY(1,1),
|
519
|
+
table_name varchar(128) NOT NULL,
|
520
|
+
attribute varchar(128) NOT NULL,
|
521
|
+
value varchar(max),
|
522
|
+
created_at timestamp DEFAULT SYSDATE,
|
523
|
+
PRIMARY KEY(id)
|
524
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
409
525
|
DROP TABLE IF EXISTS "zerofill_table";
|
410
526
|
CREATE TABLE "zerofill_table" (
|
411
527
|
"id" int4 NOT NULL,
|
@@ -426,6 +542,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
|
|
426
542
|
('zerofill_table', 'value_double', 'float8 unsigned', 5),
|
427
543
|
('zerofill_table', 'name', 'varchar(768)', 6),
|
428
544
|
('zerofill_table', 'value_small_int', 'int2(5) unsigned', 7);
|
545
|
+
DELETE FROM "flydata_ctl_tables" WHERE table_name = 'zerofill_table';
|
546
|
+
INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
|
547
|
+
('zerofill_table', 'cs', 'ISO_8859_1'),
|
548
|
+
('zerofill_table', 'revision', 1);
|
429
549
|
EOT
|
430
550
|
end
|
431
551
|
end
|