flydata 0.3.16 → 0.3.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/flydata-core/lib/flydata-core/record/record.rb +13 -0
  4. data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +107 -5
  5. data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +62 -11
  6. data/flydata-core/spec/table_def/mysql_table_def_spec.rb +37 -1
  7. data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +120 -0
  8. data/flydata-core/spec/table_def/mysqldump_test_column_charset.dump +45 -0
  9. data/flydata-core/spec/table_def/redshift_table_def_spec.rb +70 -88
  10. data/flydata.gemspec +13 -8
  11. data/lib/flydata/command/setup.rb +4 -4
  12. data/lib/flydata/command/sync.rb +18 -29
  13. data/lib/flydata/compatibility_check.rb +2 -2
  14. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +15 -10
  15. data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -3
  16. data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb +15 -8
  17. data/lib/flydata/fluent-plugins/mysql/table_meta.rb +6 -34
  18. data/lib/flydata/{fluent-plugins/mysql → mysql}/binlog_position.rb +2 -0
  19. data/lib/flydata/{util → mysql}/mysql_util.rb +34 -1
  20. data/lib/flydata/mysql/table_ddl.rb +118 -0
  21. data/lib/flydata/parser/mysql/dump_parser.rb +5 -5
  22. data/lib/flydata/parser/mysql/mysql_alter_table.treetop +29 -5
  23. data/lib/flydata/sync_file_manager.rb +15 -2
  24. data/spec/flydata/command/sync_spec.rb +22 -1
  25. data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +7 -2
  26. data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +130 -0
  27. data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +1 -0
  28. data/spec/flydata/fluent-plugins/mysql/table_meta_spec.rb +14 -8
  29. data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +2 -1
  30. data/spec/flydata/{fluent-plugins/mysql → mysql}/binlog_position_spec.rb +3 -2
  31. data/spec/flydata/{util → mysql}/mysql_util_spec.rb +2 -2
  32. data/spec/flydata/mysql/table_ddl_spec.rb +193 -0
  33. data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +37 -9
  34. data/spec/flydata/sync_file_manager_spec.rb +102 -27
  35. metadata +12 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: bc252c8c78f6a02191959f35982336680914d5e5
4
- data.tar.gz: 26741fa0cb11f348e821d25b4664a2e43e5e5afa
3
+ metadata.gz: 15226b89faab2ec73d834294f47e4b2f8fc35b69
4
+ data.tar.gz: 19d207fcea4121d051cb73f536c04769c95cfbc0
5
5
  SHA512:
6
- metadata.gz: c1c78e3a5d5602a1b54081c6260996f5d5558f99464f6ee17ab7e9b93e85b711ae2d4bd4ea729e9f3182065eca4cf96803e69e4bb94771f858060395110ee118
7
- data.tar.gz: 2af8cf366832e48235e03691b9d342ac21fc9bc6a751eb666817f21f245490a4653ef88d5a959930ff6c86709090c537bdf87078b08db8b335b63676bca7f3a3
6
+ metadata.gz: f67502dcee392df1f1016346390266e5e977450359cec43ca0a20402707a28b84547b76ea06063ef9c50de510652a8cc362ae914087e37ace2fc9c2d54384581
7
+ data.tar.gz: a2dbad269e6e2e7eb0e2042c4b05a51441cb92b3f6cca58cc2fdd47b922a0a09eb702d22882e64557757dd93b809cd2552ecbc81f45c27673b75ee16317b3d19
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.3.16
1
+ 0.3.17
@@ -0,0 +1,13 @@
1
+ module FlydataCore
2
+ module Record
3
+
4
+ V1 = 1
5
+ # - Handles string/binary as UTF-8
6
+ # - No "v" peroperty
7
+
8
+ V2 = 2
9
+ # - Supports the common string/binary format
10
+ # - "v": 2
11
+ # - additions of "attrs" property. "enc":"b" is currently supported.
12
+ end
13
+ end
@@ -87,11 +87,14 @@ class MysqlTableDef
87
87
  params ? self.new(*params) : nil
88
88
  end
89
89
 
90
- def initialize(table_def, table_name, columns, default_charset, comment)
90
+ def initialize(table_def, table_name, columns, column_def, default_charset,
91
+ default_charset_mysql, comment)
91
92
  @table_def = table_def
92
93
  @table_name = table_name
93
94
  @columns = columns
95
+ @column_def = column_def
94
96
  @default_charset = default_charset
97
+ @default_charset_mysql = default_charset_mysql
95
98
  @comment = comment
96
99
  end
97
100
 
@@ -99,7 +102,9 @@ class MysqlTableDef
99
102
  table_def = ''
100
103
  table_name = nil
101
104
  columns = []
105
+ column_def = {}
102
106
  default_charset = nil
107
+ default_charset_mysql = nil
103
108
  comment = nil
104
109
 
105
110
  position = :before_create_table
@@ -120,13 +125,19 @@ class MysqlTableDef
120
125
  stripped_line = line.strip
121
126
  # `col_smallint` smallint(6) DEFAULT NULL,
122
127
  if stripped_line.start_with?('`')
123
- columns << parse_one_column_def(line)
128
+ column = parse_one_column_def(line)
129
+ columns << column
130
+ coldef = line.strip.gsub(/,$/, '')
131
+ column_def[column[:column]] = coldef
124
132
  # PRIMARY KEY (`id`)
125
133
  elsif stripped_line.start_with?("PRIMARY KEY")
126
134
  parse_key(line, columns)
127
135
  #) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table';
128
136
  elsif stripped_line.start_with?(')')
129
- default_charset = $1 if line =~ /DEFAULT CHARSET\s*=\s*([^\s]+)/
137
+ if line =~ /DEFAULT CHARSET\s*=\s*([a-z0-9]+)/
138
+ default_charset_mysql = $1
139
+ default_charset = flydata_charset(default_charset_mysql)
140
+ end
130
141
  comment = $1 if /COMMENT='((?:\\'|[^'])*)'/.match(line)
131
142
  position = :after_create_table
132
143
  elsif stripped_line.start_with?("KEY")
@@ -146,9 +157,9 @@ class MysqlTableDef
146
157
  break
147
158
  end
148
159
  end
149
- position == :after_create_table ? [table_def, table_name, columns, default_charset, comment] : nil
160
+ position == :after_create_table ? [table_def, table_name, columns, column_def, default_charset, default_charset_mysql, comment] : nil
150
161
  end
151
- attr_reader :columns, :table_name
162
+ attr_reader :columns, :column_def, :table_name, :default_charset_mysql
152
163
 
153
164
  def to_flydata_tabledef
154
165
  tabledef = { table_name: @table_name,
@@ -207,6 +218,9 @@ class MysqlTableDef
207
218
  if /COMMENT\s+'(((?:\\'|[^'])*))'/i.match(line)
208
219
  column[:comment] = $1
209
220
  end
221
+ if /CHARACTER SET\s+([a-z0-9]+)/i.match(line)
222
+ column[:charset] = flydata_charset($1)
223
+ end
210
224
  if block_given?
211
225
  column = yield(column, query, pos)
212
226
  end
@@ -218,6 +232,93 @@ class MysqlTableDef
218
232
  column
219
233
  end
220
234
 
235
+ # 'binary' is a fixed length column type. mysqldump exports a binary column
236
+ # value at its full length by adding trailing 0's to the size of the column
237
+ # (ex. '0xAB00920000' for type 'binary(5)')
238
+ # FlyData Transport Format does not allow such paddings so they need to be
239
+ # removed. On the target database, padding may be added according to the
240
+ # destination DB's convention.
241
+ MYSQL_BINARY_PADDING_CHAR = '00'
242
+ PROC_convert_binary_value = -> (value) do
243
+ return nil unless value
244
+ while value.end_with?(MYSQL_BINARY_PADDING_CHAR)
245
+ value = value[0...-MYSQL_BINARY_PADDING_CHAR.size]
246
+ end
247
+ value
248
+ end
249
+
250
+ VALUE_CONVERTERS = {
251
+ 'binary' => PROC_convert_binary_value,
252
+ }
253
+
254
+ def self.convert_to_flydata_value(value, type)
255
+ if converter = VALUE_CONVERTERS[type]
256
+ value = converter.call(value)
257
+ end
258
+ value
259
+ end
260
+
261
+ # Charset naming conversion rule. mysql => ruby
262
+ #
263
+ # mysql
264
+ # http://dev.mysql.com/doc/refman/5.6/en/charset-charsets.html
265
+ # mysql(supported CJK character sets)
266
+ # http://dev.mysql.com/doc/refman/5.6/en/faqs-cjk.html#qandaitem-A-11-1-1
267
+ # For ruby, you can see encoding list with "Encoding.list"
268
+ CHARSET_ENCODE_RULE = {
269
+ "armscii8" => nil,
270
+ "ascii" => Encoding::US_ASCII,
271
+ "big5" => Encoding::Big5,
272
+ "binary" => Encoding::ASCII_8BIT,
273
+ "cp1250" => Encoding::Windows_1250,
274
+ "cp1251" => Encoding::Windows_1251,
275
+ "cp1256" => Encoding::Windows_1256,
276
+ "cp1257" => Encoding::Windows_1257,
277
+ "cp850" => Encoding::CP850,
278
+ "cp852" => Encoding::CP852,
279
+ "cp866" => Encoding::IBM866,
280
+ "cp932" => Encoding::Windows_31J,
281
+ "dec8" => nil,
282
+ "eucjpms" => Encoding::EucJP_ms,
283
+ "euckr" => Encoding::EUC_KR,
284
+ "gb2312" => Encoding::EUC_CN,
285
+ "gbk" => Encoding::GBK,
286
+ "geostd8" => nil,
287
+ "greek" => Encoding::ISO_8859_7,
288
+ "hebrew" => Encoding::ISO_8859_8,
289
+ "hp8" => nil,
290
+ "keybcs2" => nil,
291
+ "koi8r" => Encoding::KOI8_R,
292
+ "koi8u" => Encoding::KOI8_U,
293
+ "latin1" => Encoding::ISO_8859_1,
294
+ "latin2" => Encoding::ISO_8859_2,
295
+ "latin5" => Encoding::ISO_8859_9,
296
+ "latin7" => Encoding::ISO_8859_13,
297
+ "macce" => Encoding::MacCentEuro,
298
+ "macroman" => Encoding::MacRoman,
299
+ "sjis" => Encoding::SHIFT_JIS,
300
+ "swe7" => nil,
301
+ "tis620" => Encoding::TIS_620,
302
+ "ucs2" => Encoding::UTF_16BE,
303
+ "ujis" => Encoding::EUC_JP,
304
+ "utf16" => Encoding::UTF_16,
305
+ "utf16le" => Encoding::UTF_16LE,
306
+ "utf32" => Encoding::UTF_32,
307
+ "utf8" => Encoding::UTF_8,
308
+ "utf8mb4" => Encoding::UTF_8,
309
+ }
310
+
311
+ def self.ruby_encoding(mysql_charset)
312
+ return nil if mysql_charset.to_s.empty?
313
+ raise "Unsupported charset:#{mysql_charset}." unless CHARSET_ENCODE_RULE.has_key?(mysql_charset)
314
+ charset = CHARSET_ENCODE_RULE[mysql_charset]
315
+ return charset
316
+ end
317
+
318
+ def self.flydata_charset(mysql_charset)
319
+ ruby_encoding(mysql_charset).to_s.upcase.gsub('-', '_')
320
+ end
321
+
221
322
  private
222
323
 
223
324
  def self.parse_key(line, columns, type = :primary_key)
@@ -232,6 +333,7 @@ class MysqlTableDef
232
333
  column[type] = true
233
334
  end
234
335
  end
336
+
235
337
  end
236
338
 
237
339
  end
@@ -53,10 +53,11 @@ class RedshiftTableDef
53
53
  end
54
54
 
55
55
  FLYDATA_CTL_COLUMNS_TABLE = "flydata_ctl_columns"
56
+ FLYDATA_CTL_TABLES_TABLE = "flydata_ctl_tables"
56
57
  CREATE_SCHEMA_SQL = <<EOS
57
- CREATE SCHEMA "%s";
58
+ CREATE SCHEMA IF NOT EXISTS "%s";
58
59
  EOS
59
- CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
60
+ CREATE_FLYDATA_CTL_COLUMNS_SQL = <<EOS
60
61
  CREATE TABLE IF NOT EXISTS %s(
61
62
  id integer NOT NULL IDENTITY(1,1),
62
63
  table_name varchar(128) NOT NULL,
@@ -67,13 +68,29 @@ CREATE TABLE IF NOT EXISTS %s(
67
68
  PRIMARY KEY(id)
68
69
  ) DISTKEY(table_name) SORTKEY(table_name);
69
70
  EOS
71
+ CREATE_FLYDATA_CTL_TABLES_SQL = <<EOS
72
+ CREATE TABLE IF NOT EXISTS %s(
73
+ id integer NOT NULL IDENTITY(1,1),
74
+ table_name varchar(128) NOT NULL,
75
+ attribute varchar(128) NOT NULL,
76
+ value varchar(max),
77
+ created_at timestamp DEFAULT SYSDATE,
78
+ PRIMARY KEY(id)
79
+ ) DISTKEY(table_name) SORTKEY(table_name);
80
+ EOS
81
+ CREATE_FLYDATA_CTL_TABLE_SQL = "#{CREATE_FLYDATA_CTL_COLUMNS_SQL}#{CREATE_FLYDATA_CTL_TABLES_SQL}"
82
+
70
83
 
71
84
  def self.table_name_for_ddl(table_name, schema_name)
72
85
  schema_name.to_s.empty? ? "\"#{table_name}\"" : "\"#{schema_name}\".\"#{table_name}\""
73
86
  end
74
87
 
75
- def self.flydata_ctl_table_for_ddl(schema_name)
76
- table_name_for_ddl(FLYDATA_CTL_COLUMNS_TABLE, schema_name)
88
+ def self.flydata_ctl_table_for_ddl(schema_name, ctl_table_type = :columns)
89
+ table_name = case ctl_table_type
90
+ when :columns; FLYDATA_CTL_COLUMNS_TABLE
91
+ when :tables; FLYDATA_CTL_TABLES_TABLE
92
+ end
93
+ table_name_for_ddl(table_name, schema_name)
77
94
  end
78
95
 
79
96
  def self.create_schema_sql(schema_name)
@@ -82,8 +99,14 @@ EOS
82
99
 
83
100
  def self.create_flydata_ctl_table_sql(schema_name)
84
101
  # No drop table here intentionally because losing the data is fatal.
85
- tbl = flydata_ctl_table_for_ddl(schema_name)
86
- CREATE_FLYDATA_CTL_TABLE_SQL % [tbl, tbl]
102
+ columns_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
103
+ tables_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
104
+ CREATE_FLYDATA_CTL_TABLE_SQL % [columns_tbl, tables_tbl]
105
+ end
106
+
107
+ def self.create_flydata_ctl_tables_sql(schema_name)
108
+ tables_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
109
+ CREATE_FLYDATA_CTL_TABLES_SQL % [tables_tbl]
87
110
  end
88
111
 
89
112
  CREATE_TABLE_SQL = <<EOS
@@ -126,7 +149,7 @@ EOS
126
149
  line = %Q| "#{column[:column]}" #{rs_type}|
127
150
  line += " NOT NULL" if column[:not_null]
128
151
  if (column.has_key?(:default))
129
- val = replace_default_value(type_info[:type], column[:default])
152
+ val = replace_default_value(type, type_info[:type], column[:default])
130
153
  line += " DEFAULT #{val}"
131
154
  elsif column[:not_null] && opt[:for] == :alter_table
132
155
  # Redshift doesn't allow adding a not null column without default value
@@ -142,8 +165,16 @@ EOS
142
165
 
143
166
  NULL_STR = "NULL"
144
167
 
145
- def self.replace_default_value(redshift_type, default_value)
168
+ def self.replace_default_value(flydata_type, redshift_type, default_value)
146
169
  return NULL_STR if default_value.nil?
170
+ if flydata_type.start_with?('year')
171
+ value = convert_year_into_date(remove_single_quote(default_value))
172
+ begin
173
+ Date.parse(value)
174
+ rescue
175
+ raise "default value of YEAR type must be 2 or 4-digit, value:'#{default_value}'"
176
+ end
177
+ end
147
178
 
148
179
  case redshift_type
149
180
  when 'timestamp'
@@ -204,21 +235,41 @@ EOS
204
235
  sql
205
236
  end
206
237
 
238
+ def self.flydata_ctl_sql(flydata_tabledef, schema_name)
239
+ flydata_ctl_columns_sql(flydata_tabledef, schema_name) + "\n" +
240
+ flydata_ctl_tables_sql(flydata_tabledef, schema_name)
241
+ end
207
242
  FLYDATA_CTL_COLUMNS_SQL = <<EOS
208
243
  DELETE FROM %s WHERE table_name = '%s';
209
244
  INSERT INTO %s (table_name, column_name, src_data_type, ordinal_position) VALUES
210
245
  EOS
211
- def self.flydata_ctl_sql(flydata_tabledef, schema_name)
212
- flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name)
246
+ def self.flydata_ctl_columns_sql(flydata_tabledef, schema_name)
247
+ flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
213
248
  sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
214
249
  values = []
215
250
  flydata_tabledef[:columns].each.with_index(1) do |col, i|
216
- values << "('#{flydata_tabledef[:table_name]}', '#{col[:column]}', '#{escape(col[:type])}', #{i})"
251
+ charset = col[:charset] ? " cs:#{col[:charset]}" : ""
252
+ values << "('#{flydata_tabledef[:table_name]}', '#{col[:column]}', '#{escape(col[:type])}#{charset}', #{i})"
217
253
  end
218
254
  sql += values.join(",\n") + ';'
219
255
  sql
220
256
  end
221
257
 
258
+ FLYDATA_CTL_TABLES_SQL = <<EOS
259
+ DELETE FROM %s WHERE table_name = '%s';
260
+ INSERT INTO %s (table_name, attribute, value) VALUES
261
+ EOS
262
+
263
+ def self.flydata_ctl_tables_sql(flydata_tabledef, schema_name)
264
+ flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
265
+ sql = FLYDATA_CTL_TABLES_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
266
+ values = []
267
+ values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
268
+ values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
269
+ sql += values.join(",\n") + ';'
270
+ sql
271
+ end
272
+
222
273
  def self.escape(text)
223
274
  text.gsub("'", "\\\\'")
224
275
  end
@@ -65,7 +65,7 @@ describe MysqlTableDef do
65
65
  end
66
66
 
67
67
  it 'default_charset should be set' do
68
- expect(subject[:default_charset]).to eq('utf8')
68
+ expect(subject[:default_charset]).to eq('UTF_8')
69
69
  end
70
70
 
71
71
  it 'comment should be set' do
@@ -201,6 +201,13 @@ describe MysqlTableDef do
201
201
  expect(subject[:columns][2][:default]).to eq("x'10'")
202
202
  end
203
203
  end
204
+
205
+ context 'when a column has a character set' do
206
+ let(:dump_file_io) { file_io('mysqldump_test_column_charset.dump')}
207
+ it 'should extract charset value for the columns' do
208
+ expect(subject[:columns][1][:charset]).to eq("ISO_8859_1")
209
+ end
210
+ end
204
211
  end
205
212
 
206
213
  describe '.convert_to_flydata_type' do
@@ -338,6 +345,35 @@ describe MysqlTableDef do
338
345
  end
339
346
  end
340
347
  end
348
+ describe '.convert_to_flydata_value' do
349
+ subject { MysqlTableDef.convert_to_flydata_value(value, type) }
350
+
351
+ let(:value_body) { '0xFE0040' }
352
+ context 'binary type' do
353
+ let(:type) { 'binary' }
354
+ context 'value with padding' do
355
+ let(:value) { "#{value_body}0000000000" }
356
+ it 'truncates padding' do
357
+ is_expected.to eq value_body
358
+ end
359
+ end
360
+ context 'nil value' do
361
+ let(:value) { nil }
362
+ it 'returns nil' do
363
+ is_expected.to eq nil
364
+ end
365
+ end
366
+ end
367
+ context 'varbinary type' do
368
+ let(:type) { 'varbinary' }
369
+ context 'value_with_padding' do
370
+ let(:value) { "#{value_body}0000000000" }
371
+ it 'returns the value as is' do
372
+ is_expected.to eq value
373
+ end
374
+ end
375
+ end
376
+ end
341
377
  end
342
378
 
343
379
  end
@@ -36,6 +36,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
36
36
  ordinal_position int NOT NULL,
37
37
  PRIMARY KEY(id)
38
38
  ) DISTKEY(table_name) SORTKEY(table_name);
39
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
40
+ id integer NOT NULL IDENTITY(1,1),
41
+ table_name varchar(128) NOT NULL,
42
+ attribute varchar(128) NOT NULL,
43
+ value varchar(max),
44
+ created_at timestamp DEFAULT SYSDATE,
45
+ PRIMARY KEY(id)
46
+ ) DISTKEY(table_name) SORTKEY(table_name);
39
47
  DROP TABLE IF EXISTS "test_table_all";
40
48
  CREATE TABLE "test_table_all" (
41
49
  "id" int8 NOT NULL,
@@ -102,6 +110,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
102
110
  ('test_table_all', 'col_year', 'year', 28),
103
111
  ('test_table_all', 'col_year_4', 'year(4)', 29),
104
112
  ('test_table_all', 'col_year_2', 'year(2)', 30);
113
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_all';
114
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
115
+ ('test_table_all', 'cs', 'UTF_8'),
116
+ ('test_table_all', 'revision', 1);
105
117
  EOT
106
118
  end
107
119
  end
@@ -120,6 +132,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
120
132
  ordinal_position int NOT NULL,
121
133
  PRIMARY KEY(id)
122
134
  ) DISTKEY(table_name) SORTKEY(table_name);
135
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
136
+ id integer NOT NULL IDENTITY(1,1),
137
+ table_name varchar(128) NOT NULL,
138
+ attribute varchar(128) NOT NULL,
139
+ value varchar(max),
140
+ created_at timestamp DEFAULT SYSDATE,
141
+ PRIMARY KEY(id)
142
+ ) DISTKEY(table_name) SORTKEY(table_name);
123
143
  DROP TABLE IF EXISTS "bit_test_def_1";
124
144
  CREATE TABLE "bit_test_def_1" (
125
145
  "id" int4 NOT NULL,
@@ -132,6 +152,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
132
152
  ('bit_test_def_1', 'id', 'int4(11)', 1),
133
153
  ('bit_test_def_1', 'bit_value', 'bit(1)', 2),
134
154
  ('bit_test_def_1', 'int_value', 'int4(11)', 3);
155
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'bit_test_def_1';
156
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
157
+ ('bit_test_def_1', 'cs', 'UTF_8'),
158
+ ('bit_test_def_1', 'revision', 1);
135
159
  EOT
136
160
  end
137
161
  end
@@ -150,6 +174,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
150
174
  ordinal_position int NOT NULL,
151
175
  PRIMARY KEY(id)
152
176
  ) DISTKEY(table_name) SORTKEY(table_name);
177
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
178
+ id integer NOT NULL IDENTITY(1,1),
179
+ table_name varchar(128) NOT NULL,
180
+ attribute varchar(128) NOT NULL,
181
+ value varchar(max),
182
+ created_at timestamp DEFAULT SYSDATE,
183
+ PRIMARY KEY(id)
184
+ ) DISTKEY(table_name) SORTKEY(table_name);
153
185
  DROP TABLE IF EXISTS "product_order";
154
186
  CREATE TABLE "product_order" (
155
187
  "no" int4 NOT NULL,
@@ -164,6 +196,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
164
196
  ('product_order', 'product_category', 'int4(11)', 2),
165
197
  ('product_order', 'product_id', 'int4(11)', 3),
166
198
  ('product_order', 'customer_id', 'int4(11)', 4);
199
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'product_order';
200
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
201
+ ('product_order', 'cs', 'UTF_8'),
202
+ ('product_order', 'revision', 1);
167
203
  EOT
168
204
  end
169
205
  end
@@ -182,6 +218,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
182
218
  ordinal_position int NOT NULL,
183
219
  PRIMARY KEY(id)
184
220
  ) DISTKEY(table_name) SORTKEY(table_name);
221
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
222
+ id integer NOT NULL IDENTITY(1,1),
223
+ table_name varchar(128) NOT NULL,
224
+ attribute varchar(128) NOT NULL,
225
+ value varchar(max),
226
+ created_at timestamp DEFAULT SYSDATE,
227
+ PRIMARY KEY(id)
228
+ ) DISTKEY(table_name) SORTKEY(table_name);
185
229
  DROP TABLE IF EXISTS "test_table_column_comment";
186
230
  CREATE TABLE "test_table_column_comment" (
187
231
  "id" int4 NOT NULL DEFAULT '0',
@@ -194,6 +238,10 @@ DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_column_comment'
194
238
  INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
195
239
  ('test_table_column_comment', 'id', 'int4(11)', 1),
196
240
  ('test_table_column_comment', 'value', 'text', 2);
241
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_column_comment';
242
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
243
+ ('test_table_column_comment', 'cs', 'UTF_8'),
244
+ ('test_table_column_comment', 'revision', 1);
197
245
  EOT
198
246
  end
199
247
  end
@@ -212,6 +260,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
212
260
  ordinal_position int NOT NULL,
213
261
  PRIMARY KEY(id)
214
262
  ) DISTKEY(table_name) SORTKEY(table_name);
263
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
264
+ id integer NOT NULL IDENTITY(1,1),
265
+ table_name varchar(128) NOT NULL,
266
+ attribute varchar(128) NOT NULL,
267
+ value varchar(max),
268
+ created_at timestamp DEFAULT SYSDATE,
269
+ PRIMARY KEY(id)
270
+ ) DISTKEY(table_name) SORTKEY(table_name);
215
271
  DROP TABLE IF EXISTS "test_table_enum";
216
272
  CREATE TABLE "test_table_enum" (
217
273
  "id" int4 NOT NULL,
@@ -226,6 +282,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
226
282
  ('test_table_enum', 'enum_1', 'enum(\\'apple\\',\\'orange\\',\\'banana\\')', 2),
227
283
  ('test_table_enum', 'enum_2', 'enum(\\'a\\',\\'b\\',\\'c\\')', 3),
228
284
  ('test_table_enum', 'enum_3', 'enum(\\'e\\',\\'f\\',\\'g\\')', 4);
285
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_enum';
286
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
287
+ ('test_table_enum', 'cs', 'UTF_8'),
288
+ ('test_table_enum', 'revision', 1);
229
289
  EOT
230
290
  end
231
291
  end
@@ -244,6 +304,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
244
304
  ordinal_position int NOT NULL,
245
305
  PRIMARY KEY(id)
246
306
  ) DISTKEY(table_name) SORTKEY(table_name);
307
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
308
+ id integer NOT NULL IDENTITY(1,1),
309
+ table_name varchar(128) NOT NULL,
310
+ attribute varchar(128) NOT NULL,
311
+ value varchar(max),
312
+ created_at timestamp DEFAULT SYSDATE,
313
+ PRIMARY KEY(id)
314
+ ) DISTKEY(table_name) SORTKEY(table_name);
247
315
  DROP TABLE IF EXISTS "test_table_multi_pk";
248
316
  CREATE TABLE "test_table_multi_pk" (
249
317
  "id1" int4 NOT NULL DEFAULT '0',
@@ -256,6 +324,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
256
324
  ('test_table_multi_pk', 'id1', 'int4(11)', 1),
257
325
  ('test_table_multi_pk', 'id2', 'int4(11)', 2),
258
326
  ('test_table_multi_pk', 'value', 'text', 3);
327
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_multi_pk';
328
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
329
+ ('test_table_multi_pk', 'cs', 'UTF_8'),
330
+ ('test_table_multi_pk', 'revision', 1);
259
331
  EOT
260
332
  end
261
333
  end
@@ -282,6 +354,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
282
354
  ordinal_position int NOT NULL,
283
355
  PRIMARY KEY(id)
284
356
  ) DISTKEY(table_name) SORTKEY(table_name);
357
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
358
+ id integer NOT NULL IDENTITY(1,1),
359
+ table_name varchar(128) NOT NULL,
360
+ attribute varchar(128) NOT NULL,
361
+ value varchar(max),
362
+ created_at timestamp DEFAULT SYSDATE,
363
+ PRIMARY KEY(id)
364
+ ) DISTKEY(table_name) SORTKEY(table_name);
285
365
  DROP TABLE IF EXISTS "sample1";
286
366
  CREATE TABLE "sample1" (
287
367
  "id" int4 NOT NULL,
@@ -296,6 +376,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
296
376
  ('sample1', 'title', 'varchar(768)', 2),
297
377
  ('sample1', 'name', 'text', 3),
298
378
  ('sample1', 'num', 'int4(11)', 4);
379
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
380
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
381
+ ('sample1', 'cs', 'ISO_8859_1'),
382
+ ('sample1', 'revision', 1);
299
383
  EOT
300
384
  end
301
385
  end
@@ -314,6 +398,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
314
398
  ordinal_position int NOT NULL,
315
399
  PRIMARY KEY(id)
316
400
  ) DISTKEY(table_name) SORTKEY(table_name);
401
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
402
+ id integer NOT NULL IDENTITY(1,1),
403
+ table_name varchar(128) NOT NULL,
404
+ attribute varchar(128) NOT NULL,
405
+ value varchar(max),
406
+ created_at timestamp DEFAULT SYSDATE,
407
+ PRIMARY KEY(id)
408
+ ) DISTKEY(table_name) SORTKEY(table_name);
317
409
  DROP TABLE IF EXISTS "sample1";
318
410
  CREATE TABLE "sample1" (
319
411
  "id" int4 NOT NULL,
@@ -328,6 +420,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
328
420
  ('sample1', 'title', 'varchar(768)', 2),
329
421
  ('sample1', 'name', 'text', 3),
330
422
  ('sample1', 'num', 'int4(11)', 4);
423
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
424
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
425
+ ('sample1', 'cs', 'ISO_8859_1'),
426
+ ('sample1', 'revision', 1);
331
427
  EOT
332
428
  end
333
429
  end
@@ -346,6 +442,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
346
442
  ordinal_position int NOT NULL,
347
443
  PRIMARY KEY(id)
348
444
  ) DISTKEY(table_name) SORTKEY(table_name);
445
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
446
+ id integer NOT NULL IDENTITY(1,1),
447
+ table_name varchar(128) NOT NULL,
448
+ attribute varchar(128) NOT NULL,
449
+ value varchar(max),
450
+ created_at timestamp DEFAULT SYSDATE,
451
+ PRIMARY KEY(id)
452
+ ) DISTKEY(table_name) SORTKEY(table_name);
349
453
  DROP TABLE IF EXISTS "invoice_items";
350
454
  CREATE TABLE "invoice_items" (
351
455
  "id" int4 NOT NULL,
@@ -388,6 +492,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
388
492
  ('invoice_items', 'bill_ahead_resolved_at', 'datetime', 16),
389
493
  ('invoice_items', 'stripe_invoice_id', 'varchar(765)', 17),
390
494
  ('invoice_items', 'is_vat', 'int1(1)', 18);
495
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'invoice_items';
496
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
497
+ ('invoice_items', 'cs', 'UTF_8'),
498
+ ('invoice_items', 'revision', 1);
391
499
  EOT
392
500
  end
393
501
  end
@@ -406,6 +514,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
406
514
  ordinal_position int NOT NULL,
407
515
  PRIMARY KEY(id)
408
516
  ) DISTKEY(table_name) SORTKEY(table_name);
517
+ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
518
+ id integer NOT NULL IDENTITY(1,1),
519
+ table_name varchar(128) NOT NULL,
520
+ attribute varchar(128) NOT NULL,
521
+ value varchar(max),
522
+ created_at timestamp DEFAULT SYSDATE,
523
+ PRIMARY KEY(id)
524
+ ) DISTKEY(table_name) SORTKEY(table_name);
409
525
  DROP TABLE IF EXISTS "zerofill_table";
410
526
  CREATE TABLE "zerofill_table" (
411
527
  "id" int4 NOT NULL,
@@ -426,6 +542,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
426
542
  ('zerofill_table', 'value_double', 'float8 unsigned', 5),
427
543
  ('zerofill_table', 'name', 'varchar(768)', 6),
428
544
  ('zerofill_table', 'value_small_int', 'int2(5) unsigned', 7);
545
+ DELETE FROM "flydata_ctl_tables" WHERE table_name = 'zerofill_table';
546
+ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
547
+ ('zerofill_table', 'cs', 'ISO_8859_1'),
548
+ ('zerofill_table', 'revision', 1);
429
549
  EOT
430
550
  end
431
551
  end