flydata 0.3.16 → 0.3.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bc252c8c78f6a02191959f35982336680914d5e5
-  data.tar.gz: 26741fa0cb11f348e821d25b4664a2e43e5e5afa
+  metadata.gz: 15226b89faab2ec73d834294f47e4b2f8fc35b69
+  data.tar.gz: 19d207fcea4121d051cb73f536c04769c95cfbc0
 SHA512:
-  metadata.gz: c1c78e3a5d5602a1b54081c6260996f5d5558f99464f6ee17ab7e9b93e85b711ae2d4bd4ea729e9f3182065eca4cf96803e69e4bb94771f858060395110ee118
-  data.tar.gz: 2af8cf366832e48235e03691b9d342ac21fc9bc6a751eb666817f21f245490a4653ef88d5a959930ff6c86709090c537bdf87078b08db8b335b63676bca7f3a3
+  metadata.gz: f67502dcee392df1f1016346390266e5e977450359cec43ca0a20402707a28b84547b76ea06063ef9c50de510652a8cc362ae914087e37ace2fc9c2d54384581
+  data.tar.gz: a2dbad269e6e2e7eb0e2042c4b05a51441cb92b3f6cca58cc2fdd47b922a0a09eb702d22882e64557757dd93b809cd2552ecbc81f45c27673b75ee16317b3d19

data/VERSION

@@ -1 +1 @@
-0.3.16
+0.3.17

data/flydata-core/lib/flydata-core/record/record.rb

@@ -0,0 +1,13 @@
+module FlydataCore
+module Record
+
+  V1 = 1
+  #  - Handles string/binary as UTF-8
+  #  - No "v" peroperty
+
+  V2 = 2
+  #  - Supports the common string/binary format
+  #  - "v": 2
+  #  - additions of "attrs" property. "enc":"b" is currently supported.
+end
+end

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -87,11 +87,14 @@ class MysqlTableDef
     params ? self.new(*params) : nil
   end
 
-  def initialize(table_def, table_name, columns, default_charset, comment)
+  def initialize(table_def, table_name, columns, column_def, default_charset,
+                 default_charset_mysql, comment)
     @table_def = table_def
     @table_name = table_name
     @columns = columns
+    @column_def = column_def
     @default_charset = default_charset
+    @default_charset_mysql = default_charset_mysql
     @comment = comment
   end
 
@@ -99,7 +102,9 @@ class MysqlTableDef
     table_def = ''
     table_name = nil
     columns = []
+    column_def = {}
     default_charset = nil
+    default_charset_mysql = nil
     comment = nil
 
     position = :before_create_table
@@ -120,13 +125,19 @@ class MysqlTableDef
         stripped_line = line.strip
         # `col_smallint` smallint(6) DEFAULT NULL,
         if stripped_line.start_with?('`')
-          columns << parse_one_column_def(line)
+          column = parse_one_column_def(line)
+          columns << column
+          coldef = line.strip.gsub(/,$/, '')
+          column_def[column[:column]] = coldef
         # PRIMARY KEY (`id`)
         elsif stripped_line.start_with?("PRIMARY KEY")
           parse_key(line, columns)
         #) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table';
         elsif stripped_line.start_with?(')')
-          default_charset = $1 if line =~ /DEFAULT CHARSET\s*=\s*([^\s]+)/
+          if line =~ /DEFAULT CHARSET\s*=\s*([a-z0-9]+)/
+            default_charset_mysql = $1
+            default_charset = flydata_charset(default_charset_mysql)
+          end
           comment = $1 if /COMMENT='((?:\\'|[^'])*)'/.match(line)
           position = :after_create_table
         elsif stripped_line.start_with?("KEY")
@@ -146,9 +157,9 @@ class MysqlTableDef
         break
       end
     end
-    position == :after_create_table ? [table_def, table_name, columns, default_charset, comment] : nil
+    position == :after_create_table ? [table_def, table_name, columns, column_def, default_charset, default_charset_mysql, comment] : nil
   end
-  attr_reader :columns, :table_name
+  attr_reader :columns, :column_def, :table_name, :default_charset_mysql
 
   def to_flydata_tabledef
     tabledef = { table_name: @table_name,
@@ -207,6 +218,9 @@ class MysqlTableDef
         if /COMMENT\s+'(((?:\\'|[^'])*))'/i.match(line)
           column[:comment] = $1
         end
+        if /CHARACTER SET\s+([a-z0-9]+)/i.match(line)
+          column[:charset] = flydata_charset($1)
+        end
         if block_given?
           column = yield(column, query, pos)
         end
@@ -218,6 +232,93 @@ class MysqlTableDef
     column
   end
 
+  # 'binary' is a fixed length column type.  mysqldump exports a binary column
+  # value at its full length by adding trailing 0's to the size of the column
+  # (ex. '0xAB00920000' for type 'binary(5)')
+  # FlyData Transport Format does not allow such paddings so they need to be
+  # removed.  On the target database, padding may be added according to the
+  # destination DB's convention.
+  MYSQL_BINARY_PADDING_CHAR = '00'
+  PROC_convert_binary_value = -> (value) do
+    return nil unless value
+    while value.end_with?(MYSQL_BINARY_PADDING_CHAR)
+      value = value[0...-MYSQL_BINARY_PADDING_CHAR.size]
+    end
+    value
+  end
+
+  VALUE_CONVERTERS = {
+    'binary' => PROC_convert_binary_value,
+  }
+
+  def self.convert_to_flydata_value(value, type)
+    if converter = VALUE_CONVERTERS[type]
+      value = converter.call(value)
+    end
+    value
+  end
+
+  # Charset naming conversion rule. mysql => ruby
+  #
+  # mysql
+  #   http://dev.mysql.com/doc/refman/5.6/en/charset-charsets.html
+  # mysql(supported CJK character sets)
+  #   http://dev.mysql.com/doc/refman/5.6/en/faqs-cjk.html#qandaitem-A-11-1-1
+  # For ruby, you can see encoding list with "Encoding.list"
+  CHARSET_ENCODE_RULE = {
+    "armscii8" => nil,
+    "ascii"    => Encoding::US_ASCII,
+    "big5"     => Encoding::Big5,
+    "binary"   => Encoding::ASCII_8BIT,
+    "cp1250"   => Encoding::Windows_1250,
+    "cp1251"   => Encoding::Windows_1251,
+    "cp1256"   => Encoding::Windows_1256,
+    "cp1257"   => Encoding::Windows_1257,
+    "cp850"    => Encoding::CP850,
+    "cp852"    => Encoding::CP852,
+    "cp866"    => Encoding::IBM866,
+    "cp932"    => Encoding::Windows_31J,
+    "dec8"     => nil,
+    "eucjpms"  => Encoding::EucJP_ms,
+    "euckr"    => Encoding::EUC_KR,
+    "gb2312"   => Encoding::EUC_CN,
+    "gbk"      => Encoding::GBK,
+    "geostd8"  => nil,
+    "greek"    => Encoding::ISO_8859_7,
+    "hebrew"   => Encoding::ISO_8859_8,
+    "hp8"      => nil,
+    "keybcs2"  => nil,
+    "koi8r"    => Encoding::KOI8_R,
+    "koi8u"    => Encoding::KOI8_U,
+    "latin1"   => Encoding::ISO_8859_1,
+    "latin2"   => Encoding::ISO_8859_2,
+    "latin5"   => Encoding::ISO_8859_9,
+    "latin7"   => Encoding::ISO_8859_13,
+    "macce"    => Encoding::MacCentEuro,
+    "macroman" => Encoding::MacRoman,
+    "sjis"     => Encoding::SHIFT_JIS,
+    "swe7"     => nil,
+    "tis620"   => Encoding::TIS_620,
+    "ucs2"     => Encoding::UTF_16BE,
+    "ujis"     => Encoding::EUC_JP,
+    "utf16"    => Encoding::UTF_16,
+    "utf16le"  => Encoding::UTF_16LE,
+    "utf32"    => Encoding::UTF_32,
+    "utf8"     => Encoding::UTF_8,
+    "utf8mb4"  => Encoding::UTF_8,
+  }
+
+  def self.ruby_encoding(mysql_charset)
+    return nil if mysql_charset.to_s.empty?
+    raise "Unsupported charset:#{mysql_charset}." unless CHARSET_ENCODE_RULE.has_key?(mysql_charset)
+    charset = CHARSET_ENCODE_RULE[mysql_charset]
+    return charset
+  end
+
+  def self.flydata_charset(mysql_charset)
+    ruby_encoding(mysql_charset).to_s.upcase.gsub('-', '_')
+  end
+
   private
 
   def self.parse_key(line, columns, type = :primary_key)
@@ -232,6 +333,7 @@ class MysqlTableDef
       column[type] = true
     end
   end
+
 end
 
 end

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -53,10 +53,11 @@ class RedshiftTableDef
   end
 
   FLYDATA_CTL_COLUMNS_TABLE = "flydata_ctl_columns"
+  FLYDATA_CTL_TABLES_TABLE = "flydata_ctl_tables"
   CREATE_SCHEMA_SQL = <<EOS
-CREATE SCHEMA "%s";
+CREATE SCHEMA IF NOT EXISTS "%s";
 EOS
-  CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
+  CREATE_FLYDATA_CTL_COLUMNS_SQL = <<EOS
 CREATE TABLE IF NOT EXISTS %s(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
@@ -67,13 +68,29 @@ CREATE TABLE IF NOT EXISTS %s(
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
 EOS
+  CREATE_FLYDATA_CTL_TABLES_SQL = <<EOS
+CREATE TABLE IF NOT EXISTS %s(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
+EOS
+  CREATE_FLYDATA_CTL_TABLE_SQL = "#{CREATE_FLYDATA_CTL_COLUMNS_SQL}#{CREATE_FLYDATA_CTL_TABLES_SQL}"
+
 
   def self.table_name_for_ddl(table_name, schema_name)
     schema_name.to_s.empty? ? "\"#{table_name}\"" : "\"#{schema_name}\".\"#{table_name}\""
   end
 
-  def self.flydata_ctl_table_for_ddl(schema_name)
-    table_name_for_ddl(FLYDATA_CTL_COLUMNS_TABLE, schema_name)
+  def self.flydata_ctl_table_for_ddl(schema_name, ctl_table_type = :columns)
+    table_name = case ctl_table_type
+                 when :columns; FLYDATA_CTL_COLUMNS_TABLE
+                 when :tables;  FLYDATA_CTL_TABLES_TABLE
+                 end
+    table_name_for_ddl(table_name, schema_name)
   end
 
   def self.create_schema_sql(schema_name)
@@ -82,8 +99,14 @@ EOS
 
   def self.create_flydata_ctl_table_sql(schema_name)
     # No drop table here intentionally because losing the data is fatal.
-    tbl = flydata_ctl_table_for_ddl(schema_name)
-    CREATE_FLYDATA_CTL_TABLE_SQL % [tbl, tbl]
+    columns_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
+    tables_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
+    CREATE_FLYDATA_CTL_TABLE_SQL % [columns_tbl, tables_tbl]
+  end
+
+  def self.create_flydata_ctl_tables_sql(schema_name)
+    tables_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
+    CREATE_FLYDATA_CTL_TABLES_SQL % [tables_tbl]
   end
 
   CREATE_TABLE_SQL = <<EOS
@@ -126,7 +149,7 @@ EOS
     line = %Q|  "#{column[:column]}" #{rs_type}|
     line += " NOT NULL" if column[:not_null]
     if (column.has_key?(:default))
-      val = replace_default_value(type_info[:type], column[:default])
+      val = replace_default_value(type, type_info[:type], column[:default])
       line += " DEFAULT #{val}"
     elsif column[:not_null] && opt[:for] == :alter_table
       # Redshift doesn't allow adding a not null column without default value
@@ -142,8 +165,16 @@ EOS
 
   NULL_STR = "NULL"
 
-  def self.replace_default_value(redshift_type, default_value)
+  def self.replace_default_value(flydata_type, redshift_type, default_value)
     return NULL_STR if default_value.nil?
+    if flydata_type.start_with?('year')
+      value = convert_year_into_date(remove_single_quote(default_value))
+      begin
+        Date.parse(value)
+      rescue
+        raise "default value of YEAR type must be 2 or 4-digit, value:'#{default_value}'"
+      end
+    end
 
     case redshift_type
     when 'timestamp'
@@ -204,21 +235,41 @@ EOS
     sql
   end
 
+  def self.flydata_ctl_sql(flydata_tabledef, schema_name)
+    flydata_ctl_columns_sql(flydata_tabledef, schema_name) + "\n" +
+    flydata_ctl_tables_sql(flydata_tabledef, schema_name)
+  end
   FLYDATA_CTL_COLUMNS_SQL = <<EOS
 DELETE FROM %s WHERE table_name = '%s';
 INSERT INTO %s (table_name, column_name, src_data_type, ordinal_position) VALUES
 EOS
-  def self.flydata_ctl_sql(flydata_tabledef, schema_name)
-    flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name)
+  def self.flydata_ctl_columns_sql(flydata_tabledef, schema_name)
+    flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
     sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
     values = []
     flydata_tabledef[:columns].each.with_index(1) do |col, i|
-      values << "('#{flydata_tabledef[:table_name]}', '#{col[:column]}', '#{escape(col[:type])}', #{i})"
+      charset = col[:charset] ? " cs:#{col[:charset]}" : ""
+      values << "('#{flydata_tabledef[:table_name]}', '#{col[:column]}', '#{escape(col[:type])}#{charset}', #{i})"
     end
     sql += values.join(",\n") + ';'
     sql
   end
 
+  FLYDATA_CTL_TABLES_SQL = <<EOS
+DELETE FROM %s WHERE table_name = '%s';
+INSERT INTO %s (table_name, attribute, value) VALUES
+EOS
+
+  def self.flydata_ctl_tables_sql(flydata_tabledef, schema_name)
+    flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :tables)
+    sql = FLYDATA_CTL_TABLES_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
+    values = []
+    values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
+    values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
+    sql += values.join(",\n") + ';'
+    sql
+  end
+
   def self.escape(text)
     text.gsub("'", "\\\\'")
   end

data/flydata-core/spec/table_def/mysql_table_def_spec.rb

@@ -65,7 +65,7 @@ describe MysqlTableDef do
       end
 
       it 'default_charset should be set' do
-        expect(subject[:default_charset]).to eq('utf8')
+        expect(subject[:default_charset]).to eq('UTF_8')
       end
 
       it 'comment should be set' do
@@ -201,6 +201,13 @@ describe MysqlTableDef do
         expect(subject[:columns][2][:default]).to eq("x'10'")
       end
     end
+
+    context 'when a column has a character set' do
+      let(:dump_file_io) { file_io('mysqldump_test_column_charset.dump')}
+      it 'should extract charset value for the columns' do
+        expect(subject[:columns][1][:charset]).to eq("ISO_8859_1")
+      end
+    end
   end
 
   describe '.convert_to_flydata_type' do
@@ -338,6 +345,35 @@ describe MysqlTableDef do
       end
     end
   end
+  describe '.convert_to_flydata_value' do
+    subject { MysqlTableDef.convert_to_flydata_value(value, type) }
+
+    let(:value_body) { '0xFE0040' }
+    context 'binary type' do
+      let(:type) { 'binary' }
+      context 'value with padding' do
+        let(:value) { "#{value_body}0000000000" }
+        it 'truncates padding' do
+          is_expected.to eq value_body
+        end
+      end
+      context 'nil value' do
+        let(:value) { nil }
+        it 'returns nil' do
+          is_expected.to eq nil
+        end
+      end
+    end
+    context 'varbinary type' do
+      let(:type) { 'varbinary' }
+      context 'value_with_padding' do
+        let(:value) { "#{value_body}0000000000" }
+        it 'returns the value as is' do
+          is_expected.to eq value
+        end
+      end
+    end
+  end
 end
 
 end

data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb

@@ -36,6 +36,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "test_table_all";
 CREATE TABLE "test_table_all" (
   "id" int8 NOT NULL,
@@ -102,6 +110,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('test_table_all', 'col_year', 'year', 28),
 ('test_table_all', 'col_year_4', 'year(4)', 29),
 ('test_table_all', 'col_year_2', 'year(2)', 30);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_all';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('test_table_all', 'cs', 'UTF_8'),
+('test_table_all', 'revision', 1);
 EOT
         end
       end
@@ -120,6 +132,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "bit_test_def_1";
 CREATE TABLE "bit_test_def_1" (
   "id" int4 NOT NULL,
@@ -132,6 +152,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('bit_test_def_1', 'id', 'int4(11)', 1),
 ('bit_test_def_1', 'bit_value', 'bit(1)', 2),
 ('bit_test_def_1', 'int_value', 'int4(11)', 3);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'bit_test_def_1';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('bit_test_def_1', 'cs', 'UTF_8'),
+('bit_test_def_1', 'revision', 1);
 EOT
         end
       end
@@ -150,6 +174,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "product_order";
 CREATE TABLE "product_order" (
   "no" int4 NOT NULL,
@@ -164,6 +196,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('product_order', 'product_category', 'int4(11)', 2),
 ('product_order', 'product_id', 'int4(11)', 3),
 ('product_order', 'customer_id', 'int4(11)', 4);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'product_order';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('product_order', 'cs', 'UTF_8'),
+('product_order', 'revision', 1);
 EOT
         end
       end
@@ -182,6 +218,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "test_table_column_comment";
 CREATE TABLE "test_table_column_comment" (
   "id" int4 NOT NULL DEFAULT '0',
@@ -194,6 +238,10 @@ DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_column_comment'
 INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
 ('test_table_column_comment', 'id', 'int4(11)', 1),
 ('test_table_column_comment', 'value', 'text', 2);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_column_comment';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('test_table_column_comment', 'cs', 'UTF_8'),
+('test_table_column_comment', 'revision', 1);
 EOT
         end
       end
@@ -212,6 +260,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "test_table_enum";
 CREATE TABLE "test_table_enum" (
   "id" int4 NOT NULL,
@@ -226,6 +282,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('test_table_enum', 'enum_1', 'enum(\\'apple\\',\\'orange\\',\\'banana\\')', 2),
 ('test_table_enum', 'enum_2', 'enum(\\'a\\',\\'b\\',\\'c\\')', 3),
 ('test_table_enum', 'enum_3', 'enum(\\'e\\',\\'f\\',\\'g\\')', 4);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_enum';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('test_table_enum', 'cs', 'UTF_8'),
+('test_table_enum', 'revision', 1);
 EOT
         end
       end
@@ -244,6 +304,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "test_table_multi_pk";
 CREATE TABLE "test_table_multi_pk" (
   "id1" int4 NOT NULL DEFAULT '0',
@@ -256,6 +324,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('test_table_multi_pk', 'id1', 'int4(11)', 1),
 ('test_table_multi_pk', 'id2', 'int4(11)', 2),
 ('test_table_multi_pk', 'value', 'text', 3);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table_multi_pk';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('test_table_multi_pk', 'cs', 'UTF_8'),
+('test_table_multi_pk', 'revision', 1);
 EOT
         end
       end
@@ -282,6 +354,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "sample1";
 CREATE TABLE "sample1" (
   "id" int4 NOT NULL,
@@ -296,6 +376,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('sample1', 'title', 'varchar(768)', 2),
 ('sample1', 'name', 'text', 3),
 ('sample1', 'num', 'int4(11)', 4);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('sample1', 'cs', 'ISO_8859_1'),
+('sample1', 'revision', 1);
 EOT
         end
       end
@@ -314,6 +398,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "sample1";
 CREATE TABLE "sample1" (
   "id" int4 NOT NULL,
@@ -328,6 +420,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('sample1', 'title', 'varchar(768)', 2),
 ('sample1', 'name', 'text', 3),
 ('sample1', 'num', 'int4(11)', 4);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'sample1';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('sample1', 'cs', 'ISO_8859_1'),
+('sample1', 'revision', 1);
 EOT
         end
       end
@@ -346,6 +442,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "invoice_items";
 CREATE TABLE "invoice_items" (
   "id" int4 NOT NULL,
@@ -388,6 +492,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('invoice_items', 'bill_ahead_resolved_at', 'datetime', 16),
 ('invoice_items', 'stripe_invoice_id', 'varchar(765)', 17),
 ('invoice_items', 'is_vat', 'int1(1)', 18);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'invoice_items';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('invoice_items', 'cs', 'UTF_8'),
+('invoice_items', 'revision', 1);
 EOT
         end
       end
@@ -406,6 +514,14 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
   ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "zerofill_table";
 CREATE TABLE "zerofill_table" (
   "id" int4 NOT NULL,
@@ -426,6 +542,10 @@ INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordin
 ('zerofill_table', 'value_double', 'float8 unsigned', 5),
 ('zerofill_table', 'name', 'varchar(768)', 6),
 ('zerofill_table', 'value_small_int', 'int2(5) unsigned', 7);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'zerofill_table';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('zerofill_table', 'cs', 'ISO_8859_1'),
+('zerofill_table', 'revision', 1);
 EOT
         end
       end