flydata 0.6.4 → 0.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12c8fd94c861658f352698075555d8a868d3ca6d
-  data.tar.gz: ff5061d23bb1cee12430d471834131034bc925b2
+  metadata.gz: 614092c4a27b51744d8f8fdbd521e95205455eac
+  data.tar.gz: 3e6d10c8acd5331700c312c1982a6a9acdba3c66
 SHA512:
-  metadata.gz: 0c956f29080dbc9a517a08b2b865aceef102f26a509061cb2152ee519d0f2d99794348546001d09c48f9b7ae2ed8015b12a593520780ffe1517614a9fc1dedde
-  data.tar.gz: 731ecc427a94b7aab2afb638c8dbbb532b6055f080ed938574d26a19b187478d1fecbb75dc1c09a066b816983dfb0611711a27b8123b14370efda8128f535da0
+  metadata.gz: 0370a93f87eedacf8b56210c15363be7f040980b73eb938cf077953a16d0da942d522881601e21ded8a89a1376c480263483a61d4b0932cd41b70739c457f964
+  data.tar.gz: d870e1abfe9491ad135b1b8ee4e45a293b8a312db2823b7a1cc73fd7d337d19a47e68935c2b6fa606a1f0b2a95af87e7c0ee350170192638b109e75acf567647

data/Gemfile

@@ -9,6 +9,7 @@ gem "highline", '~> 1.6', '>= 1.6.19'
 gem "fluentd", "0.10.46"
 gem "fluent-plugin-mysql-binlog", '~> 0.0', '>= 0.0.2'
 gem "mysql2", '~> 0.3', '>= 0.3.17'
+gem "pg", '~> 0.18.4'
 gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'

data/Gemfile.lock

@@ -79,6 +79,7 @@ GEM
       multi_json (~> 1.3)
       multi_xml (~> 0.5)
       rack (~> 1.2)
+    pg (0.18.4)
     polyglot (0.3.5)
     protected_attributes (1.0.8)
       activemodel (>= 4.0.1, < 5.0)
@@ -139,6 +140,7 @@ DEPENDENCIES
   json (~> 1.8, >= 1.8.0)
   kodama (~> 0.1, >= 0.1.8)
   mysql2 (~> 0.3, >= 0.3.17)
+  pg (~> 0.18.4)
   protected_attributes (~> 1.0, >= 1.0.8)
   pry
   rake-compiler (~> 0.9, >= 0.9.5)

data/VERSION

@@ -1 +1 @@
-0.6.4
+0.6.5

data/flydata-core/lib/flydata-core/table_def/base.rb

@@ -0,0 +1,31 @@
+module FlydataCore
+module TableDef
+
+class Base
+  def initialize(table_def, table_name, columns, column_def, default_charset,
+                 default_source_charset, comment)
+    @table_def = table_def
+    @table_name = table_name
+    @columns = columns
+    @column_def = column_def
+    @default_charset = default_charset
+    @default_source_charset = default_source_charset
+    @comment = comment
+  end
+
+  attr_reader :columns, :column_def, :table_name, :default_source_charset
+
+  def to_flydata_tabledef
+    tabledef = { table_name: @table_name,
+                 columns: @columns,
+               }
+    tabledef[:default_charset] = @default_charset if @default_charset
+    tabledef[:comment] = @comment if @comment
+    tabledef[:src_ddl] = @table_def
+
+    tabledef
+  end
+end
+
+end
+end

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -1,14 +1,16 @@
+require 'flydata-core/table_def/base'
+
 module FlydataCore
 module TableDef
 
-class MysqlTableDef
+class MysqlTableDef < Base
   # Check and set the varchar(char) size which is converted from
   # length to byte size.
   # On Mysql the record size of varchar(char) is a length of characters.
   # ex) varchar(6) on mysql -> varchar(18) on flydata
   PROC_override_varchar = ->(type, mysql_type, flydata_type) do
     return type unless %w(char varchar).include?(mysql_type)
-    if type =~ /\((\d+)\)/
+    if type =~ /\((\s*\d+\s*)\)/
       # expect 3 byte UTF-8 character
       "#{flydata_type}(#{$1.to_i * 3})"
     else
@@ -18,7 +20,7 @@ class MysqlTableDef
 
   PROC_override_varbinary = ->(type, mysql_type, flydata_type) do
     return type unless %w(binary varbinary).include?(mysql_type)
-    if type =~ /\((\d+)\)/
+    if type =~ /\((\s*\d+\s*)\)/     ###DEBUG/\((\d+)\)/
       # expect 2 bytes for each original byte + 2 bytes for the prefix
       # ex) 4E5DFF => "0x4e5dff"
       "#{flydata_type}(#{$1.to_i * 2 + 2})"
@@ -87,24 +89,13 @@ class MysqlTableDef
     params ? self.new(*params) : nil
   end
 
-  def initialize(table_def, table_name, columns, column_def, default_charset,
-                 default_charset_mysql, comment)
-    @table_def = table_def
-    @table_name = table_name
-    @columns = columns
-    @column_def = column_def
-    @default_charset = default_charset
-    @default_charset_mysql = default_charset_mysql
-    @comment = comment
-  end
-
   def self._create(io, options)
     table_def = ''
     table_name = nil
     columns = []
     column_def = {}
     default_charset = nil
-    default_charset_mysql = nil
+    default_source_charset = nil
     comment = nil
 
     position = :before_create_table
@@ -135,8 +126,8 @@ class MysqlTableDef
         #) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table';
         elsif stripped_line.start_with?(')')
           if line =~ /DEFAULT CHARSET\s*=\s*([a-z0-9]+)/
-            default_charset_mysql = $1
-            default_charset = flydata_charset(default_charset_mysql)
+            default_source_charset = $1
+            default_charset = flydata_charset(default_source_charset)
           end
           comment = $1 if /COMMENT='((?:\\'|[^'])*)'/.match(line)
           position = :after_create_table
@@ -157,21 +148,9 @@ class MysqlTableDef
         break
       end
     end
-    position == :after_create_table ? [table_def, table_name, columns, column_def, default_charset, default_charset_mysql, comment] : nil
+    position == :after_create_table ? [table_def, table_name, columns, column_def, default_charset, default_source_charset, comment] : nil
   end
-  attr_reader :columns, :column_def, :table_name, :default_charset_mysql
 
-  def to_flydata_tabledef
-    tabledef = { table_name: @table_name,
-                 columns: @columns,
-               }
-    tabledef[:default_charset] = @default_charset if @default_charset
-    tabledef[:comment] = @comment if @comment
-    tabledef[:src_ddl] = @table_def
-
-    tabledef
-  end
-  
   # Replaces a MySQL string with an empty string ('')
   def self.strip_string(line)
     line.gsub(/'(\\'|''|[^'])*'/, "''")

data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb

@@ -0,0 +1,111 @@
+require 'flydata-core/table_def/base'
+
+module FlydataCore
+module TableDef
+
+class PostgresqlTableDef < Base
+
+  VALUE_CONVERTERS = {}
+
+  TYPE_MAP_P2F = {
+    'bigint' => {type: 'int8'},
+    'character varying' => {type: 'varchar',
+                            width_attrs:["character_octet_length"]}, # TODO check def_width
+    'integer' => {type: 'int4'},
+    'timestamp with time zone' => {type: 'datetime'}, # TODO may need override?
+    'timestamp without time zone' => {type: 'datetime'}, # TODO may need override?
+    'bytea' => {type: 'varbinary'}, # TODO need value conversion
+    'numeric' => {type: 'numeric',
+                  width_attrs:["numeric_precision", "numeric_scale"],
+                  def_width:[nil, 0],
+                 },
+    # TODO add more types
+  }
+
+  def self.convert_to_flydata_type(information_schema_columns)
+    pg_type = information_schema_columns["data_type"]
+    unless TYPE_MAP_P2F.has_key?(pg_type)
+      raise "Unknown PostgreSQL type or internal error.  type:#{pg_type}"
+    end
+    type_hash = TYPE_MAP_P2F[pg_type]
+    flydata_type = type_hash[:type]
+    ret_type = flydata_type
+
+    width_values = get_width_values(information_schema_columns, type_hash)
+    if width_values
+      ret_type += "(#{width_values.join(",")})"
+    end
+
+    if type_hash[:override]
+      ret_type = type_hash[:override].call(ret_type, pg_type, flydata_type)
+    end
+    ret_type
+  end
+
+  def self.create(information_schema_columns, options)
+    params = _create(information_schema_columns, options)
+    params ? self.new(*params) : nil
+  end
+
+  private
+
+  def self.get_width_values(information_schema_columns, type_hash)
+    values = []
+    if type_hash.has_key?(:width_attrs)
+      values = type_hash[:width_attrs].collect{|attr| information_schema_columns[attr] }
+    end
+
+    if type_hash.has_key?(:def_width)
+      if values.nil? || values.size != type_hash[:def_width].size
+        raise "The number of the default values must match the number of width attributes def_width:#{type_hash[:def_width].inspect} width_attrs:#{type_hash[:width_attrs].inspect}"
+      end
+      values = values.each_with_index.collect {|v, i| v ? v : type_hash[:def_width][i]}
+    end
+    values.pop until values.empty? || values.last # remove trailing nil
+    if values.any?{|v| v.nil?}
+       raise "nil value is not allowed"
+    end
+    values.empty? ? nil : values
+  end
+
+  def self._create(information_schema_columns, options)
+    table_def = information_schema_columns.inspect
+    table_name = nil
+    columns = []
+    column_def = {}
+    # SourcePostgresql uses UTF8 client encoding no matter what the server
+    # encoding is.
+    default_charset = 'UTF_8'
+    default_charset_postgresql = 'UTF8'
+    comment = nil
+
+    information_schema_columns.each do |iscol|
+      column = parse_one_column_def(iscol)
+      if table_name
+        unless table_name == column[:table]
+          raise "Table name must match through all columns.  Got `#{table_name}` and `#{column[:table]}`"
+        end
+      else
+        table_name = column[:table]
+      end
+      columns << column
+      coldef = iscol.inspect
+      column_def[column[:column]] = coldef
+    end
+    [table_def, table_name, columns, column_def, default_charset, default_charset_postgresql, comment]
+  end
+
+  def self.parse_one_column_def(information_schema_column)
+    column = {}
+    column[:table] = information_schema_column["table_name"]
+    column[:column] = information_schema_column["column_name"]
+    column[:type] = convert_to_flydata_type(information_schema_column)
+    column[:not_null] = true if information_schema_column["is_nullable"] == "NO"
+    column[:primary_key] = true if information_schema_column["is_primary"]
+    column[:default] = information_schema_column["column_default"] # TODO nill handling
+    column
+  end
+end
+
+end
+end

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -229,7 +229,7 @@ EOS
 
     if (column.has_key?(:default))
       val = replace_default_value(type, type_info[:type], column[:default])
-      line += " DEFAULT #{val}"
+      line += " DEFAULT #{val}" if val
     elsif not_null && opt[:for] == :alter_table
       # Redshift doesn't allow adding a not null column without default value
       # Add a defalt value
@@ -275,6 +275,9 @@ EOS
         default_value.oct
       elsif /^'.*'$/.match(default_value)
         default_value
+      elsif /\s*nextval\(/.match(default_value)
+        # Redshift does not support nextval() function.  Set no default.
+        nil
       else
         "'#{default_value}'"
       end

data/flydata-core/spec/table_def/postgresql_table_def_spec.rb

@@ -0,0 +1,348 @@
+require 'spec_helper'
+require 'flydata-core/table_def/postgresql_table_def'
+
+module FlydataCore
+module TableDef
+
+describe PostgresqlTableDef do
+  let(:subject_object) {
+    described_class.new(table_def, table_name, columns, column_def,
+                        default_charset, default_charset_postgresql, comment)
+  }
+
+  let(:table_def) { double('table_def') }
+  let(:table_name) { double('table_name') }
+  let(:columns) { double('columns') }
+  let(:column_def) { double('column_def') }
+  let(:default_charset) { double('default_charset') }
+  let(:default_charset_postgresql) { double('default_charset_postgresql') }
+  let(:comment) { double('comment') }
+
+  let(:information_schema_columns) { {} }
+
+  describe '.convert_to_flydata_type' do
+    subject { described_class.convert_to_flydata_type(information_schema_columns) }
+
+    before do
+      information_schema_columns["data_type"] = data_type
+    end
+
+    let(:type_hash) { double('type_hash') }
+    let(:numeric_precision) { nil }
+    let(:numeric_scale) { nil }
+    let(:width_values) do
+      a = [ numeric_precision, numeric_scale ]
+      a.pop until a.empty? || a.last
+      a.empty? ? nil : a
+    end
+    before do
+      allow(described_class).to receive(:get_width_values).
+        with(information_schema_columns, type_hash).
+        and_return(width_values)
+    end
+
+    context "with data type numeric" do
+      let(:data_type) { "numeric" }
+
+      before do
+        information_schema_columns["numeric_precision"] = numeric_precision
+        information_schema_columns["numeric_scale"] = numeric_scale
+      end
+
+      let(:type_hash) { PostgresqlTableDef::TYPE_MAP_P2F[data_type] }
+
+      context "with precision nil" do
+        let(:numeric_precision) { nil }
+        context "with scale nil" do
+          let(:numeric_scale) { nil }
+          it { is_expected.to eq "numeric" }
+        end
+        context "with scale 2" do
+          let(:numeric_scale) { 2 }
+          it do
+            expect(described_class).to receive(:get_width_values).
+              with(information_schema_columns, type_hash).and_raise("test")
+
+            expect{subject}.to raise_error("test")
+          end
+        end
+      end
+      context "with precision 23" do
+        let(:numeric_precision) { 23 }
+        context "with scale nil" do
+          let(:numeric_scale) { nil }
+          it { is_expected.to eq "numeric(23)" }
+        end
+        context "with scale 2" do
+          let(:numeric_scale) { 2 }
+          it { is_expected.to eq "numeric(23,2)" }
+        end
+      end
+    end
+    # TODO Add data type specific specs here
+    context "with unknown data type" do
+      let(:data_type) { "duck" }
+      it { expect{subject}.to raise_error /Unknown PostgreSQL type/ }
+    end
+    context "with no data type" do
+      let(:data_type) { nil }
+      it { expect{subject}.to raise_error /Unknown PostgreSQL type/ }
+    end
+    context "with test data type" do
+      let(:data_type) { test_data_type }
+
+      before do
+        allow(PostgresqlTableDef::TYPE_MAP_P2F).to receive(:has_key?).
+          with(test_data_type).and_return true
+        allow(PostgresqlTableDef::TYPE_MAP_P2F).to receive(:[]).
+          with(test_data_type).and_return test_type_hash
+      end
+      let(:test_data_type) { double('test_data_type') }
+
+      let(:type_hash) { test_type_hash }
+      let(:test_type_hash) { {type: test_fd_type} }
+      let(:test_fd_type) { "testfdtype" }
+
+      let(:override_proc) { double('override_proc') }
+      let(:overriden_type) { double('overriden_type') }
+
+      context "no width values" do
+        let(:width_values) { nil }
+
+        it { is_expected.to eq test_fd_type }
+      end
+      context "single width value" do
+        let(:width_values) { [10] }
+
+        it { is_expected.to eq "#{test_fd_type}(10)" }
+      end
+      context "two width values" do
+        let(:width_values) { [10,2] }
+
+        it { is_expected.to eq "#{test_fd_type}(10,2)"}
+      end
+      context "with override proc" do
+        before do
+          test_type_hash[:override] = override_proc
+        end
+        it 'calls the override proc of the type and return its result' do
+          expect(override_proc).to receive(:call).
+            with(test_fd_type, test_data_type, test_fd_type).
+            and_return(overriden_type)
+
+          is_expected.to eq overriden_type
+        end
+      end
+    end
+  end
+
+  describe '.get_width_values(information_schema_columns, type_hash)' do
+    subject { described_class.send(:get_width_values,
+                                   information_schema_columns, type_hash) }
+
+    let(:type_hash) { {} }
+
+    context 'with no width attr' do
+      before do
+        type_hash.delete(:width_attrs)
+      end
+      context 'without default' do
+        before do
+          type_hash.delete(:def_width)
+        end
+
+        it { is_expected.to be_nil }
+      end
+      context 'with default' do
+        before do
+          type_hash[:def_width] = [10]
+        end
+
+        it  { expect{ subject }.to raise_error }
+      end
+    end
+    context 'with a single width attr' do
+      before do
+        type_hash[:width_attrs] = ["char_octet_length"]
+      end
+      context 'without default' do
+        before do
+          type_hash.delete(:def_width)
+        end
+        context 'when attr value exists' do
+          before do
+            information_schema_columns["char_octet_length"] = 1024
+          end
+
+          it { is_expected.to eq [ 1024 ] }
+        end
+        context 'when attr value does not exist' do
+          before do
+            information_schema_columns.delete("char_octet_length")
+          end
+
+          it { is_expected.to be_nil }
+        end
+      end
+      context 'with default' do
+        before do
+          type_hash[:def_width] = [10]
+        end
+        context 'when attr value exists' do
+          before do
+            information_schema_columns["char_octet_length"] = 1024
+          end
+
+          it { is_expected.to eq [1024] }
+        end
+        context 'when attr value does not exist' do
+          before do
+            information_schema_columns.delete("char_octet_length")
+          end
+
+          it { is_expected.to eq [10] }
+        end
+      end
+    end
+    context 'with two width attrs' do
+      before do
+        type_hash[:width_attrs] = ["precision", "scale"]
+      end
+      context 'without default' do
+        before do
+          type_hash.delete(:def_width)
+        end
+        context 'when the 1st attr value exists' do
+          before do
+            information_schema_columns["precision"] = 23
+          end
+          context 'when the 2nd attr value exists' do
+            before do
+              information_schema_columns["scale"] = 2
+            end
+
+            it { is_expected.to eq [23, 2] }
+          end
+          context 'when the 2nd attr value does not exist' do
+            before do
+              information_schema_columns.delete("scale")
+            end
+
+            it 'omits trailing nils' do
+              is_expected.to eq [23]
+            end
+          end
+        end
+        context 'when the 1st attr value does not exist' do
+          before do
+            information_schema_columns.delete("precision")
+          end
+          context 'when the 2nd attr value exists' do
+            before do
+              information_schema_columns["scale"] = 2
+            end
+
+            it { expect{subject}.to raise_error }
+          end
+          context 'when the 2nd attr value does not exist' do
+            before do
+              information_schema_columns.delete("scale")
+            end
+
+            it { is_expected.to be_nil }
+          end
+        end
+      end
+      context 'with default' do
+        before do
+          type_hash[:def_width] = [10, 0]
+        end
+        context 'when the 1st attr value exists' do
+          before do
+            information_schema_columns["precision"] = 23
+          end
+          context 'when the 2nd attr value exists' do
+            before do
+              information_schema_columns["scale"] = 2
+            end
+
+            it { is_expected.to eq [23, 2] }
+          end
+          context 'when the 2nd attr value does not exist' do
+            before do
+              information_schema_columns.delete("scale")
+            end
+
+            it { is_expected.to eq [23, 0] }
+          end
+        end
+        context 'when the 1st attr value does not exist' do
+          before do
+            information_schema_columns.delete("precision")
+          end
+          context 'when the 2nd attr value exists' do
+            before do
+              information_schema_columns["scale"] = 2
+            end
+
+            it { is_expected.to eq [10, 2] }
+          end
+          context 'when the 2nd attr value does not exist' do
+            before do
+              information_schema_columns.delete("scale")
+            end
+
+            it { is_expected.to eq [10, 0] }
+          end
+        end
+      end
+      context 'when the number of default values does not match
+               the attr number' do
+        before do
+          type_hash[:def_width] = [10]
+        end
+        context 'when the 1st attr value exists' do
+          before do
+            information_schema_columns["precision"] = 23
+          end
+          context 'when the 2nd attr value exists' do
+            before do
+              information_schema_columns["scale"] = 2
+            end
+
+            it { expect{subject}.to raise_error }
+          end
+          context 'when the 2nd attr value does not exist' do
+            before do
+              information_schema_columns.delete("scale")
+            end
+
+            it { expect{subject}.to raise_error }
+          end
+        end
+        context 'when the 1st attr value does not exist' do
+          before do
+            information_schema_columns.delete("precision")
+          end
+          context 'when the 2nd attr value exists' do
+            before do
+              information_schema_columns["scale"] = 2
+            end
+
+            it { expect{subject}.to raise_error }
+          end
+          context 'when the 2nd attr value does not exist' do
+            before do
+              information_schema_columns.delete("scale")
+            end
+
+            it { expect{subject}.to raise_error }
+          end
+        end
+      end
+    end
+  end
+end
+
+end
+end