flydata 0.6.14 → 0.7.0

data/flydata-core/lib/flydata-core/postgresql/source_pos.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'flydata-core/postgresql/snapshot'
 
 module FlydataCore
 module Postgresql
@@ -6,27 +7,87 @@ module Postgresql
 class SourcePos
   include Comparable
 
-  def initialize(snapshot_id, pk_values = nil)
-    @snapshot_id = snapshot_id
-    @pk_values = pk_values
+  # Source Position data for PostgreSQL
+  #
+  # Format:
+  #   <from-snaphost><TAB><to-snapshot><TAB><pk-values(JSON)>
+  #
+  #   from-snapshot: Store txid_snapshot guaranteeing the changes up to this snapshot are sent
+  #   pk_values: Store primary key values of the last record on the last request/transaction
+  #
+  #
+  # ex1: No resume position
+  #
+  #   1031:1031:<TAB>
+  #
+  #   -> Changes up to 1031:1031: were sent.
+  #      Agent will fetch the change between the latest snapshot and 1031:1031:
+  #
+  # ex2: Resume position with single primary key
+  #
+  #   1031:1031:<TAB>1032:1032:<TAB>[{"id":10000}]
+  #
+  #   -> Changes up to 1031:1031: and partial changes (id <= 10000) between 1032:1032: and 1031:1031: were sent.
+  #      Agent will fetch the change between 1032:1032: and 1031:1031: with a primary key condisions(id > 10000)
+  #
+  # ex3: Resume position with multiple primary keys
+  #
+  #   1031:1031:<TAB>1033:1033:<TAB>[{"group":"apple"},{"name":"TOM"}]
+  #
+  #   -> Changes up to 1031:1031: and partial changes (group<='apple' and name<='TOM') between 1033:1033: and 1031:1031: were sent.
+  #      Agent will fetch the change between 1033:1033: and 1031:1031: with a primary key condisions(group >= 'apple' and name >= 'TOM')
+  #
+  def initialize(snapshot_or_obj, to_snapshot = nil, pk_values = nil)
+    if snapshot_or_obj.kind_of?(self.class)
+      snapshot_or_obj.tap do |s|
+        @snapshot = s.snapshot
+        @to_snapshot = s.to_snapshot
+        @pk_values = s.pk_values
+      end
+    else
+      @snapshot = Snapshot.new(snapshot_or_obj)
+      @to_snapshot = if to_snapshot.to_s.empty?
+                       nil
+                     else
+                       Snapshot.new(to_snapshot)
+                     end
+      @pk_values = pk_values  # must be array or nil
+    end
   end
 
-  attr_reader :snapshot_id
+  attr_reader :snapshot
+  attr_reader :to_snapshot
+  attr_reader :pk_values
 
   def to_s
     pk_values = @pk_values ? @pk_values.to_json : ''
-    "#{@snapshot_id}\t#{pk_values}"
+    "#{@snapshot}\t#{@to_snapshot}\t#{pk_values}"
   end
 
+  #TODO: Need to revisit the logic along with the spec
+  #  http://www.postgresql.org/docs/9.5/static/functions-info.html#FUNCTIONS-TXID-SNAPSHOT-PARTS
   def <=>(other)
-    # TODO take pk_values into account
-    @snapshot_id <=> other.snapshot_id
+    if @snapshot != other.snapshot
+      return @snapshot <=> other.snapshot
+    elsif @pk_values.nil? && !other.pk_values.nil?
+      1
+    elsif !@pk_values.nil? && other.pk_values.nil?
+      -1
+    elsif @pk_values == other.pk_values
+      0
+    else
+      @pk_values.to_s <=> other.pk_values.to_s
+    end
   end
 
   def self.load(str)
-    snapshot_id, pk_valules = str.split("\t")
-    pk_values = JSON.parse(pk_values) if pk_values
-    self.new(snapshot_id, pk_values)
+    snapshot, to_snapshot, pk_values = str.split("\t").collect{|v| v.strip}
+    pk_values = if pk_values.to_s.empty?
+                  nil
+                else
+                  JSON.parse(pk_values)
+                end
+    self.new(snapshot, to_snapshot, pk_values)
   end
 end
 

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -21,7 +21,7 @@ class MysqlTableDef < Base
 
   PROC_override_varbinary = ->(type, mysql_type, flydata_type) do
     return type unless %w(binary varbinary).include?(mysql_type)
-    if type =~ /\((\s*\d+\s*)\)/     ###DEBUG/\((\d+)\)/
+    if type =~ /\((\s*\d+\s*)\)/
       # expect 2 bytes for each original byte + 2 bytes for the prefix
       # ex) 4E5DFF => "0x4e5dff"
       "#{flydata_type}(#{$1.to_i * 2 + 2})"

data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb

@@ -9,39 +9,82 @@ class PostgresqlTableDef < Base
 
   TYPE_MAP_P2F = {
     'bigint' => {type: 'int8'},
-    'bigserial' => {type: 'int8'},
-    'boolean' => {type: 'int1', def_width: '1'}, # TODO introduce bool type
-    'bytea' => {type: 'varbinary'}, # TODO need value conversion
-    'character' => {type: 'varchar', def_width: '1'},
+    'int8' => {type: 'int8'},
+    'bigserial' => {type: 'serial8'}, # TODO support
+    'serial8' => {type: 'serial8'}, # TODO support
+    'bit' => {type: 'bit',
+              width_attrs:["character_maximum_length"]},
+    'bit varying' => {type: 'varbit',
+                      width_attrs:["character_maximum_length"]},
+    'varbit' => {type: 'varbit',
+                 width_attrs:["character_maximum_length"]},
+    'boolean' => {type: 'boolean'},
+#    'box'
+    'bytea' => {type: 'bytea'},
+    'character' => {type: 'varchar', width_attrs:["character_octet_length"], def_width:[1]},
     'character varying' => {type: 'varchar',
                             width_attrs:["character_octet_length"],
                             def_width:[1]},
+    'varchar' => {type: 'varchar',
+                            width_attrs:["character_octet_length"],
+                            def_width:[1]},
+#    'cidr'
+#    'circle'
     'date' => {type: 'date'},
-    'decimal' => {type: 'numeric',
-                  width_attrs:["numeric_precision", "numeric_scale"],
-                  def_width:[nil, 0] },
     'double precision' => {type: 'float8'},
+    'float8' => {type: 'float8'},
     'integer' => {type: 'int4'},
-    'money' => {type: 'money'},
+    'int' => {type: 'int4'},
+    'int4' => {type: 'int4'},
+#    'interval'
+#    'json'
+#    'line'
+#    'lseg'
+#    'macaddr'
+    'money' => {type: 'money',
+                width_attrs:["numeric_precision", "numeric_scale"]},
     'numeric' => {type: 'numeric',
-                  width_attrs:["numeric_precision", "numeric_scale"],
-                  def_width:[nil, 0],
+                  width_attrs:["numeric_precision", "numeric_scale"]
+                  #can be no width values
                  },
+#    'path'
+#    'pg_lsn'
+#    'point'
+#    'polygon'
     'real' => {type: 'float4'},
-    'serial' => {type: 'int4'},
+    'float4' => {type: 'float4'},
     'smallint' => {type: 'int2'},
+    'int2' => {type: 'int2'},
+    'smallserial' => {type: 'serial2'},  # TODO support
+    'serial2' => {type: 'serial2'}, # TODO support
+    'serial' => {type: 'serial4'},       # TODO support
+    'serial4' => {type: 'serial4'},       # TODO support
     'text' => {type: 'text'},
-    'time with time zone' => {type: 'time'},
+    'time' => {type: 'time'},
     'time without time zone' => {type: 'time'},
-    'timestamp with time zone' => {type: 'datetime'},
+    'time with time zone' => {type: 'timetz'},
+    'timetz' => {type: 'timetz'},
+    'timestamp' => {type: 'datetime'},
     'timestamp without time zone' => {type: 'datetime'},
-    # TODO add more types
+    'timestamp with time zone' => {type: 'datetimetz'},
+    'timestamptz' => {type: 'datetimetz'},
+    :default => {type: '_unsupported'},
+#    'tsquery'
+#    'tsvector'
+#    'txid_snapshot'
+#    'uuid'
+#    'xml'
+#--- PosgreSQL automatically converts following data types to another data type ---
+#    'decimal' (converted to 'numeric')
+#    'bool'    ('boolean')
+#    'char'    ('character')
   }
 
   def self.convert_to_flydata_type(information_schema_columns)
     pg_type = information_schema_columns["data_type"]
+    raise "Unknown PostgreSQL type or internal error.  type:#{pg_type}" unless pg_type
     unless TYPE_MAP_P2F.has_key?(pg_type)
-      raise "Unknown PostgreSQL type or internal error.  type:#{pg_type}"
+      pg_type = :default
     end
     type_hash = TYPE_MAP_P2F[pg_type]
     flydata_type = type_hash[:type]
@@ -73,7 +116,7 @@ class PostgresqlTableDef < Base
 
     if type_hash.has_key?(:def_width)
       if values.nil? || values.size != type_hash[:def_width].size
-        raise "The number of the default values must match the number of width attributes def_width:#{type_hash[:def_width].inspect} width_attrs:#{type_hash[:width_attrs].inspect}"
+        raise "The number of the default values must match the number of width attributes. column:#{information_schema_columns[:column_name]} type:#{type_hash[:type]} def_width:#{type_hash[:def_width].inspect} width_attrs:#{type_hash[:width_attrs].inspect}"
       end
       values = values.each_with_index.collect {|v, i| v ? v : type_hash[:def_width][i]}
     end
@@ -118,9 +161,25 @@ class PostgresqlTableDef < Base
     column[:type] = convert_to_flydata_type(information_schema_column)
     column[:not_null] = true if information_schema_column["is_nullable"] == "NO"
     column[:primary_key] = true if information_schema_column["is_primary"]
-    column[:default] = information_schema_column["column_default"] # TODO nill handling
+    column[:default] = case column[:type]
+      when 'boolean'
+        to_boolean(information_schema_column["column_default"])
+      else
+        information_schema_column["column_default"] # TODO nil handling
+      end
     column
   end
+
+  def self.to_boolean(col_value)
+    return nil if col_value.nil?
+    # Catch all possible literal for boolean type in PostgreSQL.
+    # Actual col_value coming in here is:
+    #   'true' or 'false' for default value
+    #   't' or 'f' for column value
+    return true if col_value.to_s =~ /^(t|true|y|yes|on|1)$/i
+    return false if col_value.to_s =~ /^(f|false|n|no|off|0)$/i
+    raise "Invalid default value for PostgreSQL boolean type:`#{col_value}`"
+  end
 end
 
 end

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -10,9 +10,13 @@ class RedshiftTableDef
   TYPE_MAP_F2R = {
     'binary' => {type: 'varchar', use_params: true, default_value: ''},
     'bit' => {type: 'bigint', default_value: '0'},
+    'boolean' => {type: 'boolean', default_value: false},
+    'varbit' => {type: 'bigint', default_value: '0'},
+    'bytea' => {type: 'varchar(max)', default_value: ''},
     'char' => {type: 'char', use_params: true, default_value: ''},
     'date' => {type: 'date', default_value: '0000-01-01'},
     'datetime' => {type: 'timestamp', default_value: '0000-01-01'},
+    'datetimetz' => {type: 'timestamp', default_value: '0000-01-01'},
     'enum' => {type: 'varchar encode bytedict', default_value: ''},
     'float4' => {type: 'float4', default_value: '0'},
     'float4 unsigned' => {type: 'float4', default_value: '0'},
@@ -28,14 +32,17 @@ class RedshiftTableDef
     'int4 unsigned' => {type: 'int8', unsigned: true, default_value: '0'},
     'int8' => {type: 'int8', default_value: '0'},
     'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true, default_value: '0'},
-    'numeric' => {type: 'numeric', use_params: true, max_size: [38,37], default_value: '0'},
-    'numeric unsigned' => {type: 'numeric', use_params: true, max_size: [38,37], default_value: '0'},
+    'money' => {type: 'numeric', use_params: true, max_size: [38,37], default_value: '0'},
+    'numeric' => {type: 'numeric', use_params: true, default_params: [18,8], max_size: [38,37], default_value: '0'},
+    'numeric unsigned' => {type: 'numeric', use_params: true, default_params: [18,8], max_size: [38,37], default_value: '0'},
     'set' => {type: 'varchar encode bytedict', default_value: ''},
     'text' => {type: 'varchar(max)', default_value: ''},
     'time' => {type: 'timestamp', default_value: '0000-01-01'},
+    'timetz' => {type: 'timestamp', default_value: '0000-01-01'},
     'varbinary' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
     'varchar' => {type: 'varchar', use_params: true, max_size: 65535, default_value: ''},
     'year' => {type: 'date', default_value: '0001-01-01'},
+    '_unsupported' => {type: 'varchar(max)', default_value: ''},
   }
   def self.from_flydata_tabledef(flydata_tabledef, options = {})
     options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
@@ -209,8 +216,14 @@ EOS
     type_info = TYPE_MAP_F2R[type]
     raise "Unsupported type '#{column[:type]}'" if type_info.nil?
 
-    rs_type = if type_info[:use_params] && params && !params.nil?
-                params = check_and_replace_max(params, Array(type_info[:max_size])) if type_info[:max_size]
+    rs_type = if type_info[:use_params]
+                if params
+                  params = check_and_replace_max(params, Array(type_info[:max_size])) if type_info[:max_size]
+                else
+                  #source data type has no parameter. use default parameters.
+                  raise "No default pramameters for type:`#{column[:type]}`" unless type_info[:default_params]
+                  params = type_info[:default_params].join(',')
+                end
                 type_info[:type] + "(#{params})"
               else
                 type_info[:type]
@@ -246,6 +259,11 @@ EOS
 
   def self.replace_default_value(flydata_type, redshift_type, default_value)
     return NULL_STR if default_value.nil?
+    # strip type cast
+    if default_value.kind_of?(String) &&
+       /^(.+?)(::"?[a-z ]+"?)*$/.match(default_value)
+      default_value = $1
+    end
     if flydata_type.start_with?('year')
       value = convert_year_into_date(remove_single_quote(default_value))
       begin
@@ -253,31 +271,36 @@ EOS
       rescue
         raise "default value of YEAR type must be 2 or 4-digit, value:'#{default_value}'"
       end
+    elsif flydata_type.start_with?('money')
+      default_value = self.parse_money(remove_single_quote(default_value))
     end
 
     case redshift_type
     when 'timestamp'
-      if /^CURRENT_TIMESTAMP/ === default_value.upcase
+      case default_value.upcase
+      when /^CURRENT_TIMESTAMP\b/, /^CURRENT_TIME\b/, /^NOW\(\)/
         'SYSDATE'
       else
         "'#{self.parse_timestamp(remove_single_quote(default_value))}'"
       end
     when 'date'
       "'#{self.parse_date(remove_single_quote(default_value))}'"
+    when 'boolean'
+      default_value.to_s.upcase
     else
       if !default_value.kind_of?(String)
         "'#{default_value}'"
-      elsif /^b'.+'$/.match(default_value)
+      elsif /^[Bb]'.+'$/.match(default_value)
         "0b#{default_value[2..-2]}".oct
       elsif /^[xX]'.+'$/.match(default_value)
         "0x#{default_value[2..-2]}".oct
       elsif /^0[bx].+/.match(default_value)
         default_value.oct
-      elsif /^'.*'$/.match(default_value)
-        default_value
       elsif /\s*nextval\(/.match(default_value)
         # Redshift does not support nextval() function.  Set no default.
         nil
+      elsif /^'.+'/.match(default_value)
+        default_value
       else
         "'#{default_value}'"
       end
@@ -400,7 +423,7 @@ EOS
   end
 
   APACHE_TIMESTAMP_REGEXP = Regexp.new('^(?<apache_time_format>\[[0-3]\d\/\D{3}\/[1-2]\d{3}:[0-2]\d:[0-5]\d:[0-5]\d ?[\+\-]\d{2}:?\d{2}\])$')
-  TIME_REGEXP = Regexp.new('^(?<sign>-)?(?<hour>\d{2,3}):(?<minute>[0-5][0-9]):(?<second>[0-5][0-9](\.\d+)?)$')
+  TIME_REGEXP = Regexp.new('^(?<sign>-)?(?<hour>\d{2,3}):(?<minute>[0-5][0-9]):(?<second>[0-5][0-9](\.\d+)?)\s*(?<tz>[+-][0-2][0-9](:?\d{2})?)?$')
 
   def self.parse_timestamp(value)
     value_str = value.to_s
@@ -412,7 +435,13 @@ EOS
       # apache time format
       t = DateTime.strptime(value, "[%d/%b/%Y:%H:%M:%S %Z]")
     elsif time_match = TIME_REGEXP.match(value_str)
-      t = convert_time_into_timestamp(time_match)
+      if time_match[:tz]
+        # let DateTime.parse handle timezone
+        t = DateTime.parse("0001-01-01 #{value_str}")
+      else
+        # use own converter which takes care of negative hours
+        t = convert_time_into_timestamp(time_match)
+      end
     elsif /^(\d+)\.(\d+)$/ === value_str # epoch with fraction
       epoch = $1
       fraction = $2
@@ -477,6 +506,26 @@ EOS
       value # Return the value as is
     end
   end
+
+  def self.parse_money(value)
+    # It's impossible to determine a decimal point of a currency sting without
+    # knowing its format.  Here, we're making a best effort guess to support
+    # as many currencty formats as we can without knowing the format.
+    value = value.gsub(/[^0-9\.,]/, '') # remove all chars but numbers and possible decimal point chars
+
+    whole_part = value
+    fractional_part = nil
+    if idx = value.rindex(/[\.,]/)
+      if idx == value.size - 3
+        # it's the decimal point with two digit fraction
+        whole_part = value[0...idx]
+        fractional_part = value[-2..-1]
+      end
+    end
+    value = whole_part.gsub(/[^0-9]/, '')
+    value += ".#{fractional_part}" if fractional_part
+    value
+  end
 end
 
 end

data/flydata-core/spec/mysql/binlog_pos_spec.rb

@@ -8,7 +8,7 @@ module Mysql
     let(:binlog_str) { "#{filename}\t#{pos}" }
     let(:filename) { 'mysql-bin.000064' }
     let(:pos) { 104 }
-    
+
     describe '#pos' do
       subject { subject_object.pos }
       it { is_expected.to eq(pos) }
@@ -35,7 +35,7 @@ module Mysql
         let(:args) { [filename, pos] }
         it_behaves_like "properly constructed"
       end
-    
+
       let(:args) { [ arg1 ] }
       context 'when given a String' do
         let(:arg1) { binlog_str }
@@ -48,6 +48,14 @@ module Mysql
       end
     end
 
+    describe '.load' do
+      subject { described_class.load(binlog_str) }
+      it do
+        expect(subject.filename).to eq(filename)
+        expect(subject.pos).to eq(pos)
+      end
+    end
+
     let(:arg) { described_class.new (arg_str) }
     let(:arg_str) { arg_binlog_str }
     let(:arg_binlog_str) { "#{arg_filename}\t#{arg_pos}" }

data/flydata-core/spec/postgresql/compatibility_checker_spec.rb

@@ -0,0 +1,148 @@
+require 'spec_helper'
+require 'flydata-core/postgresql/compatibility_checker'
+
+module FlydataCore
+  module Postgresql
+    describe CompatibilityChecker do
+      let(:database) { 'test_db' }
+      let(:tables) { %w(table_1 table_2 table_3) }
+      let(:option) do
+        { database: database, tables: tables }
+      end
+
+      describe TableExistenceChecker do
+        let(:subject_object) { described_class.new(option) }
+
+        describe '#create_query' do
+          subject { subject_object.create_query }
+
+          context 'when schema is not set' do
+            it { is_expected.to eq <<EOT
+SELECT
+  table_name
+FROM
+  information_schema.tables
+WHERE
+  table_schema in (select current_schema)
+  AND
+  table_name in ('table_1','table_2','table_3');
+EOT
+            }
+          end
+          context 'when schema is set' do
+            before { option.merge!(schema: 'test_schema') }
+            it { is_expected.to eq <<EOT
+SELECT
+  table_name
+FROM
+  information_schema.tables
+WHERE
+  table_schema in ('test_schema')
+  AND
+  table_name in ('table_1','table_2','table_3');
+EOT
+            }
+          end
+
+        end
+
+        describe '#check_reesult' do
+          let(:result) { [] }
+          subject { subject_object.check_result(result) }
+
+          context 'when all tables exist' do
+            let(:result) do
+              [ {'table_name' => 'table_1'},
+                {'table_name' => 'table_2'},
+                {'table_name' => 'table_3'}, ]
+            end
+            it { expect{subject}.not_to raise_error }
+          end
+
+          context 'when some tables does not exist' do
+            let(:result) do
+              [{"table_name" => "table_2"}]
+            end
+            it do
+              expect{subject}.to raise_error {|e|
+                expect(e).to be_kind_of(FlydataCore::PostgresqlCompatibilityError)
+                expect(e.to_s.include?("table_1")).to be_truthy
+                expect(e.to_s.include?("table_2")).to be_falsey
+                expect(e.to_s.include?("table_3")).to be_truthy
+              }
+            end
+          end
+        end
+      end
+
+      describe PrimaryKeyChecker do
+        let(:subject_object) { described_class.new(option) }
+
+        describe '#create_query' do
+          subject { subject_object.create_query }
+
+          context 'when schema is not set' do
+            it { is_expected.to eq <<EOT
+SELECT
+  t.table_name
+FROM
+  (select * from information_schema.tables where table_schema in (select current_schema) AND table_name in ('table_1','table_2','table_3')) t
+  LEFT OUTER JOIN
+  (select * from information_schema.table_constraints where table_schema in (select current_schema) AND table_name in ('table_1','table_2','table_3')) tc
+  USING (table_schema, table_name)
+GROUP BY
+  t.table_schema, t.table_name
+HAVING
+  SUM(CASE WHEN tc.constraint_type='PRIMARY KEY' THEN 1 ELSE 0 END) = 0;
+EOT
+            }
+          end
+
+          context 'when schema is set' do
+            before { option.merge!(schema: 'test_schema') }
+            it { is_expected.to eq <<EOT
+SELECT
+  t.table_name
+FROM
+  (select * from information_schema.tables where table_schema in ('test_schema') AND table_name in ('table_1','table_2','table_3')) t
+  LEFT OUTER JOIN
+  (select * from information_schema.table_constraints where table_schema in ('test_schema') AND table_name in ('table_1','table_2','table_3')) tc
+  USING (table_schema, table_name)
+GROUP BY
+  t.table_schema, t.table_name
+HAVING
+  SUM(CASE WHEN tc.constraint_type='PRIMARY KEY' THEN 1 ELSE 0 END) = 0;
+EOT
+            }
+          end
+        end
+
+        describe '#check_reesult' do
+          let(:result) { [] }
+          subject { subject_object.check_result(result) }
+
+          context 'when all tables have pk' do
+            let(:result) do
+              []
+            end
+            it { expect{subject}.not_to raise_error }
+          end
+
+          context 'when some tables does not have pk' do
+            let(:result) do
+              [{"table_name" => "table_1"}, {"table_name" => "table_3"}]
+            end
+            it do
+              expect{subject}.to raise_error {|e|
+                expect(e).to be_kind_of(FlydataCore::PostgresqlCompatibilityError)
+                expect(e.to_s.include?("table_1")).to be_truthy
+                expect(e.to_s.include?("table_2")).to be_falsey
+                expect(e.to_s.include?("table_3")).to be_truthy
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end