flydata 0.6.14 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be893f658b13cbde2228307938f47f98d101f168
-  data.tar.gz: 4bdcaa201f75760393348f3910d333d91e9e7e6f
+  metadata.gz: b335ff2222169fc42a78d4dec2a5829aa0c94230
+  data.tar.gz: 15042a74de15a15b3784c6917973154d1ddd02c7
 SHA512:
-  metadata.gz: 6205f32c0d7199f0f285c7d6e61de96ee72e44498f1973bc37349af00564012c23640ef5d45a4e47bbd87464ef45155b144b93dd58ebf6d8860f48eb16478a7b
-  data.tar.gz: d90c9ee2fc5e362df5da45995ea3287b7845df17e46330d51115e0fcb5f506c3e3ca50d1d2c24f3203bac2de77412367dc5bd66184424affbb7634182e3b61be
+  metadata.gz: 86fb1aa9fe1449e11f0d8755773cd7a083054bb668df7dbf489973828b018f9ebe2e9a10850bcf8ca7afff7682f157cd7d3a3082907b458ad1b5cbba99cf73b5
+  data.tar.gz: d36c033143bdd9a1b5931d891406ec6dfbaeaeba70cdae08d78c3282ccf55bebbe3125921bec021bbbb9405907a52431ac0f220722fa10f771db421134c7f348

data/VERSION

@@ -1 +1 @@
-0.6.14
+0.7.0

data/flydata-core/Gemfile

@@ -9,6 +9,7 @@ group :development, :test do
    gem 'ruby-prof'
    gem 'rubocop'
    gem 'mysql2'
+   gem 'pg'
    gem 'rspec', '~> 3.1'
    gem 'simplecov', :require => false
 end

data/flydata-core/Gemfile.lock

@@ -17,6 +17,7 @@ GEM
     parser (2.2.0.pre.8)
       ast (>= 1.1, < 3.0)
       slop (~> 3.4, >= 3.4.5)
+    pg (0.18.4)
     powerpack (0.0.9)
     rainbow (2.0.0)
     rspec (3.1.0)
@@ -54,8 +55,12 @@ PLATFORMS
 DEPENDENCIES
   aws-sdk (= 1.18.0)
   mysql2
+  pg
   rspec (~> 3.1)
   rubocop
   ruby-prof
   simplecov
   timecop
+
+BUNDLED WITH
+   1.10.6

data/flydata-core/lib/flydata-core/errors.rb

@@ -317,12 +317,14 @@ end
 class CompatibilityError < StandardError
 end
 
-class AgentCompatibilityError < StandardError
+class AgentCompatibilityError < CompatibilityError
 end
 
-class MysqlCompatibilityError < StandardError
+class MysqlCompatibilityError < CompatibilityError
 end
 
+class PostgresqlCompatibilityError < CompatibilityError
+end
 
 ## Error container
 

data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb

@@ -29,6 +29,10 @@ class BinlogPos
     @pos = @pos.to_i
   end
 
+  def self.load(binlog_pos_str)
+    self.new(binlog_pos_str)
+  end
+
   attr_reader :filename, :pos
 
   def empty?

data/flydata-core/lib/flydata-core/postgresql/compatibility_checker.rb

@@ -0,0 +1,119 @@
+require 'flydata-core/errors'
+require 'flydata-core/postgresql/config'
+require 'pg'
+
+module FlydataCore
+  module Postgresql
+    class CompatibilityChecker
+      def initialize(option = {})
+        option ||= {}
+        @option = option.merge FlydataCore::Postgresql::Config.build_db_opts(option)
+      end
+
+      def do_check(option = @option, &block)
+        result = block.call create_query(option)
+        check_result(result, option)
+      end
+
+      # Override
+      #def create_query(option = @option)
+      #end
+
+      # Override
+      #def validate_result(result, option = @option)
+      #end
+    end
+
+    class PostgresqlCompatibilityChecker < CompatibilityChecker
+      def do_check(option = @option, &block)
+        query = create_query(option)
+        result = if block
+                   block.call query
+                 else
+                   exec_query(query)
+                 end
+        check_result(result, option)
+      end
+
+      def exec_query(query)
+        begin
+          client = PGconn.connect(FlydataCore::Postgresql::Config.opts_for_pg(@option))
+          client.exec(query)
+        ensure
+          client.close rescue nil if client
+        end
+      end
+
+      def schema_and_table_in_query(option = @option)
+        schema = if option[:schema].to_s.strip.empty?
+                   "select current_schema"
+                 else
+                   "'#{option[:schema]}'"
+                 end
+        {
+          schema_name: schema,
+          table_names: option[:tables].collect{|tn| "'#{tn}'"}.join(',')
+        }
+      end
+    end
+
+    class TableExistenceChecker < PostgresqlCompatibilityChecker
+      TABLE_EXISTENCE_CHECK_QUERY_TMPLT = <<EOT
+SELECT
+  table_name
+FROM
+  information_schema.tables
+WHERE
+  table_schema in (%{schema_name})
+  AND
+  table_name in (%{table_names});
+EOT
+
+      def create_query(option = @option)
+        TABLE_EXISTENCE_CHECK_QUERY_TMPLT % schema_and_table_in_query(option)
+      end
+
+      def check_result(result, option = @option)
+        existing_tables = []
+        result.each {|r| existing_tables << r['table_name']}
+        missing_tables = option[:tables] - existing_tables
+
+        unless missing_tables.empty?
+          raise FlydataCore::PostgresqlCompatibilityError,
+            "These tables are missing. Create these tables on your database or remove them from the data entry : #{missing_tables.join(", ")}"
+        end
+      end
+    end
+
+    class PrimaryKeyChecker < PostgresqlCompatibilityChecker
+      PK_CHECK_QUERY_TMPLT = <<EOT
+SELECT
+  t.table_name
+FROM
+  (select * from information_schema.tables where table_schema in (%{schema_name}) AND table_name in (%{table_names})) t
+  LEFT OUTER JOIN
+  (select * from information_schema.table_constraints where table_schema in (%{schema_name}) AND table_name in (%{table_names})) tc
+  USING (table_schema, table_name)
+GROUP BY
+  t.table_schema, t.table_name
+HAVING
+  SUM(CASE WHEN tc.constraint_type='PRIMARY KEY' THEN 1 ELSE 0 END) = 0;
+EOT
+
+      def create_query(option = @option)
+        PK_CHECK_QUERY_TMPLT % schema_and_table_in_query(option)
+      end
+
+      def check_result(result, option = @option)
+        missing_pk_tables = []
+        result.each {|r| missing_pk_tables << r['table_name']}
+
+        unless missing_pk_tables.empty?
+          raise FlydataCore::PostgresqlCompatibilityError,
+            "Primary key is required for tables to sync. " +
+            "Add primary key or remove following tables from data entry: #{missing_pk_tables.join(", ")}"
+        end
+      end
+    end
+  end
+end

data/flydata-core/lib/flydata-core/postgresql/config.rb

@@ -0,0 +1,58 @@
+module FlydataCore
+  module Postgresql
+    class Config
+      def self.build_db_opts(db_conf)
+        db_opts = [:host,
+                   :port,
+                   :username,
+                   :password,
+                   :database,
+                   :dbname,
+                   :user,
+                   :connect_timeout,
+                   :sslmode,
+                   :options,
+                   ].inject({}) { |h, sym|
+                     if db_conf.has_key?(sym)
+                       h[sym] = db_conf[sym]
+                     elsif db_conf[sym.to_s]
+                       h[sym] = db_conf[sym.to_s]
+                     end
+                     h
+        }
+
+        # for pg gem
+        update_opts(db_opts, :user, :username)
+        update_opts(db_opts, :dbname, :database)
+
+        db_opts
+      end
+
+      # Returns options for making pg connection
+      # pg does not accept unsupported keys
+      # http://www.rubydoc.info/gems/pg/PG%2FConnection%3Ainitialize
+      def self.opts_for_pg(db_conf)
+        db_opts = build_db_opts(db_conf)
+
+        [:host,
+         :port,
+         :user,
+         :password,
+         :dbname,
+         :connect_timeout,
+         :sslmode,
+         :options,
+        ].inject({}) do |ret, k|
+           ret[k] = db_opts[k] if db_opts[k]
+           ret
+        end
+      end
+
+      def self.update_opts(opts, dst_key, src_key)
+        if opts[dst_key].nil? && opts[src_key]
+          opts[dst_key] = opts[src_key]
+        end
+      end
+    end
+  end
+end

data/flydata-core/lib/flydata-core/postgresql/pg_client.rb

@@ -0,0 +1,170 @@
+require 'pg'
+require 'socket'
+require 'delegate'
+require 'flydata-core/postgresql/config'
+
+module FlydataCore
+module Postgresql
+
+class PGClient
+  PG_CONNECT_TIMEOUT = 10.0
+
+  def initialize(dbconf)
+    @dbconf = FlydataCore::Postgresql::Config.opts_for_pg(dbconf)
+  end
+
+  attr_reader :dbconf
+
+  def establish_connection
+    @conn = create_connection if @conn.nil?
+  end
+
+  def query(query, params = [])
+    establish_connection
+
+    if query.respond_to?(:placeholder_start_num) && query.placeholder_start_num
+      placeholders = placeholder_string(query.placeholder_size,
+                                        query.placeholder_start_num)
+      q = query % [placeholders]
+    else
+      q = query
+    end
+
+    if (params.nil? || params.empty?) && query.respond_to?(:binding_params)
+      params = query.binding_params || []
+    end
+
+    result = @conn.query(q, params)
+
+    query.respond_to?(:value_overriders) && query.value_overriders ?
+      EnumerableDelegator.new(result, HashValueOverrider, query.value_overriders)
+      : result
+  end
+
+  def close
+    if @conn
+      @conn.finish
+      @conn = nil
+    end
+  end
+
+  private
+
+  def create_connection
+    conn = nil
+    retry_on(Errno::EBADF, 3) do
+      hostaddr = IPSocket.getaddress(dbconf[:host])
+      dbconf[:hostaddr] = hostaddr
+      conn = PG::Connection.connect_start(dbconf)
+      raise PG::Error.new("Unable to create a new connection.") unless conn
+      raise PG::Error.new("Connection failed: %s" % [ conn.error_message ]) if conn.status == PG::CONNECTION_BAD
+
+      socket = conn.socket_io
+      poll_status = PG::PGRES_POLLING_WRITING
+      until poll_status == PG::PGRES_POLLING_OK || poll_status == PG::PGRES_POLLING_FAILED
+        case poll_status
+        when PG::PGRES_POLLING_READING
+          IO.select([socket], nil, nil, PG_CONNECT_TIMEOUT) or raise PG::Error.new("Asynchronous connection timed out!(READING)")
+        when PG::PGRES_POLLING_WRITING
+          IO.select(nil, [socket], nil, PG_CONNECT_TIMEOUT) or raise PG::Error.new("Asynchronous connection timed out!(WRITING)")
+        end
+        poll_status = conn.connect_poll
+      end
+    end
+    unless conn.status == PG::CONNECTION_OK
+      raise PG::Error.new("Connect failed: %s" % [ conn.error_message.to_s.lines.uniq.join(" ") ])
+    end
+    conn
+  rescue Errno::EBADF => e
+    raise PG::Error.new("Failed to connect redshift due to Errno::EBADF. #{e.to_s}")
+  rescue SocketError => e
+    if e.to_s == 'getaddrinfo: nodename nor servname provided, or not known'
+      raise PG::Error.new("Connection failed: FATAL:  unknown host(#{dbconf[:host]}).")
+    end
+    raise e
+  end
+
+  # Retry the given block if +exception+ happens
+  def retry_on(exception = StandardError, try_count = 3, interval = 1.0)
+    count = 0
+    begin
+      count += 1
+      yield
+    rescue exception
+      if count < try_count
+        sleep interval
+        interval *= 2
+        retry
+      else
+        raise
+      end
+    end
+  end
+
+  def placeholder_string(num_items, start_num)
+    num_items.times.collect{|i| "$#{i + start_num}"}.join(",")
+  end
+
+  class EnumerableDelegator
+    include Enumerable
+
+    def initialize(delegate, item_delegator_class, *args)
+      @delegate = delegate
+      @item_delegator_class = item_delegator_class
+      @args = args
+    end
+
+    def each(&block)
+      @delegate.each do |item|
+        block.call(@item_delegator_class.new(item, *@args))
+      end
+    end
+  end
+
+  class HashValueOverrider < SimpleDelegator
+    def initialize(delegate, overriders)
+      super(delegate)
+      @overriders = overriders
+    end
+
+    def [](key)
+      val = __getobj__[key]
+      override(key, val)
+    end
+
+    # TODO: all methods returning hash value(s) must be overridden
+
+    def first
+      key, val = __getobj__.first
+      [key, override(key, val)]
+    end
+
+    def values
+      __getobj__.keys.collect{|k| override(k, __getobj__[k]) }
+    end
+
+    def kind_of?(klass)
+      __getobj__.kind_of?(klass)
+    end
+
+    private
+
+    def override(key, val)
+      @overriders.has_key?(key) ? @overriders[key].call(val) : val
+    end
+  end
+end
+
+class PGQuery < SimpleDelegator
+  def initialize(query_text, opts)
+    super(query_text)
+    @value_overriders = opts[:value_overriders]
+    @placeholder_start_num = opts[:placeholder_start_num]
+    @placeholder_size = opts[:placeholder_size]
+    @binding_params = opts[:binding_params]
+  end
+  attr_reader :value_overriders, :placeholder_start_num, :placeholder_size, :binding_params
+end
+
+end
+end

data/flydata-core/lib/flydata-core/postgresql/snapshot.rb

@@ -0,0 +1,49 @@
+# Snapshot components
+# http://www.postgresql.org/docs/8.3/static/functions-info.html#FUNCTIONS-TXID-SNAPSHOT-PARTS
+#
+# Example) 10:20:10,14,15 means xmin=10, xmax=20, xip_list=10, 14, 15.
+
+module FlydataCore
+module Postgresql
+
+class Snapshot
+  include Comparable
+
+  def initialize(txid_snapshot)
+    @txid_snapshot_str = txid_snapshot.to_s
+    xmin_str, xmax_str, xip_list_str = @txid_snapshot_str.split(':')
+
+    raise ArgumentError, "Invalid snapshot - xmin is empty." if xmin_str.to_s.empty?
+    raise ArgumentError, "Invalid snapshot - xmax is empty." if xmax_str.to_s.empty?
+
+    @xmin = xmin_str.to_i
+    @xmax = xmax_str.to_i
+    @xip_list = xip_list_str.to_s.split(',').collect(&:to_i)
+  end
+
+  attr_reader :xmin, :xmax, :xip_list
+
+  def to_s
+    @txid_snapshot_str
+  end
+
+  def <=>(other)
+    if @xmin == other.xmin
+      if @xmax == other.xmax
+        # items xip_list will disappear after the transaction is completed,
+        # because xip_list is an active transaction list.
+        #
+        # The following comparison needs to be true
+        #   "10:18:10,11,12" < "10:18:11,12"
+        (other.xip_list.size <=> @xip_list.size)
+      else
+        @xmax <=> other.xmax
+      end
+    else
+      @xmin <=> other.xmin
+    end
+  end
+end
+
+end
+end