flydata 0.6.14 → 0.7.0

data/lib/flydata/source_mysql/plugin_support/source_position_file.rb

@@ -0,0 +1,14 @@
+require 'flydata/plugin_support/source_position_file'
+require 'flydata-core/mysql/binlog_pos'
+
+module Flydata
+module SourceMysql
+
+module PluginSupport
+  class SourcePositionFile < Flydata::PluginSupport::SourcePositionFile
+    SOURCE_POS_CLASS = FlydataCore::Mysql::BinlogPos
+  end
+end
+
+end
+end

data/lib/flydata/source_mysql/table_ddl.rb

@@ -50,8 +50,8 @@ EOS
       if binlog_pos.nil?
         # get binlog position
         binlog_pos = FlydataCore::Mysql::BinlogPos.new(File.open(position_file){|f| f.read })
-        original_binlog_file = context.current_binlog_file
-        context.current_binlog_file = binlog_pos.filename
+        original_binlog_file = context.cur_src_pos_file
+        context.cur_src_pos_file = binlog_pos.filename
       end
       # get charset
       charset = mysql_tabledef.default_source_charset
@@ -77,7 +77,7 @@ EOS
       # update generated_ddl
       sync_fm.save_generated_ddl([table], V2_TARGET_VERSION.to_s)
     end
-    context.current_binlog_file = original_binlog_file if binlog_pos
+    context.cur_src_pos_file = original_binlog_file if binlog_pos
   end
 end
 

data/lib/flydata/source_mysql/{plugin_support/table_meta.rb → table_meta.rb}

@@ -1,3 +1,4 @@
+require 'flydata/table_meta'
 require 'mysql2'
 require 'flydata-core/mysql/config'
 require 'flydata-core/table_def/mysql_table_def'
@@ -5,8 +6,7 @@ require 'flydata-core/table_def/mysql_table_def'
 module Flydata
 module SourceMysql
 
-module PluginSupport
-  class TableMeta
+  class TableMeta < Flydata::TableMeta
     MANDATORY_OPTS = [
       :host, :port, :username, :password,
       :database, :tables,
@@ -38,7 +38,7 @@ EOT
       @table_meta = Hash.new{|h, k| h[k] = {}}
     end
 
-    def update
+    def reload
       conn = Mysql2::Client.new(@db_opts)
       sql = GET_TABLE_META_SQL % {
         database: @database, tables: @tables.collect{|t| "'#{t}'"}.join(',') }
@@ -51,14 +51,7 @@ EOT
     ensure
       conn.close rescue nil if conn
     end
-
-    # Return table meta
-    #   :character_set_name
-    def [](table_name)
-      @table_meta[table_name.to_sym]
-    end
   end
 end
 
 end
-end

data/lib/flydata/source_postgresql/generate_source_dump.rb

@@ -1,8 +1,10 @@
 require 'flydata/source/generate_source_dump'
 require 'flydata/preference/data_entry_preference'
 require 'flydata/source_postgresql/postgresql_component'
-require 'flydata/source_postgresql/pg_client'
+require 'flydata/source_postgresql/query_based_sync/diff_query_generator'
+require 'flydata/source_postgresql/table_meta'
 require 'flydata-core/postgresql/source_pos'
+require 'flydata-core/postgresql/pg_client'
 require 'msgpack'
 
 module Flydata
@@ -41,42 +43,49 @@ EOS
   TABLE_PLACEHOLDER_START_NUM = 2 # because $1 is used by table_schema
 
   def dump_size(tables)
-    cli = PGClient.new(de_prefs)
+    cli = FlydataCore::Postgresql::PGClient.new(de_prefs)
 
-    res = cli.query(DUMP_SIZE_QUERY, [de_prefs['schema']] + tables, placeholder_size: tables.size, placeholder_start_num: TABLE_PLACEHOLDER_START_NUM)
+    query = FlydataCore::Postgresql::PGQuery.new(DUMP_SIZE_QUERY,
+              placeholder_size: tables.size,
+              placeholder_start_num: TABLE_PLACEHOLDER_START_NUM)
+    res = cli.query(query, [de_prefs['schema']] + tables)
 
     res.first['total_size'].to_i
+
+  ensure
+    cli.close if cli
   end
 
   def dump(tables, file_path = nil, &src_pos_callback)
+    io = nil
     if file_path
       io = File.open(file_path, "w")
     else
       raise "dump via pipe has not been implemented yet"
     end
 
-    cli = PGClient.new(de_prefs)
-
-    source_pos = get_source_pos(cli, &src_pos_callback)
+    table_meta = Flydata::SourcePostgresql::TableMeta.new(de_prefs, tables)
+    cli = FlydataCore::Postgresql::PGClient.new(de_prefs)
+    table_meta.reload(cli)
 
     context = source.sync_generate_table_ddl(dp, nil)
-    missing_tables = context.each_source_tabledef(tables, de_prefs) do |tabledef, error|
+    source_pos = get_source_pos(table_meta.current_snapshot, &src_pos_callback)
+
+    options = de_prefs.merge(table_meta: table_meta)
+    missing_tables = context.each_source_tabledef(tables, options) do |tabledef, error|
       dump_table(tabledef, source_pos, io, cli) if tabledef
     end
+
     nil
+  ensure
+    cli.close if cli
+    io.close if io
   end
 
   private
 
-  CURRENT_SNAPSHOT_QUERY = <<EOS
-SELECT txid_current_snapshot() AS current_snapshot;
-EOS
-
-  def get_source_pos(cli, &src_pos_callback)
-    res = cli.query(CURRENT_SNAPSHOT_QUERY)
-
-    current_snapshot = res.first['current_snapshot']
-    src_pos = FlydataCore::Postgresql::SourcePos.new(current_snapshot)
+  def get_source_pos(snapshot, &src_pos_callback)
+    src_pos = FlydataCore::Postgresql::SourcePos.new(snapshot)
     src_pos_callback.call(nil, src_pos)
 
     src_pos
@@ -85,16 +94,13 @@ EOS
   NUM_ROWS = 500000
 
   def dump_table(tabledef, source_pos, io, cli)
-    fqtn = fq_table_name(de_prefs['schema'], tabledef.table_name)
-    pk_columns = tabledef.pk_columns
-    columns = tabledef.column_names
-
     dump_source_table(tabledef, io)
 
     last_pks = nil
     loop do
-      num_rows, last_pks = dump_table_chunk(fqtn, source_pos, NUM_ROWS,
-                                            columns, pk_columns, last_pks, io, cli)
+      num_rows, last_pks =
+        dump_table_chunk(tabledef.table_name, de_prefs['schema'], source_pos, NUM_ROWS,
+                         tabledef, tabledef.pk_columns, last_pks, io, cli)
       break if num_rows < NUM_ROWS
     end
   end
@@ -112,24 +118,29 @@ EOS
     io.write(source_table_hash.to_msgpack)
   end
 
-  SELECT_QUERY = <<EOS
-SELECT * FROM %s%s%s LIMIT %s;
-EOS
-
-  def dump_table_chunk(fqtn, source_pos, num_rows, columns, pk_columns, last_pks, io, cli)
-    where_clause = build_where_clause(source_pos, pk_columns, last_pks)
-    params = last_pks || []
-    order_by_clause = build_order_by_clause(pk_columns)
-    query = SELECT_QUERY % [fqtn, where_clause, order_by_clause, num_rows]
+  def dump_table_chunk(table, schema, source_pos, num_rows,
+                       tabledef, pk_columns, last_pks, io, cli)
+    query = Flydata::SourcePostgresql::QueryBasedSync::DiffQueryGenerator.new(
+              table, schema,
+              columns: tabledef.columns,
+              to_sid: source_pos.snapshot,
+              pk_columns: pk_columns,
+              last_pks: last_pks,
+              limit: num_rows).build_query
+
+    column_names = tabledef.columns.inject([]) do |ary, h|
+       ary << h[:column]
+    end
 
-    res = cli.query(query, params)
+    res = cli.query(query)
     count = 0
     last_row = nil
     res.each do |row|
       count += 1
       last_row = row
-      dump_row(row, columns, io)
+      dump_row(row, column_names, io)
     end
+
     last_pks = last_row ? pk_columns.collect{|col| last_row[col]} : nil
 
     [count, last_pks]
@@ -139,36 +150,6 @@ EOS
     data = columns.collect{|col| row[col] }.to_msgpack
     io.write(data)
   end
-
-  def build_where_clause(source_pos, pk_columns, last_pks)
-    clause = " WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '#{source_pos.snapshot_id}'::TXID_SNAPSHOT)"
-    if last_pks
-      clause += pk_conditions(pk_columns)
-    end
-  end
-
-  def build_order_by_clause(pk_columns)
-    " ORDER BY #{pk_columns.collect{|col| %Q|"#{col}"| }.join(",")}"
-  end
-
-  def pk_conditions(pk_columns)
-    i = pk_columns.size - 1
-    str = nil
-    while i >= 0
-      colname = pk_columns[i]
-      if str
-        str = %Q|"#{colname}" > $#{i + 1} OR ("#{colname}" = $#{i + 1} AND (#{str}))|
-      else
-        str = %Q|"#{colname}" > $#{i + 1}|
-      end
-      i -= 1
-    end
-    " AND (#{str})"
-  end
-
-  def fq_table_name(schema, table)
-    schema ? %Q|"#{schema}"."#{table}"| : %Q|"#{table}"|
-  end
 end
 
 end

data/lib/flydata/source_postgresql/parse_dump_and_send.rb

@@ -87,6 +87,7 @@ class DumpParser
 
   # {"table_name"=>"users", "columns"=>{"id"=>{"column_name"=>"id", "format_type"=>"bigint"}, "name"=>{"column_name"=>"name", "format_type"=>"character varying"}, "another_id"=>{"column_name"=>"another_id", "format_type"=>"integer"}}}
   def handle_table_info(table_info)
+    call_insert_record_block # flush rows if any
     call_check_point_block(Parser::State::CREATE_TABLE)
     columns = table_info["columns"].inject({}) do |h, (k, v)|
                 h[k] = %w(column_name format_type).inject({}) do |hh, kk|
@@ -114,6 +115,7 @@ class DumpParser
   end
 
   def call_insert_record_block
+    return if @rows.empty?
     if @insert_record_block.call(@current_table, @rows)
       call_check_point_block(Parser::State::INSERT_RECORD)
     end

data/lib/flydata/source_postgresql/plugin_support/context.rb

@@ -0,0 +1,13 @@
+require 'flydata/plugin_support/context'
+require 'flydata-core/postgresql/source_pos'
+
+module Flydata
+module SourcePostgresql
+module PluginSupport
+  class Context < ::Flydata::PluginSupport::Context
+    register_mandatory_opts :database, :dbconf
+    SOURCE_POS_CLASS = ::FlydataCore::Postgresql::SourcePos
+  end
+end
+end
+end

data/lib/flydata/source_postgresql/plugin_support/source_position_file.rb

@@ -0,0 +1,14 @@
+require 'flydata/plugin_support/source_position_file'
+require 'flydata-core/postgresql/source_pos'
+
+module Flydata
+module SourcePostgresql
+
+module PluginSupport
+  class SourcePositionFile < Flydata::PluginSupport::SourcePositionFile
+    SOURCE_POS_CLASS = FlydataCore::Postgresql::SourcePos
+  end
+end
+
+end
+end

data/lib/flydata/source_postgresql/query_based_sync/client.rb

@@ -0,0 +1,16 @@
+require 'flydata/query_based_sync/client.rb'
+require 'flydata/source_postgresql/query_based_sync/resource_requester.rb'
+require 'flydata/source_postgresql/query_based_sync/response_handler.rb'
+
+module Flydata
+module SourcePostgresql
+module QueryBasedSync
+
+class Client < Flydata::QueryBasedSync::Client
+  RESOURCE_REQUESTER_CLASS = Flydata::SourcePostgresql::QueryBasedSync::ResourceRequester
+  RESPONSE_HANDLER_CLASS = Flydata::SourcePostgresql::QueryBasedSync::ResponseHandler
+end
+
+end
+end
+end

data/lib/flydata/source_postgresql/query_based_sync/diff_query_generator.rb

@@ -0,0 +1,135 @@
+require 'flydata-core/postgresql/pg_client'
+
+module Flydata
+module SourcePostgresql
+module QueryBasedSync
+
+class DiffQueryGenerator
+
+  DEFAULT_LIMIT = 10000
+
+  FETCH_RECORDS_SQL = <<EOS
+SELECT %{columns} FROM %{table_name} WHERE %{where_clause} ORDER BY %{order_by_clause} LIMIT %{max_rows};
+EOS
+
+  TO_SID_WHERE_CLAUSE = "txid_visible_in_snapshot(xmin::TEXT::BIGINT, '%{to_sid}')"
+  FROM_SID_WHERE_CLAUSE = "AND NOT txid_visible_in_snapshot(xmin::TEXT::BIGINT, '%{from_sid}')"
+
+  def initialize(table, schema, query_cond = {})
+    @table = table
+    @schema = schema
+
+    @columns = query_cond[:columns] or raise ArgumentError, "columns is required"
+    @from_sid = query_cond[:from_sid]
+    @to_sid = query_cond[:to_sid] or raise ArgumentError, "to_sid is required"
+    @pk_columns = query_cond[:pk_columns] or raise ArgumentError, "pk_columns is required"
+    @last_pks = query_cond[:last_pks]
+    @limit = query_cond[:limit] || DEFAULT_LIMIT
+  end
+
+  def build_query(binding_param_enabled = true)
+    from_sid_where_clause = if @from_sid
+                              FROM_SID_WHERE_CLAUSE % {from_sid: @from_sid}
+                            else
+                              nil
+                            end
+
+    column_names, types = @columns.inject([[],[]]) do |arys, h|
+      arys[0] << h[:column]; arys[1] << h[:type].gsub(/\(.+?\)/,''); arys
+    end
+    columns = column_list(column_names, types)
+
+    query = FETCH_RECORDS_SQL % {
+      columns: columns,
+      table_name: fq_table_name(@table, @schema),
+      where_clause: build_where_clause(@from_sid, @to_sid, @pk_columns, @last_pks, binding_param_enabled),
+      order_by_clause: build_order_by_clause(@pk_columns),
+      max_rows: @limit,
+    }
+
+    opts = {
+      value_overriders: build_value_overriders(column_names, types)
+    }
+
+    if binding_param_enabled
+      opts[:binding_params] = @last_pks
+    end
+
+    FlydataCore::Postgresql::PGQuery.new(query, opts)
+  end
+
+  private
+
+  def build_where_clause(from_sid, to_sid, pk_columns, last_pks, binding_param_enabled)
+    clauses = []
+    clauses << TO_SID_WHERE_CLAUSE % { to_sid: to_sid }
+    clauses << FROM_SID_WHERE_CLAUSE % { from_sid: from_sid } if from_sid
+    if last_pks && !last_pks.empty?
+      clauses << pk_conditions(pk_columns, last_pks, binding_param_enabled)
+    end
+    clauses.join(" ")
+  end
+
+  def build_order_by_clause(pk_columns)
+    "#{pk_columns.collect{|col| %Q|"#{col}"| }.join(",")}"
+  end
+
+  def pk_conditions(pk_columns, last_pks, binding_param_enabled)
+    i = pk_columns.size - 1
+    str = nil
+    while i >= 0
+      colname = pk_columns[i]
+      val = value_in_where(i, last_pks, binding_param_enabled)
+      if str
+        str = %Q|"#{colname}" > #{val} OR ("#{colname}" = #{val} AND (#{str}))|
+      else
+        str = %Q|"#{colname}" > #{val}|
+      end
+      i -= 1
+    end
+    " AND (#{str})"
+  end
+
+  def value_in_where(index, last_pks, binding_param_enabled)
+    if binding_param_enabled
+      "$#{index + 1}"
+    else
+      %Q|'#{last_pks[index]}'|
+    end
+  end
+
+  def fq_table_name(table, schema)
+    !schema.to_s.empty? ? %Q|"#{schema}"."#{table}"| : %Q|"#{table}"|
+  end
+
+  # return a list of columns with necessary conversion as a string
+  QUERY_VAL_CONV_RULES = Hash.new do |h, k|
+    h[k] =
+      case k
+        when "bytea"; %Q['0x' || encode("%<column>s", 'hex') AS "%<column>s"]
+        when "money"; %Q["%<column>s"::numeric]
+        else %Q["%<column>s"]
+      end
+  end
+
+  VALUE_OVERRIDERS = Hash.new do |h, k|
+    h[k] =
+      case k
+      when "bit", "varbit"; -> (v) { v ? '%d' % v.to_i(2) : nil }
+      else
+        nil
+      end
+  end
+
+  def column_list(columns, types)
+    columns.each_with_index.collect{|c, i| QUERY_VAL_CONV_RULES[types[i]] % {column:c}}.join(", ")
+  end
+
+  def build_value_overriders(columns, types)
+    columns.each_with_index.inject({}) {|h, (c, i)| overrider = VALUE_OVERRIDERS[types[i]]; h[c] = overrider if overrider; h}
+  end
+end
+
+end
+end
+end