flydata 0.6.11 → 0.6.12

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/dml_record_handler.rb

@@ -1,6 +1,9 @@
-require 'flydata/fluent-plugins/mysql/binlog_record_handler'
+require 'flydata/source_mysql/plugin_support/binlog_record_handler'
 
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
   class DmlRecordHandler < BinlogRecordHandler
     ROW = :row
     OLD = :old
@@ -106,3 +109,6 @@ module Mysql
     end
   end
 end
+
+end
+end

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/drop_database_query_handler.rb

@@ -1,6 +1,9 @@
-require 'flydata/fluent-plugins/mysql/ddl_query_handler'
+require 'flydata/source_mysql/plugin_support/ddl_query_handler'
 
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
   class DropDatabaseQueryHandler < DatabaseDdlQueryHandler
     # For MySQL, database and schema are exchangable
     PATTERN = /^DROP (DATABASE|SCHEMA)/i
@@ -22,3 +25,6 @@ module Mysql
     end
   end
 end
+
+end
+end

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/table_meta.rb

@@ -3,7 +3,9 @@ require 'flydata-core/mysql/config'
 require 'flydata-core/table_def/mysql_table_def'
 
 module Flydata
-module Mysql
+module SourceMysql
+
+module PluginSupport
   class TableMeta
     MANDATORY_OPTS = [
       :host, :port, :username, :password,
@@ -57,4 +59,6 @@ EOT
     end
   end
 end
+
+end
 end

data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/truncate_table_query_handler.rb

@@ -1,6 +1,9 @@
-require 'flydata/fluent-plugins/mysql/ddl_query_handler'
+require 'flydata/source_mysql/plugin_support/ddl_query_handler'
 
-module Mysql
+module Flydata
+module SourceMysql
+
+module PluginSupport
   class TruncateTableQueryHandler < TableDdlQueryHandler
     PATTERN = /^TRUNCATE/i
 
@@ -22,3 +25,6 @@ module Mysql
     end
   end
 end
+
+end
+end

data/lib/flydata/source_postgresql/generate_source_dump.rb

@@ -0,0 +1,175 @@
+require 'flydata/source/generate_source_dump'
+require 'flydata/preference/data_entry_preference'
+require 'flydata/source_postgresql/postgresql_component'
+require 'flydata/source_postgresql/pg_client'
+require 'flydata-core/postgresql/source_pos'
+require 'msgpack'
+
+module Flydata
+module SourcePostgresql
+
+class GenerateSourceDump < Source::GenerateSourceDump
+  include PostgresqlComponent
+
+  def run_compatibility_check(dump_dir, backup_dir)
+    %w(host username database schema).each do |k|
+      if de_prefs[k].to_s.empty?
+        raise "'#{k}' is required. Set the value in the conf file " +
+              "-> #{Flydata::Preference::DataEntryPreference.conf_path(de)}"
+      end
+    end
+  end
+
+  def confirmation_items
+    items = {
+      "host"     => de_prefs['host'],
+      "port"     => de_prefs['port'],
+      "username" => de_prefs['username'],
+      "database" => de_prefs['database'],
+      "schema"   => de_prefs['schema'],
+    }
+
+    items
+  end
+
+DUMP_SIZE_QUERY = <<EOS
+SELECT sum(pg_total_relation_size(c.oid)) AS "total_size"
+  FROM pg_class c
+  LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
+  WHERE nspname = $1 AND relname in (%s)
+EOS
+  TABLE_PLACEHOLDER_START_NUM = 2 # because $1 is used by table_schema
+
+  def dump_size(tables)
+    cli = PGClient.new(de_prefs)
+
+    res = cli.query(DUMP_SIZE_QUERY, [de_prefs['schema']] + tables, placeholder_size: tables.size, placeholder_start_num: TABLE_PLACEHOLDER_START_NUM)
+
+    res.first['total_size'].to_i
+  end
+
+  def dump(tables, file_path = nil, &src_pos_callback)
+    if file_path
+      io = File.open(file_path, "w")
+    else
+      raise "dump via pipe has not been implemented yet"
+    end
+
+    cli = PGClient.new(de_prefs)
+
+    source_pos = get_source_pos(cli, &src_pos_callback)
+
+    context = source.sync_generate_table_ddl(dp, nil)
+    missing_tables = context.each_source_tabledef(tables, de_prefs) do |tabledef, error|
+      dump_table(tabledef, source_pos, io, cli) if tabledef
+    end
+    nil
+  end
+
+  private
+
+  CURRENT_SNAPSHOT_QUERY = <<EOS
+SELECT txid_current_snapshot() AS current_snapshot;
+EOS
+
+  def get_source_pos(cli, &src_pos_callback)
+    res = cli.query(CURRENT_SNAPSHOT_QUERY)
+
+    current_snapshot = res.first['current_snapshot']
+    src_pos = FlydataCore::Postgresql::SourcePos.new(current_snapshot)
+    src_pos_callback.call(nil, src_pos)
+
+    src_pos
+  end
+
+  NUM_ROWS = 500000
+
+  def dump_table(tabledef, source_pos, io, cli)
+    fqtn = fq_table_name(de_prefs['schema'], tabledef.table_name)
+    pk_columns = tabledef.pk_columns
+    columns = tabledef.column_names
+
+    dump_source_table(tabledef, io)
+
+    last_pks = nil
+    loop do
+      num_rows, last_pks = dump_table_chunk(fqtn, source_pos, NUM_ROWS,
+                                            columns, pk_columns, last_pks, io, cli)
+      break if num_rows < NUM_ROWS
+    end
+  end
+
+  def dump_source_table(tabledef, io)
+    columns = {}
+    tabledef.column_def.each do |column_name, columndef_str|
+      columndef_hash = eval columndef_str
+      columns[column_name] = {"column_name" => column_name,
+                              "format_type" => columndef_hash["data_type"]}
+    end
+    source_table_hash = { "table_name" => tabledef.table_name,
+                          "columns" => columns }
+
+    io.write(source_table_hash.to_msgpack)
+  end
+
+  SELECT_QUERY = <<EOS
+SELECT * FROM %s%s%s LIMIT %s;
+EOS
+
+  def dump_table_chunk(fqtn, source_pos, num_rows, columns, pk_columns, last_pks, io, cli)
+    where_clause = build_where_clause(source_pos, pk_columns, last_pks)
+    params = last_pks || []
+    order_by_clause = build_order_by_clause(pk_columns)
+    query = SELECT_QUERY % [fqtn, where_clause, order_by_clause, num_rows]
+
+    res = cli.query(query, params)
+    count = 0
+    last_row = nil
+    res.each do |row|
+      count += 1
+      last_row = row
+      dump_row(row, columns, io)
+    end
+    last_pks = last_row ? pk_columns.collect{|col| last_row[col]} : nil
+
+    [count, last_pks]
+  end
+
+  def dump_row(row, columns, io)
+    data = columns.collect{|col| row[col] }.to_msgpack
+    io.write(data)
+  end
+
+  def build_where_clause(source_pos, pk_columns, last_pks)
+    clause = " WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '#{source_pos.snapshot_id}'::TXID_SNAPSHOT)"
+    if last_pks
+      clause += pk_conditions(pk_columns)
+    end
+  end
+
+  def build_order_by_clause(pk_columns)
+    " ORDER BY #{pk_columns.collect{|col| %Q|"#{col}"| }.join(",")}"
+  end
+
+  def pk_conditions(pk_columns)
+    i = pk_columns.size - 1
+    str = nil
+    while i >= 0
+      colname = pk_columns[i]
+      if str
+        str = %Q|"#{colname}" > $#{i + 1} OR ("#{colname}" = $#{i + 1} AND (#{str}))|
+      else
+        str = %Q|"#{colname}" > $#{i + 1}|
+      end
+      i -= 1
+    end
+    " AND (#{str})"
+  end
+
+  def fq_table_name(schema, table)
+    schema ? %Q|"#{schema}"."#{table}"| : %Q|"#{table}"|
+  end
+end
+
+end
+end

data/lib/flydata/source_postgresql/parse_dump_and_send.rb

@@ -0,0 +1,126 @@
+require 'flydata/source/parse_dump_and_send'
+require 'flydata/source_postgresql/postgresql_component'
+require 'flydata-core/table_def/postgresql_table_def'
+
+module Flydata
+module SourcePostgresql
+
+class ParseDumpAndSend < Source::ParseDumpAndSend
+  include PostgresqlComponent
+
+  def value_converters
+    FlydataCore::TableDef::PostgresqlTableDef::VALUE_CONVERTERS
+  end
+
+  # dump format
+  # dump file is in msgpack.  Each table data starts with a source table hash
+  # followed by row arrays.
+  #
+  # {"table_name"=>"users", "columns"=>{"id"=>{"column_name"=>"id", "format_type"=>"bigint"}, "name"=>{"column_name"=>"name", "format_type"=>"character varying"}, "another_id"=>{"column_name"=>"another_id", "format_type"=>"integer"}}}
+  # ["2", "hay", "1"]
+  # ["3", "hoe", "2"]
+  def parse_dump(dump_pos_info, dmpio, create_table_block, insert_record_block,
+                 check_point_block)
+    parser = DumpParser.new(dump_pos_info, dmpio, create_table_block,
+                            insert_record_block, check_point_block)
+    parser.parse_all
+  end
+end
+
+class DumpParser
+  MAX_ROW_BYTES = 1 * 1024 * 1024 # Parser holds rows until the total byte size
+                                  # reaches this number
+
+  def initialize(dump_pos_info, dmpio, create_table_block, insert_record_block,
+                 check_point_block)
+    @source_pos = dump_pos_info[:source_pos]
+    raise ArgumentError.new("source position is required") unless @source_pos
+    @current_table = nil
+    @last_pos = 0
+    @row_head_pos = nil
+    @rows = []
+    @dmpio = dmpio
+    @create_table_block = create_table_block
+    @insert_record_block = insert_record_block
+    @check_point_block = check_point_block
+
+    resume(dump_pos_info)
+  end
+
+  def parse_all
+    u = MessagePack::Unpacker.new(@dmpio)
+    u.each do |obj|
+      @last_pos = @dmpio.pos - u.buffer.size
+      parse(obj)
+    end
+    close
+  end
+
+  def parse(obj)
+    obj.kind_of?(Hash) ? handle_table_info(obj) : handle_data_row(obj)
+  end
+
+  def close
+    unless @rows.empty?
+      call_insert_record_block
+      # Core logic expects a check point callback with CREATE_TABLE at the
+      # end of table data insertion of each table.  #handle_table_info takes
+      # care of all tables but the last table.  This is for the last table.
+      call_check_point_block(Parser::State::CREATE_TABLE)
+    end
+  end
+
+
+  private
+
+  def resume(dump_pos_info)
+    last_pos = dump_pos_info[:last_pos] ? dump_pos_info[:last_pos].to_i : -1
+    if last_pos == -1
+      # no resume point
+      return
+    end
+
+    @last_pos = last_pos
+    @dmpio.pos = last_pos
+    @current_table = dump_pos_info[:source_table]
+  end
+
+  # {"table_name"=>"users", "columns"=>{"id"=>{"column_name"=>"id", "format_type"=>"bigint"}, "name"=>{"column_name"=>"name", "format_type"=>"character varying"}, "another_id"=>{"column_name"=>"another_id", "format_type"=>"integer"}}}
+  def handle_table_info(table_info)
+    call_check_point_block(Parser::State::CREATE_TABLE)
+    columns = table_info["columns"].inject({}) do |h, (k, v)|
+                h[k] = %w(column_name format_type).inject({}) do |hh, kk|
+                         hh[kk.to_sym] = v[kk]
+                         hh
+                       end
+                h
+              end
+    @current_table = Parser::SourceTable.new(table_info["table_name"], columns)
+    @create_table_block.call(@current_table)
+    call_check_point_block(Parser::State::INSERT_RECORD)
+  end
+
+  def handle_data_row(row)
+    @row_head_pos = @last_pos unless @row_head_pos
+    @rows << row
+    if @last_pos - @row_head_pos > MAX_ROW_BYTES
+      call_insert_record_block
+    end
+  end
+
+  def call_check_point_block(state)
+    @check_point_block.call(@current_table, @last_pos, @last_pos, @source_pos,
+                            state)
+  end
+
+  def call_insert_record_block
+    if @insert_record_block.call(@current_table, @rows)
+      call_check_point_block(Parser::State::INSERT_RECORD)
+    end
+    @rows = []
+    @row_head_pos = nil
+  end
+end
+
+end
+end

data/lib/flydata/source_postgresql/pg_client.rb

@@ -0,0 +1,43 @@
+require 'pg'
+
+module Flydata
+module SourcePostgresql
+
+class PGClient
+  def initialize(de_prefs)
+    @de_prefs = de_prefs
+    @cli = PG::Connection.new(pg_opts_from_de_prefs(de_prefs))
+  end
+
+  def query(query, params = [], opts = {})
+    if opts.has_key?(:placeholder_start_num)
+      placeholders = placeholder_string(opts[:placeholder_size],
+                                        opts[:placeholder_start_num])
+      q = query % [placeholders]
+    else
+      q = query
+    end
+
+    @cli.query(q, params)
+  end
+
+  private
+
+  def pg_opts_from_de_prefs(de_prefs)
+    {
+      host: de_prefs['host'],
+      port: de_prefs['port'],
+      dbname: de_prefs['database'],
+      user: de_prefs['username'],
+      password: de_prefs['password'],
+      sslmode: :prefer,
+    }
+  end
+
+  def placeholder_string(num_items, start_num)
+    num_items.times.collect{|i| "$#{i + start_num}"}.join(",")
+  end
+end
+
+end
+end

data/lib/flydata/source_postgresql/postgresql_component.rb

@@ -0,0 +1,12 @@
+module Flydata
+module SourcePostgresql
+
+# Postgresql specific Component helper methods
+module PostgresqlComponent
+  def de_prefs
+    de['postgresql_data_entry_preference']
+  end
+end
+
+end
+end

data/lib/flydata/source_postgresql/setup.rb

@@ -0,0 +1,24 @@
+require 'flydata/source/setup'
+require 'flydata/sync_file_manager'
+
+module Flydata
+module SourcePostgresql
+
+class Setup < Source::Setup
+  def initial_run_need_restart?
+    overinstall?
+  end
+
+  def initial_run_complete_message
+    overinstall? ? :all_done : :initial_sync
+  end
+
+  private
+
+  def overinstall?
+    File.exists?(Flydata::SyncFileManager.new(de).source_pos_path)
+  end
+end
+
+end
+end