flydata 0.6.11 → 0.6.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa75cb40b7d49f2aeb912086e3e2e35e837cb2e0
-  data.tar.gz: 5234a32b9c3f7e2a40702d4cbeb305e3dbed548b
+  metadata.gz: 181cbcfe32036e4e5e580f5208c183c2aeede5d5
+  data.tar.gz: d74e286636bafd5a3ade8482cd808c69597ff723
 SHA512:
-  metadata.gz: 5482b2ef22c8b0b0e0c49841190fd02829a74811adb8a95f487e2dc9c2f6447bbb2a5e3d6a108a81c035c8886cc11132903cd57f90ccf4b9c076dba4312e7ea2
-  data.tar.gz: 01ccf0a83e53aac78ca543e2b422c0163d9d5775250b7e61317f24d8fd915f1bd8e8cd4d0dcf2bcad7512b89bb7e5c07a51f66392e41898ecf95ba2b2f72e96e
+  metadata.gz: d01fd1622e3573ed4b2989af7ad77a1fd9f8c945bc6d8f959e65818a4733c161e32793c908c51ce71974066bc86db80d69aabff904d18a4336ddc60f95bd40bf
+  data.tar.gz: f829d5633875735e6fef04eec4fadd7778b9e35eedbc7decbcbea150ebb4a2bf64d91bfcc50550f14f4a59ff8aecc2599cae93bb249e8ec775d2b58a71927f43

data/Gemfile

@@ -14,7 +14,7 @@ gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
 gem "io-console", '~> 0.4.2', '>= 0.4.2'
-gem "kodama", '~> 0.1', '>= 0.1.10'
+gem "kodama", '~> 0.1', '>= 0.1.11'
 gem "serverengine", '~> 1.5'
 
 group :development do

data/Gemfile.lock

@@ -61,8 +61,8 @@ GEM
       rdoc
     json (1.8.1)
     jwt (1.0.0)
-    kodama (0.1.10)
-      ruby-binlog (~> 1.0, >= 1.0.8)
+    kodama (0.1.11)
+      ruby-binlog (~> 1.0, >= 1.0.9)
     method_source (0.8.2)
     mime-types (2.3)
     minitest (4.7.5)
@@ -108,7 +108,7 @@ GEM
     rspec-mocks (3.0.3)
       rspec-support (~> 3.0.0)
     rspec-support (3.0.3)
-    ruby-binlog (1.0.8)
+    ruby-binlog (1.0.9)
     ruby-prof (0.15.1)
     serverengine (1.5.10)
       sigdump (~> 0.2.2)
@@ -138,7 +138,7 @@ DEPENDENCIES
   io-console (~> 0.4.2, >= 0.4.2)
   jeweler (~> 1.8, >= 1.8.8)
   json (~> 1.8, >= 1.8.0)
-  kodama (~> 0.1, >= 0.1.10)
+  kodama (~> 0.1, >= 0.1.11)
   mysql2 (~> 0.3, >= 0.3.17)
   pg (~> 0.18.4)
   protected_attributes (~> 1.0, >= 1.0.8)

data/VERSION

@@ -1 +1 @@
-0.6.11
+0.6.12

data/flydata-core/lib/flydata-core/postgresql/source_pos.rb

@@ -0,0 +1,34 @@
+require 'json'
+
+module FlydataCore
+module Postgresql
+
+class SourcePos
+  include Comparable
+
+  def initialize(snapshot_id, pk_values = nil)
+    @snapshot_id = snapshot_id
+    @pk_values = pk_values
+  end
+
+  attr_reader :snapshot_id
+
+  def to_s
+    pk_values = @pk_values ? @pk_values.to_json : ''
+    "#{@snapshot_id}\t#{pk_values}"
+  end
+
+  def <=>(other)
+    # TODO take pk_values into account
+    @snapshot_id <=> other.snapshot_id
+  end
+
+  def self.load(str)
+    snapshot_id, pk_valules = str.split("\t")
+    pk_values = JSON.parse(pk_values) if pk_values
+    self.new(snapshot_id, pk_values)
+  end
+end
+
+end
+end

data/flydata-core/lib/flydata-core/table_def/base.rb

@@ -25,6 +25,16 @@ class Base
 
     tabledef
   end
+
+  def column_names
+    @columns.collect{|col| col[:column]}
+  end
+
+  def pk_columns
+    @columns.select{|col| col.has_key?(:primary_key) }.
+             sort{|a,b| a[:primary_key] <=> b[:primary_key]}.
+             collect{|col| col[:column]}
+  end
 end
 
 end

data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb

@@ -9,16 +9,32 @@ class PostgresqlTableDef < Base
 
   TYPE_MAP_P2F = {
     'bigint' => {type: 'int8'},
+    'bigserial' => {type: 'int8'},
+    'boolean' => {type: 'int1', def_width: '1'}, # TODO introduce bool type
+    'bytea' => {type: 'varbinary'}, # TODO need value conversion
+    'character' => {type: 'varchar', def_width: '1'},
     'character varying' => {type: 'varchar',
-                            width_attrs:["character_octet_length"]}, # TODO check def_width
+                            width_attrs:["character_octet_length"],
+                            def_width:[1]},
+    'date' => {type: 'date'},
+    'decimal' => {type: 'numeric',
+                  width_attrs:["numeric_precision", "numeric_scale"],
+                  def_width:[nil, 0] },
+    'double precision' => {type: 'float8'},
     'integer' => {type: 'int4'},
-    'timestamp with time zone' => {type: 'datetime'}, # TODO may need override?
-    'timestamp without time zone' => {type: 'datetime'}, # TODO may need override?
-    'bytea' => {type: 'varbinary'}, # TODO need value conversion
+    'money' => {type: 'money'},
     'numeric' => {type: 'numeric',
                   width_attrs:["numeric_precision", "numeric_scale"],
                   def_width:[nil, 0],
                  },
+    'real' => {type: 'float4'},
+    'serial' => {type: 'int4'},
+    'smallint' => {type: 'int2'},
+    'text' => {type: 'text'},
+    'time with time zone' => {type: 'time'},
+    'time without time zone' => {type: 'time'},
+    'timestamp with time zone' => {type: 'datetime'},
+    'timestamp without time zone' => {type: 'datetime'},
     # TODO add more types
   }
 

data/flydata-core/spec/postgresql/source_pos_spec.rb

@@ -0,0 +1,43 @@
+require 'flydata-core/postgresql/source_pos'
+
+module FlydataCore
+module Postgresql
+
+describe SourcePos do
+  let(:subject_object) { described_class.new(snapshot_id, pk_values) }
+
+  let(:snapshot_id) { "1234:1234:" }
+  let(:pk_values) { nil }
+
+  describe '#to_s' do
+    subject { subject_object.to_s }
+
+    context 'when pk_values is nil' do
+      let(:pk_values) { nil }
+      it { is_expected.to eq %Q|#{snapshot_id}\t| }
+    end
+    context 'when pk_values is not nil' do
+      let(:pk_values) { { "user_id" => 1, "address_id" => 3 } }
+      it { is_expected.to eq %Q|#{snapshot_id}\t{"user_id":1,"address_id":3}| }
+    end
+  end
+
+  describe '.load' do
+    subject { described_class.load(str) }
+
+    context 'without pk_values' do
+      let(:str) { %Q|#{snapshot_id}\t| }
+      it { is_expected.to eq described_class.new(snapshot_id) }
+    end
+    context 'with pk_values' do
+      let(:str) { %Q|#{snapshot_id}\t{"user_id":1,"address_id":3}| }
+      it do
+        is_expected.to eq described_class.new(snapshot_id,
+                                           "user_id" => 1, "address_id" => 3)
+      end
+    end
+  end
+end
+
+end
+end

data/flydata-core/spec/table_def/base_spec.rb

@@ -0,0 +1,51 @@
+require 'flydata-core/table_def/base'
+
+module FlydataCore
+module TableDef
+
+describe Base do
+  let(:subject_object) { described_class.new(table_def, table_name, columns,
+                           column_def, default_charset,
+                           default_source_charset, comment) }
+
+  let(:table_def) { double('table_def') }
+  let(:table_name) { double('table_name') }
+  let(:columns) { double('columns') }
+  let(:column_def) { double('column_def') }
+  let(:default_charset) { double('default_charset') }
+  let(:default_source_charset) { double('default_source_charset') }
+  let(:comment) { double('comment') }
+
+  describe '#pk_columns' do
+    subject { subject_object.pk_columns }
+
+    let(:columns) do
+      [ {column: "name"},
+        {column: "address_id", primary_key: pk1_value},
+        {column: "zip"},
+        {column: "user_id", primary_key: pk2_value},
+      ]
+    end
+    
+
+    context 'with unordered primary keys' do
+      let(:pk1_value) { 't' }
+      let(:pk2_value) { 't' }
+
+      it "returns primary keys in the column order" do
+        is_expected.to eq %w(address_id user_id)
+      end
+    end
+    context 'with ordered primary keys' do
+      let(:pk1_value) { 2 }
+      let(:pk2_value) { 1 }
+
+      it "returns primary keys in the pk order" do
+        is_expected.to eq %w(user_id address_id)
+      end
+    end
+  end
+end
+
+end
+end

data/lib/flydata/command/sender.rb

@@ -110,8 +110,8 @@ EOF
           log_info_stdout('Restarting sender process.') unless options[:quiet]
           begin
             stop(options)
-          rescue
-            log_warn_stderr 'Something has gone wrong... force stopping the FlyData process...'
+          rescue => e
+            log_warn_stderr "Something has gone wrong. Force stopping the FlyData process. error:#{e.to_s}"
             kill_all(options)
           end
         else
@@ -132,7 +132,7 @@ EOF
         `[ -f #{pid_file} ] && pgrep -P \`cat #{pid_file}\``.to_i > 0
       end
       def kill_all(options = {})
-        if Kernel.system("ps ax | grep 'flydata' | grep -v grep | grep fluentd | awk '{print \"kill  \" $1}' | sh")
+        if Kernel.system("ps x | grep 'flydata' | grep -v grep | grep fluentd | awk '{print \"kill -9 \" $1}' | sh")
           log_info_stdout("Done.") unless options[:quiet]
           return true
         else
@@ -192,11 +192,14 @@ EOF
         raise "Somthing has gone wrong... Please try setup command again."
       end
       def wait_until_client_stop(options = {})
-        retry_count = 10
+        retry_count = 120
+        show_message_at_retry = 4
         1.upto(retry_count) do |i|
           return true unless process_exist?
-          log_info_stdout("Waiting for the client to stop... (#{i}/#{retry_count})") unless options[:quiet]
-          Kernel.sleep 3
+          Kernel.sleep 5
+          if i % show_message_at_retry == 0
+            log_info_stdout("Waiting for the client to stop...") unless options[:quiet]
+          end
         end
         false
       end

data/lib/flydata/command/setup.rb

@@ -76,18 +76,12 @@ What's next?
         Flydata::Command::Conf.new.copy_templates
         puts
         de_exists = !!retrieve_data_entries.first
-        ret = if de_exists && source.setup.initial_run_need_restart?
-                sender = Flydata::Command::Sender.new
-                if sender.process_exist?
-                  sender.stop(quiet: true)
-                  true
-                else
-                  false
-                end
-              else
-                false
-              end
-        Flydata::Command::Sender.new.restart(quiet: true) if ret
+        if de_exists && source.setup.initial_run_need_restart?
+          sender = Flydata::Command::Sender.new
+          if sender.process_exist?
+            Flydata::Command::Sender.new.restart(quiet: true)
+          end
+        end
         last_message = de_exists ? source.setup.initial_run_complete_message
                                   : :no_de_cancel
         msgs = standard_messages

data/lib/flydata/command/sync.rb

@@ -34,6 +34,7 @@ module Flydata
       SERVER_DATA_PROCESSING_TIMEOUT = 3600 # seconds
 
       # for dump.pos file
+      STATUS_START = 'START'       # only :source_pos is available at the begining of parse
       STATUS_PARSING = 'PARSING'
       STATUS_PARSED = 'WAITING'    # the value is different from the constant name on purpose for backward compatibility.
       STATUS_COMPLETE = 'COMPLETE'
@@ -783,24 +784,27 @@ EOM
       def parse_dump_and_send(dump_io, dp, de, sync_fm, source_pos)
 
         context = source.parse_dump_and_send
-        # Prepare forwarder
-        de_tag_name = de["tag_name#{env_suffix}"]
-        server_port = dp['server_port']
-        data_servers = source.sync.data_servers
-        servers = data_servers ?  data_servers.split(',') :
-                   dp["servers#{env_suffix}"].collect{|s| "#{s}:#{server_port}"}
-        forwarder_type = source.sync.forwarder ||
-          (dp['ssl_enabled'] ? 'sslforwarder' : 'tcpforwarder')
-        forwarder = Flydata::Output::ForwarderFactory.create(forwarder_type, de_tag_name, servers)
 
         # Load dump.pos file for resume
         dump_pos_info = sync_fm.load_dump_pos
-        dump_pos_info ||= {}
+        if dump_pos_info.empty?
+          # First time to parse. No dump pos file available.
+          # Re-initialize dump_pos_info hash with new dump pos file.
+          if source_pos.nil?
+            raise "Cannot parse dump because source position is not available. Reset unfinished initial sync by command: flydata sync:reset --init"
+          end
+          sync_fm.save_dump_pos(STATUS_START, nil, nil, source_pos)
+          dump_pos_info = sync_fm.load_dump_pos
+        end
+
+        source_pos = dump_pos_info[:source_pos]
+        if source_pos.nil?
+          # this should never happen but just in case...
+          raise "Source position is not available. Parse dump failed. Reset unfinished initial sync by command: flydata sync:reset --init"
+        end
+
         if dump_pos_info[:table_name] && dump_pos_info[:last_pos].to_i != -1
-          source_pos = dump_pos_info[:source_pos]
           log_info_stdout("Resuming... Last processed table: #{dump_pos_info[:table_name]}")
-        else
-          dump_pos_info[:source_pos] = source_pos
         end
         if dump_pos_info[:source_table] && dump_pos_info[:source_table].value_converters.nil?
           # Old marshal dumped source_table object may not have value_converters
@@ -810,6 +814,17 @@ EOM
           # Old marshal dumped source_table object may not have column_names
           dump_pos_info[:source_table].set_column_names
         end
+
+        # Prepare forwarder
+        de_tag_name = de["tag_name#{env_suffix}"]
+        server_port = dp['server_port']
+        data_servers = source.sync.data_servers
+        servers = data_servers ?  data_servers.split(',') :
+                   dp["servers#{env_suffix}"].collect{|s| "#{s}:#{server_port}"}
+        forwarder_type = source.sync.forwarder ||
+          (dp['ssl_enabled'] ? 'sslforwarder' : 'tcpforwarder')
+        forwarder = Flydata::Output::ForwarderFactory.create(forwarder_type, de_tag_name, servers)
+
         log_info_stdout("Sending data to FlyData Server...")
 
         # initialize stats
@@ -884,6 +899,8 @@ EOM
                   if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
                     ret = forwarder.flush # send buffer data to the server before checkpoint
                     sync_fm.save_record_count_stat(table_name, ret[:record_count]) if ret
+                    # save check point
+                    sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, source_pos, state, substate)
                   end
 
                   # show the current progress
@@ -893,9 +910,6 @@ EOM
                     log_info_stdout("  -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
                   end
 
-                  # save check point
-                  sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, source_pos, state, substate)
-
                   # send record count for the table
                   if table_name.to_s != '' &&
                      state == Flydata::Parser::State::CREATE_TABLE
@@ -925,10 +939,10 @@ EOM
         send_record_counts_threads.each{|t| t.join}
         output_thread.join
         forwarder.close
+        sync_fm.save_dump_pos(STATUS_PARSED, '', dump_file_size, source_pos)
         log_info_stdout("  -> Done")
         #log_info_stdout("  -> Records sent to the server")
         #log_info_stdout("  -> #{sync_fm.load_stats}")
-        sync_fm.save_dump_pos(STATUS_PARSED, '', dump_file_size, source_pos)
 
         if ENV['FLYDATA_BENCHMARK']
           bench_end_time = Time.now

data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb

@@ -15,9 +15,8 @@ module Fluent
         config_param :tables_append_only, :string
 
         # binlog plugin
-        # TODO: Need to check if not configured or not
-        #config_param :tag, :string
-        #config_param :position_file, :string, :default => 'position.log'
+        config_param :tag, :string
+        config_param :position_file, :string, default: 'position.log'
       end
     end
 

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -9,15 +9,15 @@ require 'kodama'
 require 'flydata/sync_file_manager'
 require 'flydata-core/mysql/command_generator'
 require 'flydata/fluent-plugins/flydata_plugin_ext/preference'
-require 'flydata/fluent-plugins/mysql/binlog_position_file'
-require 'flydata/fluent-plugins/mysql/binlog_record_dispatcher'
-require 'flydata/fluent-plugins/mysql/context'
 require 'flydata/fluent-plugins/flydata_plugin_ext/idle_event_detector'
-require 'flydata/fluent-plugins/mysql/table_meta'
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync'
+require 'flydata/source_mysql/plugin_support/binlog_position_file'
+require 'flydata/source_mysql/plugin_support/binlog_record_dispatcher'
+require 'flydata/source_mysql/plugin_support/context'
+require 'flydata/source_mysql/plugin_support/table_meta'
 require 'flydata/source_mysql/table_ddl'
 require 'flydata-core/fluent/config_helper'
 require 'flydata-core/mysql/ssl'
-require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync'
 
 
 class MysqlBinlogFlydataInput < MysqlBinlogInput
@@ -26,11 +26,9 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
   Plugin.register_input('mysql_binlog_flydata', self)
 
-  BINLOG_POSITION_FILE_CLASS = Mysql::BinLogPositionFile
+  BINLOG_POSITION_FILE_CLASS = Flydata::SourceMysql::PluginSupport::BinLogPositionFile
 
   config_param :database, :string
-  config_param :tables, :string
-  config_param :tables_append_only, :string
   config_param :initial_idle_interval, :integer, :default => 30     # 30 sec
   config_param :continuous_idle_interval, :integer, :default => 600 # 10 min
   config_param :idle_timeout, :time, :default => 1800               # 30 min
@@ -51,7 +49,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
   # Supported ssl cipher list for mysql(openssl)
   # https://dev.mysql.com/doc/refman/5.6/en/ssl-options.html
 
-  config_param :secondary_ssl_cipher, :string, :default => FlydataCore::Mysql::Ssl::NON_DH_SSL_CIPHER
+  config_param :secondary_ssl_cipher, :string, default: FlydataCore::Mysql::Ssl::NON_DH_SSL_CIPHER
 
   def configure(conf)
     super
@@ -77,7 +75,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       #exit 1   # causes retry loop
     end
 
-    table_meta = Flydata::Mysql::TableMeta.new(@db_opts.merge(tables: @tables))
+    table_meta = Flydata::SourceMysql::PluginSupport::TableMeta.new(@db_opts.merge(tables: @tables))
 
     table_revs = tables.inject({}) do |h, table_name|
       h[table_name] = @sync_fm.table_rev(table_name)
@@ -85,12 +83,12 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     end
 
     # Set context
-    @context = Mysql::Context.new(
+    @context = Flydata::SourceMysql::PluginSupport::Context.new(
       database: @database, tables: @tables,
       tag: @tag, sync_fm: @sync_fm, omit_events: @omit_events,
       table_meta: table_meta, table_revs: table_revs,
     )
-    @record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
+    @record_dispatcher = Flydata::SourceMysql::PluginSupport::FlydataBinlogRecordDispatcher.new(@context)
     @idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval, @idle_timeout)
   end
 
@@ -184,6 +182,9 @@ EOS
         elsif /binlog file.*does not exist/.match(e.to_s)
           $log.error("#{e.to_s}.  Sync must be reset.  Terminating the agent.")
           Process.kill(:TERM, Process.ppid)
+        elsif e.kind_of?(Kodama::ConnectionEstablishError)
+          $log.error("#{e.to_s}.  Connection to MySQL server could not be established.  Terminating the agent.")
+          Process.kill(:TERM, Process.ppid)
         else
           raise e
         end
@@ -228,8 +229,8 @@ EOS
       begin
         @kodama_client.stop_request
         if wait_till_safe_to_stop
-          $log.info "Killing Kodama client"
-          Thread.kill(@thread)
+          @thread.join
+          $log.info "Kodama has stopped successfully"
         else
           $log.error "Unable to stop Kodama"
         end