flydata 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6afa928a6d8e0ee05f33e9a14d81be9659164e8f
-  data.tar.gz: d6323ad8aa4f7d04cee5263b92e19b3208c8c651
+  metadata.gz: 4ab029d2a33bb77838fe20cf625de7f473efdb30
+  data.tar.gz: 8313b3c85c3f98201194fd33b68b60cab15c9417
 SHA512:
-  metadata.gz: c273cafd862d6eab962afe5594b7b7b3f72c5536a39b6e6825c98418ee3589e61005a42b4aa77503f3fcbd3d0011ddf550072550f60ba91c6f34c5e9d18bc52d
-  data.tar.gz: 951b50b74c206488fba05241722c09f7bb5f531b8e2097e2cf810bd6cdd863fb1f83471289287f347d1679e4d8ab4f96d27870f535031ea2e4375c3014b7d9ed
+  metadata.gz: 0b2b0a184db03ca9d157ee044b349cdf7ab1c94e5338d536ad51c3be474c8cc6363809646c48406578363901917ac9bb453f0426e461f143b7685d51c95edbde
+  data.tar.gz: b7adf39c00e853819265dfc598b4873aef9e469afadd457dc729fce00f682fe1924321f1bd2a361707db7099a96c040300df9b31d2f4ae2a04840151681ed579

data/VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1

data/ext/flydata/json/json_ext.cpp

@@ -247,23 +247,24 @@ static VALUE rb_json_generate_kv_pairs(VALUE self, VALUE key_ary, VALUE value_ar
     rb_raise(rb_eArgError, "length of arrays must match");
   }
 
-  std::stringstream ss;
+  std::stringstream* ss = new std::stringstream();
 
-  ss << "{";
+  *ss << "{";
 
   for (long i = 0; i < key_len; i++)
   {
-    if (i > 0) { ss << ","; }
+    if (i > 0) { *ss << ","; }
 
     VALUE key = rb_ary_entry(key_ary, i);
     VALUE value = rb_ary_entry(value_ary, i);
 
-    add_kv_pair(key, value, ss);
+    add_kv_pair(key, value, *ss);
   }
-  ss << "}";
+  *ss << "}";
 
-  const std::string& str = ss.str();
+  const std::string& str = ss->str();
   VALUE json = rb_str_new(str.data(), str.size());
+  delete ss; ss = 0;
   int enc = rb_enc_find_index("UTF-8");
   rb_enc_associate_index(json, enc);
 

data/flydata-core/lib/flydata-core/event/event_dictionary.rb

@@ -0,0 +1,10 @@
+module FlydataCore
+  module Event
+    #Nouns
+    INIT_SYNC = "init_sync"
+
+    #Verbs
+    STARTED = "started"
+    FINISHED = "finished"
+  end
+end

data/flydata-core/lib/flydata-core/event/event_handler_base.rb

@@ -11,6 +11,7 @@ module FlydataCore
       end
 
       protected
+
       def self.register_handler(opts)
         @noun = opts[:noun]
         @verb = opts[:verb] || EVENT_VERB_ANY
@@ -21,6 +22,7 @@ module FlydataCore
       end
 
       private
+
       def self.handlers(noun, verb)
         handlers = (@@handlers_hash[key(noun, verb)] || []).clone
 

data/flydata-core/lib/flydata-core/event/flydata_event.rb

@@ -6,23 +6,44 @@ module FlydataCore
     class FlydataEvent
       attr_accessor :noun
       attr_accessor :verb
-      attr_accessor :data
+      attr_accessor :data #recommended that it be a hash
       attr_accessor :id
       attr_accessor :created
       attr_accessor :data_entry_id
       attr_accessor :data_port_id
 
-      def initialize (noun, verb, data=nil, de_id=nil,dp_id=nil, id=nil, timestamp=nil)
+      # params:
+      #   data_port_id: nil,
+      #   data_entry_id: nil,
+      #   id: nil,
+      #   data: nil,
+      #   timestamp: nil
+      def initialize (noun, verb, params = {})
+        params ||= {}
+
         @noun = noun
         @verb = verb
-        @data_entry_id = de_id
-        @data_port_id = dp_id
-        @data = data
-        @id = id.nil? ? SecureRandom.uuid: id
-        @created = timestamp.nil? ? Time.now : timestamp
+        @data_entry_id = params[:data_entry_id] ? params[:data_entry_id].to_i : nil
+        @data_port_id = params[:data_port_id] ? params[:data_port_id].to_i : nil
+        @data = params[:data]
+        @id = params[:id].nil? ? SecureRandom.uuid : params[:id]
+        @created = params[:timestamp].nil? ? Time.now : params[:timestamp]
+      end
+
+      def to_h
+        {
+          noun: @noun,
+          verb: @verb,
+          data_entry_id: @data_entry_id,
+          data_port_id: @data_port_id,
+          id: @id,
+          created: @created,
+          data: @data,
+        }
       end
+
       def to_json
-        {:noun  => @noun, :verb => @verb, :data_entry_id=>@data_entry_id, :data_port_id=> @data_port_id, id => @id, :created => @created, :data => @data}.to_json
+        to_h.to_json
       end
     end
   end

data/flydata-core/lib/flydata-core/event/flydata_event_processor.rb

@@ -3,7 +3,6 @@ require 'flydata-core/event/flydata_event_handler_registry'
 module FlydataCore
   module Event
     class FlydataEventProcessor
-
       def self.process(event)
         handlers = FlydataEventHandlerRegistry.handlers(event.noun, event.verb)
         if handlers.nil? ||  handlers.empty?
@@ -14,6 +13,5 @@ module FlydataCore
         end
       end
     end
-
   end
 end

data/flydata-core/lib/flydata-core/event/flydata_event_sender.rb

@@ -4,20 +4,24 @@ module FlydataCore
   module Event
 
     class FlydataEventSender
-
-      def send_event(noun, verb, data=nil, data_entry_id=nil, data_port_id=nil, routing_key="default")
-        event = FlydataEvent.new(noun, verb, data, data_entry_id, data_port_id)
+      def send_event(noun, verb, params = {})
+        params ||= {}
+        routing_key = params[:routing_key] || 'default'
+        event = FlydataEvent.new(noun, verb, params)
         send(event, routing_key)
       end
 
       protected
+
       def send(event, routing_key)
         raise "send method not implemented"
       end
     end
 
     class SimpleEventSender <FlydataEventSender
+
       protected
+
       def send(event, routing_key)
         FlydataEventProcessor.process(event)
       end

data/flydata-core/lib/flydata-core/logger.rb

@@ -109,19 +109,13 @@ module FlydataCore
       msg = build_log_message(level, message, log_params, options)
       before_logging(level, message, msg, log_params, options)
       logger = get_logger(options)
-      set_mutex(logger)
-      logger.mutex.synchronize do
-        original_depth_offset = nil
-        if logger.instance_variable_defined? :@depth_offset
-          original_depth_offset = logger.instance_variable_get(:@depth_offset)
-          depth_offset = 3  # log_common + synchronize + synchronized block 
-          depth_offset += options[:depth_offset] if options.has_key?(:depth_offset)
-          logger.instance_variable_set(:@depth_offset, original_depth_offset + depth_offset)
-        end
+      # For patched Fluent::Log
+      if logger.respond_to? :flydata_patched
+        depth_offset = 2  # original_depth_offset + log_common
+        depth_offset += options[:depth_offset] if options.has_key?(:depth_offset)
+        logger.send(level, msg, depth_offset: depth_offset)
+      else
         logger.send(level, msg)
-        if original_depth_offset
-          logger.instance_variable_set(:@depth_offset, original_depth_offset)
-        end
       end
     end
 
@@ -154,17 +148,6 @@ module FlydataCore
       options[:logger] || logger || log_context_logger || $log
     end
 
-    # set mutex if logger doesn't have one
-    def set_mutex(logger)
-      return if logger.respond_to?(:mutex)
-
-      mtx = Mutex.new
-      logger.instance_variable_set(:@mutex, mtx)
-      class << logger
-        attr_reader :mutex
-      end
-    end
-
     def build_log_message(level, raw_message, log_params = {}, options = {})
       message = raw_message.dup
       # check keys

data/flydata-core/spec/event/flydata_event_processor_spec.rb

@@ -9,7 +9,7 @@ module FlydataCore
       describe "#process" do
 
         context "when event has one verb handler and one any verb handler," do
-          let(:event) {FlydataEvent.new("test2","testing",nil, 1)}
+          let(:event) {FlydataEvent.new("test2", "testing", id: 1)}
           it "should call both handlers" do
             expect(AnyHandler).to receive(:handle).with(event)
             expect(Handler3).to receive(:handle).with(event)
@@ -18,7 +18,7 @@ module FlydataCore
         end
 
         context "when event has two handlers," do
-          let(:event) {FlydataEvent.new("test","testing",nil, 1)}
+          let(:event) {FlydataEvent.new("test","testing", id: 1)}
           it "should call both handlers" do
             expect(Handler1).to receive(:handle).with(event)
             expect(Handler2).to receive(:handle).with(event)

data/flydata-core/spec/event/flydata_event_sender_spec.rb

@@ -11,7 +11,7 @@ module FlydataCore
 
       context "#send_event" do
         subject{
-          send_event("test","testing",nil, 1)
+          send_event("test","testing", id: 1)
         }
         it "should raise exception" do
           expect { raise Exception}

data/flydata-core/spec/event/flydata_event_spec.rb

@@ -5,34 +5,43 @@ module FlydataCore
   module Event
     describe  FlydataEvent do
       describe "#initialize" do
-        let(:noun){"test"}
-        let(:verb){"testing"}
-        let(:data){{test:123}}
+        let(:noun) { "test" }
+        let(:verb) { "testing" }
+        let(:data) { { 'tables' => %w(table_a table_b) } }
 
-        context "when noun and verb are passed" do
-          it {
-            event = FlydataEvent.new(noun,verb, nil,1)
-            expect(event.noun).to eql(noun)
-            expect(event.verb).to eql(verb)
-            expect(event.data).to be_nil
-            expect(event.id).not_to be_nil
+        context 'when all parameters are given' do
+          let(:parameters) do
+            { data_entry_id: 1,
+              data_port_id: 2,
+              data: data,
+              id: 12,
+              created: Time.now + 60, }
+          end
+
+          it 'sets given data to instance variables' do
+            event = FlydataEvent.new(noun, verb, parameters)
+            expect(event.noun).to eq(noun)
+            expect(event.verb).to eq(verb)
+            expect(event.data).to eq(data)
+            expect(event.id).to eq(12)
             expect(event.data_entry_id).to eql(1)
-            expect(event.created).not_to be_nil
-          }
+            expect(event.data_port_id).to eql(2)
+            expect(event.created).to be_kind_of(Time)
+          end
         end
 
-        context "when noun, verb and data are passed" do
-          it {
-            event = FlydataEvent.new(noun,verb,data,1)
-            expect(event.noun).to eql(noun)
-            expect(event.verb).to eql(verb)
-            expect(event.data).to eql(data)
-            expect(event.data_entry_id).to eql(1)
+        context 'when parameters are empty' do
+          it 'sets default or nil to instance variables' do
+            event = FlydataEvent.new(noun, verb)
+            expect(event.noun).to eq(noun)
+            expect(event.verb).to eq(verb)
+            expect(event.data).to be_nil
             expect(event.id).not_to be_nil
+            expect(event.data_entry_id).to be_nil
+            expect(event.data_port_id).to be_nil
             expect(event.created).not_to be_nil
-          }
+          end
         end
-
       end
     end
   end

data/lib/flydata/command/restart.rb

@@ -6,13 +6,13 @@ module Flydata
   module Command
     class Restart < Base
       def self.slop
-        Slop.new do
-          on 'skip-helper', 'Skip restarting the Helper'
-        end
+        sender_opts = Flydata::Command::Sender.slop_start # Needs options for Sender#start
+        sender_opts.on 'skip-helper', 'Skip starting the Helper'
+        sender_opts
       end
 
       def run
-        sender = Flydata::Command::Sender.new
+        sender = Flydata::Command::Sender.new(opts)
         sender.restart
         unless opts.skip_helper?
           helper = Flydata::Command::Helper.new

data/lib/flydata/command/sender.rb

@@ -8,6 +8,7 @@ module Flydata
       def self.slop_start
         Slop.new do
           on 'n', 'no-daemon', 'Start FlyData agent as a regular program'
+          on 'e', 'no-email', 'Skip sending init-sync-start notification email'
         end
       end
       def start(options_or_show_final_message = {show_final_message: true}) # For backward compatibility. Use only as options going forward
@@ -58,7 +59,8 @@ module Flydata
         # surpress messages if fluentd is started in #try_mysql_sync
         options[:quiet] = true
         Flydata::Command::Sync.new.try_mysql_sync(
-                                        binlog_ready_callback: start_fluentd)
+          binlog_ready_callback: start_fluentd,
+          no_email: opts.no_email?)
         options[:quiet] = quiet_option
         start_fluentd.call unless fluentd_started
         if options[:show_final_message] && !options[:quiet]

data/lib/flydata/command/sync.rb

@@ -20,6 +20,7 @@ require 'flydata-core/mysql/binlog_pos'
 require 'flydata/mysql/table_ddl'
 require 'flydata-core/mysql/command_generator'
 require 'flydata/event/api_event_sender'
+require 'flydata-core/event/event_dictionary'
 require 'sigdump/setup'
 #require 'ruby-prof' # to enable profiling, also set the class' RUN_PROFILE
 
@@ -150,7 +151,7 @@ EOS
           log_info_stdout <<EOS
 ERROR!  You cannot reset tables because the previous initial sync has not been completed.  Reset the unfinished initial sync first with the following command:
 
-  flyudata sync:reset --init
+  flydata sync:reset --init
 EOS
           return
         end
@@ -672,9 +673,14 @@ EOM
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
           sync_fm.save_sync_info(@full_initial_sync, target_tables)
           # This notification will be uncommented after init_sync_finish email integration is released
-          #unless options[:sync_resumed]
-          #  FlydataCore::Event::ApiEventSender.instance.send_event("init_sync", "started", {"tables" => target_tables}, de['id'])
-          #end
+          unless options[:sync_resumed]
+            FlydataCore::Event::ApiEventSender.instance.send_event(
+              FlydataCore::Event::INIT_SYNC,
+              FlydataCore::Event::STARTED,
+              data: {"tables" => target_tables, 'no_email' => options[:no_email]},
+              data_entry_id: de['id'],
+              data_port_id: de['data_port_id'])
+          end
           dump_generator = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference'].merge('tables' => target_tables))
           if file_dump
             binlog_pos = nil
@@ -877,10 +883,10 @@ EOM
                   sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
 
                   # send record count for the table
-                  if mysql_table &&
+                  if table_name.to_s != '' &&
                      state == Flydata::Parser::Mysql::MysqlDumpParser::State::CREATE_TABLE
                     # all records for `mysql_table` have been sent
-                    send_record_counts(de, sync_fm, mysql_table.table_name,
+                    send_record_counts(de, sync_fm, table_name,
                                        send_record_counts_threads)
                     log_info_stdout("  -> Finished sending data for table '#{table_name}'...")
                   end

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -29,6 +29,9 @@ module Mysql
     end
 
     private
+    def binlog_pos(record)
+      "#{@context.current_binlog_file}\t#{record['next_position'] - record['event_length']}"
+    end
 
     def supported_database
       @context.database
@@ -42,8 +45,7 @@ module Mysql
       acceptable = @context.tables.include?(table)
 
       if acceptable and @table_binlog_pos[record['table_name']]
-        if @table_binlog_pos[record['table_name']] >= Flydata::Mysql::BinLogPosition.new(
-          "#{@context.current_binlog_file}\t#{record['next_position'] - record['event_length']}")
+        if @table_binlog_pos[record['table_name']] >= Flydata::Mysql::BinLogPosition.new(binlog_pos(record))
           acceptable = false
         else
           @context.sync_fm.delete_table_binlog_pos(record['table_name'])
@@ -73,7 +75,6 @@ module Mysql
       return unless acceptable_table?(record, table) && acceptable_event?(type, table)
 
       table_rev = @context.table_revs[table]
-      position = record['next_position'] - record['event_length']
 
       # Add common information to each record
       records.each do |r|
@@ -84,7 +85,7 @@ module Mysql
         r[TYPE] = type
         r[RESPECT_ORDER] = true
         r[TABLE_NAME] = table
-        r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
+        r[SRC_POS] = binlog_pos(record)
         r[TABLE_REV] = table_rev
         r[V] = FlydataCore::Record::V2
       end

data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb

@@ -51,13 +51,11 @@ class DatabaseDdlQueryHandler < DdlQueryHandler
 
     return unless acceptable_event?(type)
 
-    position = record['next_position'] - record['event_length']
-
     # Add common information to each record
     records.each do |r|
       r[TYPE] = type
       r[RESPECT_ORDER] = true
-      r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
+      r[SRC_POS] = binlog_pos(record)
       r[V] = FlydataCore::Record::V2
     end
 

data/lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb

@@ -15,8 +15,8 @@ module Mysql
 
     def process(record)
       #Issuing warning message only for the current database.
-      if record["db_name"] == @context.database
-        $log.warn("DROP DATABASE detected. A full re-sync is required to provide sync consistency. db_name: #{record["db_name"]}")
+      if acceptable_db?(record)   #record["db_name"] == @context.database
+        $log.warn("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record['normalized_query']}' binlog_pos:'#{binlog_pos(record)}'")
       end
       #NOTE: No emit_record here because this record should not be sent to data servers for now
     end

data/spec/flydata/command/sync_spec.rb

@@ -108,6 +108,7 @@ module Flydata
           expect(subject).to receive(:ask_yes_no).and_return(true).at_least(:once)
           Flydata::Parser::Mysql::DatabaseSizeCheck.any_instance.should_receive(:get_db_bytesize).and_return(db_byte)
           Flydata::MysqlCompatibilityCheck.any_instance.should_receive(:check)
+          expect_any_instance_of(FlydataCore::Event::ApiEventSender).to receive(:send_event).once
         end
         context 'with no stream option' do
           before do

data/spec/flydata/fluent-plugins/mysql/drop_database_query_handler_spec.rb

@@ -19,9 +19,11 @@ module Mysql
       shared_examples "test different dbs" do
         context "for the target db" do
           let(:database) { target_database }
-  
+          let(:binlog_pos) { "#{context.current_binlog_file}\t#{record['next_position'] - record['event_length']}" }
+
           it "shows a warning message" do
-            expect($log).to receive(:warn).with("DROP DATABASE detected. A full re-sync is required to provide sync consistency. db_name: #{record["db_name"]}").once
+            expect($log).to receive(:warn).with("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record["normalized_query"]}' binlog_pos:'#{binlog_pos}'")
+
             subject
           end
         end

data/spec/flydata/json/json_ext_spec.rb

@@ -65,5 +65,29 @@ describe JSON do
         it_behaves_like "producing the expected result"
       end
     end
+
+    context 'testing memory' do
+      let(:keys) { ['id', 'user_id', 'name', 'timestamp'] }
+      let(:values) { ['29388381012', '192938439', 'Muy Fiel y Reconquistadora Ciudad de San Felipe y Santiago de Montevideo', '2015-12-01 12:34:56' ] }
+      let(:expected_result) { %Q|{"id":"29388381012","user_id":"192938439","name":"Muy Fiel y Reconquistadora Ciudad de San Felipe y Santiago de Montevideo","timestamp":"2015-12-01 12:34:56"}| }
+      xcontext 'when run many many times' do
+        thread = nil
+        before do
+          GC.stress = true
+          thread = Thread.new { loop {rand(9).to_s * rand(9)} }
+        end
+        it do
+          6000.times do
+            result = described_class.generate_kv_pairs(keys, values)
+            rand(9).to_s * rand(9)
+            expect(result).to eq expected_result
+          end
+        end
+        after do
+          GC.stress = false
+          thread.kill
+        end
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-01 00:00:00.000000000 Z
+date: 2015-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -513,6 +513,7 @@ files:
 - flydata-core/lib/flydata-core/core_ext/object.rb
 - flydata-core/lib/flydata-core/core_ext/object/prepend.rb
 - flydata-core/lib/flydata-core/errors.rb
+- flydata-core/lib/flydata-core/event/event_dictionary.rb
 - flydata-core/lib/flydata-core/event/event_handler_base.rb
 - flydata-core/lib/flydata-core/event/flydata_event.rb
 - flydata-core/lib/flydata-core/event/flydata_event_handler_registry.rb
@@ -744,7 +745,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: FlyData Agent