flydata 0.5.20 → 0.5.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2731322ee5aa072665e488692665d0ba9732f720
-  data.tar.gz: 681f290e03fdaeccf47ecb80d17d1a039d528458
+  metadata.gz: 516813af4ea0557232dfd85e16c84b3666fa6576
+  data.tar.gz: 8fa4a979e70fde765532a000952f1deab76a0973
 SHA512:
-  metadata.gz: c443a473619d395c61bb6e08b6ce5de4b0c874efecafe0a24bff411c28287aa4a6e0b5d33537a316be7428a7f9700f9f519d781cfcf16d2c808fc93e6832fa96
-  data.tar.gz: 860cd66add7614fdbc5f68bba8be6a41adcfbaf704a02caa4bb22c3aca85f6b82261147ab52eaa3e2edea379ae02b7c04b13df812ffcbcab716bf720f5478ceb
+  metadata.gz: b3eac9c57d1adced1b2657d12797fd076744f8659f032017c6826f92b6f6b1f7b7a6a40bdc476a9fd369402da50eeb92bc9c6022a712b9a6fa70c00648c55a1d
+  data.tar.gz: 25f4a8c298d41f3efdbf9f9560c2d215699af1ebfad086fd8d52ae51a00004a8893bdc288ef0ac7331f7d96030a542e1ec18cf941991907ca8ec8e1daee082b2

data/VERSION

@@ -1 +1 @@
-0.5.20
+0.5.21

data/lib/flydata/command/sync.rb

@@ -560,7 +560,6 @@ EOS
 
       # Don't call the method unless init sync needs to be run for a table
       def sync_mysql_to_redshift(de, options = {})
-        FlydataCore::Event::ApiEventSender.instance.send_event("init_sync", "started", {"tables"=>target_tables}, de['id'])
         dp = flydata.data_port.get
         sync_fm = create_sync_file_manager(de)
 
@@ -655,6 +654,10 @@ EOM
           log_info_stdout("Setting binary log position and exporting data from the database.")
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
           sync_fm.save_sync_info(@full_initial_sync, target_tables)
+          # This notification will be uncommented after init_sync_finish email integration is released
+          #unless options[:sync_resumed]
+          #  FlydataCore::Event::ApiEventSender.instance.send_event("init_sync", "started", {"tables" => target_tables}, de['id'])
+          #end
           dump_generator = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference'].merge('tables' => target_tables))
           if file_dump
             binlog_pos = nil

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -156,7 +156,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       when :event_arrived_finally
         $log.info "Binary log event has come at #{timestamp}"
       when :event_idle_timeout
-        $log.error "No binary log event since #{timestamp}. Restart the process."
+        $log.warn "No binary log event since #{timestamp}. Restarting the process."
         Process.kill(:HUP, Process.ppid)
       end
     end

data/lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb

@@ -14,7 +14,10 @@ module Mysql
     end
 
     def process(record)
-      $log.warn("DROP DATABASE detected. A full re-sync is required to provide sync consistency")
+      #Issuing warning message only for the current database.
+      if record["db_name"] == @context.database
+        $log.warn("DROP DATABASE detected. A full re-sync is required to provide sync consistency. db_name: #{record["db_name"]}")
+      end
       #NOTE: No emit_record here because this record should not be sent to data servers for now
     end
   end

data/spec/flydata/fluent-plugins/mysql/drop_database_query_handler_spec.rb

@@ -4,52 +4,44 @@ require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
 
 module Mysql
   describe DropDatabaseQueryHandler do
+    let(:subject_object) { described_class.new(context) }
     include_context "query handler context"
 
-    describe '#process' do
-      let(:drop_database_query) { "DROP database foo" }
+    let(:drop_keyword) { "DATABASE" }
+    let(:drop_database_query) { "DROP #{drop_keyword} #{database}" }
 
+    describe '#process' do
+      subject { subject_object.process(record) }
       before do
-        allow(record).to receive(:[]).with("query").and_return(drop_database_query)
-        allow(record).to receive(:[]).with("normalized_query").and_return(drop_database_query)
+        expect(Fluent::Engine).to receive(:emit).never
       end
 
-      shared_examples "process drop database queries correctly" do
-        let(:expected_record) do
-          {
-            query: drop_database_query,
-            type: :drop_database,
-            respect_order: true,
-            src_pos: "#{current_binlog_file}\t#{next_position - event_length}",
-            v: flydata_record_version
-          }
+      shared_examples "test different dbs" do
+        context "for the target db" do
+          let(:database) { target_database }
+  
+          it "shows a warning message" do
+            expect($log).to receive(:warn).with("DROP DATABASE detected. A full re-sync is required to provide sync consistency. db_name: #{record["db_name"]}").once
+            subject
+          end
         end
-
-        context "for the registered database" do
-          it "should call Fluent's emit with appropriate params" do
-            # Should mock up the method to return true
-            #expect(Fluent::Engine).to receive(:emit).with(tag, timestamp, expected_record)
-            expect(Fluent::Engine).to receive(:emit).never
-            expect($log).to receive(:warn)
-            expect(subject.process(record))
+        context "for another db" do
+          let(:database) { "another_db" }
+          it "does not show a warning message" do
+            expect($log).not_to receive(:warn)
+            subject
           end
         end
       end
 
-      shared_examples "skip processing queries" do
-        it "should not call Fluent's emit" do
-          expect(Fluent::Engine).to receive(:emit).never
-          subject.process(record)
-        end
+      context "with drop database query" do
+        let(:drop_keyword) { "DATABASE" }
+        include_examples "test different dbs"
       end
 
-      context "drop database queries with database keyword" do
-        let(:drop_database_query) { "DROP DATABASE foo" }
-        include_examples "process drop database queries correctly"
-      end
-      context "drop database queries with schema keyword" do
-        let(:drop_database_query) { "DROP SCHEMA foo" }
-        include_examples "process drop database queries correctly"
+      context "with drop schema query" do
+        let(:drop_keyword) { "SCHEMA" }
+        include_examples "test different dbs"
       end
     end
   end

data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb

@@ -1,6 +1,7 @@
 module Mysql
   shared_context "query handler context" do
-    let(:database) { "testdb" }
+    let(:target_database) { "testdb" }
+    let(:database) { target_database }
     let(:table) { "foo" }
     let(:seq) { 200 }
     let(:sync_fm) do
@@ -18,7 +19,7 @@ module Mysql
     let(:context) do
       r = double('context')
       allow(r).to receive(:sync_fm).and_return(sync_fm)
-      allow(r).to receive(:database).and_return(database)
+      allow(r).to receive(:database).and_return(target_database)
       allow(r).to receive(:tables).and_return([table])
       allow(r).to receive(:table_meta).and_return(table_meta)
       allow(r).to receive(:current_binlog_file).and_return(current_binlog_file)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.20
+  version: 0.5.21
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-20 00:00:00.000000000 Z
+date: 2015-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -743,7 +743,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: FlyData Agent