flydata 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ff848158232f9a2b6ad6154f56963c64ce6b41a1
-  data.tar.gz: 1b9083d2ff0f6f8ea9647fa27a6be4b5fede2ec4
+  metadata.gz: 45df6073e7d4b59b307a695058cd1ee70366eb89
+  data.tar.gz: cdac2f4feb8dfdd4531c14f97a38f1dd2df91f48
 SHA512:
-  metadata.gz: b1d53be1bed30298c0f8f3b49b4c9c1882a86044e28b3f3c5d29253d99c6137bcae518d9cc8e22d2e5ab0ee83c91c29cbadb6e4e7188f42a29a02cfac162f4ef
-  data.tar.gz: 246b84391256eaeb7c97148eb9a31e3a85157b303ebc2538d70896faa175a4b570ca7f75b1a574ff9cf964a81b9c51df6ccff8aad75b8877b002e48f143de049
+  metadata.gz: df38e5c265ee0b0e4f9cfcde1a6f219db1fa0b4df013d4f79d1a9dc648d4d5684210560153d91ace7a5267cde76c4d26d00d5ee95e757c0acee2867b4ea6be9c
+  data.tar.gz: 7ec85d6e13314f445649d64c1769597f2fd192d35c5a6b76b42043accfca4be1d2fc5d95b7c40de7f00e462876bd67076e0102efa0634f755af4d88ebba2f2f0

data/VERSION

@@ -1 +1 @@
-0.6.2
+0.6.3

data/lib/flydata/api/data_entry.rb

@@ -12,7 +12,7 @@ module Flydata
 
       def buffer_stat(data_entry_id, options = {})
         tables = options[:tables] ? options[:tables].join(',') : ''
-        @client.get("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}?tables=#{tables}")
+        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
       end
 
       def table_status(data_entry_id, options = {})

data/lib/flydata/command/exclusive_runnable.rb

@@ -74,6 +74,14 @@ module ExclusiveRunnable
       cmd = modules.last.downcase
       subcommand == :run ? cmd : "#{cmd}:#{subcommand}"
     end
+    def command_process_exist?
+      exclusive_run_info = load_exclusive_run_info
+      if exclusive_run_info
+        process_exist?(exclusive_run_info['pid'])
+      else
+        false
+      end
+    end
   end
 end
 

data/lib/flydata/command/sync.rb

@@ -338,6 +338,17 @@ EOS
 
       private
 
+      def generate_kv_pairs(keys, values)
+        h = {}
+        i = 0
+        sz = keys.size
+        while i < sz
+          h[keys[i]] = values[i]
+          i += 1
+        end
+        JSON.generate(h)
+      end
+
       def _check(options = {})
         options[:stop_agent] ||= false
 
@@ -350,7 +361,7 @@ EOS
         abnormal_shutdown = false
         begin
           begin
-            flush_buffer_and_stop(@full_tables, force: false, timeout: 55)
+            flush_buffer_and_stop(@full_tables, force: false, timeout: 7200) # 2 hrs.  It can take more but 2 hours are long enough to pull someone's attention.
           rescue ServerDataProcessingTimeout => e
             data_stuck_at = e.state
           end
@@ -844,6 +855,9 @@ EOM
                 convert_to_flydata_values(mysql_table, values)
                 json = JSON.generate_kv_pairs(mysql_table.column_names, values)
                 values.clear
+                unless json.start_with? '{'
+                  raise DumpParseError.new("Broken JSON record json:#{json[0..100]}")
+                end
                 {table_name: mysql_table_name, log: json}
               end
               # release resources to prevent the closure from keeping it

data/lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb

@@ -3,7 +3,7 @@ require 'flydata/fluent-plugins/mysql/ddl_query_handler'
 module Mysql
   class DropDatabaseQueryHandler < DatabaseDdlQueryHandler
     # For MySQL, database and schema are exchangable
-    PATTERN = /^DROP DATABASE|SCHEMA/i
+    PATTERN = /^DROP (DATABASE|SCHEMA)/i
 
     def initialize(context)
       super
@@ -16,7 +16,7 @@ module Mysql
     def process(record)
       #Issuing warning message only for the current database.
       if acceptable_db?(record)   #record["db_name"] == @context.database
-        $log.warn("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record['normalized_query']}' binlog_pos:'#{binlog_pos(record)}'")
+        $log.error("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record['normalized_query']}' binlog_pos:'#{binlog_pos(record)}'")
       end
       #NOTE: No emit_record here because this record should not be sent to data servers for now
     end

data/lib/flydata/helper/action/check_abnormal_shutdown.rb

@@ -1,4 +1,5 @@
 require 'flydata'
+require 'flydata/command/base'
 require 'flydata/helper/base_action'
 
 module Flydata
@@ -7,8 +8,12 @@ module Flydata
       class CheckAbnormalShutdown < BaseAction
         def execute(opts = {})
           if abnormal_shutdown_occurred?
-            log_info "Abnormal shutdown occurred.  Restart the agent."
-            yield :restart_agent, nil
+            if Command::Base.command_process_exist?
+              log_info "Abnormal shutdown occurred but another command is stil running.  Skip agent restart."
+            else
+              log_info "Abnormal shutdown occurred.  Restart the agent."
+              yield :restart_agent, nil
+            end
           end
         end
 

data/spec/flydata/fluent-plugins/mysql/drop_database_query_handler_spec.rb

@@ -21,16 +21,16 @@ module Mysql
           let(:database) { target_database }
           let(:binlog_pos) { "#{context.current_binlog_file}\t#{record['next_position'] - record['event_length']}" }
 
-          it "shows a warning message" do
-            expect($log).to receive(:warn).with("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record["normalized_query"]}' binlog_pos:'#{binlog_pos}'")
+          it "shows a error message" do
+            expect($log).to receive(:error).with("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record["normalized_query"]}' binlog_pos:'#{binlog_pos}'")
 
             subject
           end
         end
         context "for another db" do
           let(:database) { "another_db" }
-          it "does not show a warning message" do
-            expect($log).not_to receive(:warn)
+          it "does not show a error message" do
+            expect($log).not_to receive(:error)
             subject
           end
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-08 00:00:00.000000000 Z
+date: 2016-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client