flydata 0.8.1 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68c50586cde303759f5ac328ff7562286e7480e3
-  data.tar.gz: 83c940bbdccc612ee4fb965d98d28e62b53c8e44
+  metadata.gz: 8d414dcb17fd8fac585af5e47bc79b26809a276c
+  data.tar.gz: e2ee59e6864c7c75e4a36bb439d4b172e307ba40
 SHA512:
-  metadata.gz: 1031061296b5f598ce63109738fc5b75d83bfa1a754ae5ea5afa71ad1680159205ca28e54d441797204edd2a3865901e5087ac2c4e8207b71f2ddc601dba76fa
-  data.tar.gz: c40898079b909de3f880f9afd0ffbd8622fc5058b10ce5780e8440cca9d5d5c047160583e3a2c330e1864a4c7000163b079520c6e07bf5569a969200ed443343
+  metadata.gz: a205d6dfb530938d493d39e264b2efe6f4c319cf2c94951e7a19a9be99e3f13ea0db0bbe9453643072e46adc4be35385fc1e54ec08510c2c43c8b092e03e8fc6
+  data.tar.gz: 4ea437373319cfaab4dd2a6a0ae28401df803767824c425e82b795dbe12e121a8cdab89d6f17a9cb1d00117b6053188b5b58df20ef497c828510dc8f8d63ad10

data/VERSION

@@ -1 +1 @@
-0.8.1
+0.8.2

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.8.1"
+  s.version = "0.8.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2017-02-08"
+  s.date = "2017-02-11"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]

data/lib/flydata.rb

@@ -26,6 +26,8 @@ module Flydata
   FLYDATA_GEM_BIN = File.join(FLYDATA_GEM_HOME, 'bin')
   FLYDATA_TMPL_DIR = File.join(FLYDATA_GEM_HOME, 'tmpl')
   FLYDATA_FLUENT_PLUGIN_DIR = File.join(FLYDATA_GEM_HOME, 'lib', 'flydata', 'fluent-plugins')
+  FLYDATA_DUMP_DIR = ENV['FLYDATA_DUMP_DIR'] || File.join(FLYDATA_HOME, 'dump')
+  FLYDATA_BACKUP_DIR = ENV['FLYDATA_BACKUP_DIR'] || File.join(FLYDATA_HOME, 'backup')
 
   FLYDATA_SERVERINFO = File.join(FLYDATA_GEM_BIN, 'serverinfo')
   FLYDATA_LOG = File.join(FLYDATA_HOME, 'flydata.log')

data/lib/flydata/command/helper.rb

@@ -66,9 +66,11 @@ module Flydata
       def restart(options = {})
         if running?
           log_info_stdout("Restarting Helper.", {}, options)
-          run_command(stop_cmd, options)
+          unless run_command(stop_cmd, options)
+            raise "[error] Failed to stop helper process. Please run 'flydata helper:restart' after checking the process."
+          end
         end
-        raw_start
+        start(options)
       end
 
        def reload(options = {})
@@ -119,8 +121,8 @@ if #{running_cmd}; then
   while #{running_cmd}; do
     sleep 1.0
     if [ $i -gt 10 ]; then  # 10sec
-      echo "Timeout warning: It failed to stop within 10 seconds. The process will be killed with SIGKILL."
-      kill -9 -$(ps --pid `cat #{@helper_config.helper_pid_path}` --no-heading -o "%r" | tr -d ' ')
+      echo "Timeout warning: It failed to stop within 10 seconds. The process will be left." 1>&2
+      exit 1
     else
       i=`expr $i + 1`
     fi

data/lib/flydata/command/sync.rb

@@ -86,7 +86,7 @@ module Flydata
       rescue Source::UnsupportedSourceError
         return
       rescue => e
-        log_error("[error] Unexpcted error happend during inital sync. error:#{e}")
+        log_error("[error] Unexpected error happened during initial sync. error:#{e}")
         raise e
       end
 

data/lib/flydata/helper/action_ownership_channel.rb

@@ -91,6 +91,13 @@ module Flydata
         @action_ownership_map.dup
       end
 
+      def remote_actions_exist?
+        queue.any?{|k, v| v.kind_of?(Hash) &&
+                   v[:action_info].kind_of?(Hash) &&
+                   v[:action_info].has_key?(:id) }
+
+      end
+
       def dump
         queue_info = @queue.collect{|k,v|
           "- #{k} #{v[:action_info]}"

data/lib/flydata/helper/server.rb

@@ -57,6 +57,11 @@ module Flydata
       def after_run
         log_debug "after_run"
         @scheduler.shutdown if @scheduler
+
+        if @action_ownership_channel &&
+            @action_ownership_channel.remote_actions_exist?
+          log_error("[error] Unprocessed actions are left in action_ownership_channel queue #{@action_ownership_channel.dump}")
+        end
       end
 
       def custom_log_items

data/lib/flydata/sync_file_manager.rb

@@ -7,8 +7,8 @@ require 'flydata/source/source_pos'
 
 module Flydata
   class SyncFileManager
-    DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
-    BACKUP_DIR = ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
+    DUMP_DIR = FLYDATA_DUMP_DIR
+    BACKUP_DIR = FLYDATA_BACKUP_DIR
     TABLE_POSITIONS_DIR = FLYDATA_TABLE_POSITIONS_DIR
 
     INITIAL_SYNC_SEQ = 1

data/spec/flydata/command/helper_spec.rb

@@ -84,6 +84,7 @@ module Flydata
         context "when the Helper process is running" do
           it "stops and starts the process" do
             expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
+            expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
             expect(Open3).to receive(:capture3) do |env, cmd|
               expect(cmd).to match(STOP_REGEX)
             end
@@ -93,7 +94,7 @@ module Flydata
         end
         context "when the Helper process is not running" do
           it "starts the process" do
-            expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
+            expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0").twice
             expect(Flydata::Helper::Server).to receive(:run)
             subject
           end

data/spec/flydata/helper/action_ownership_channel_spec.rb

@@ -142,6 +142,28 @@ module Flydata
             to eq(Time.local(2014, 8, 19, 16, 11, 24).to_f)
         end
       end
+
+      describe '#remote_actions_exist?' do
+        subject { channel.remote_actions_exist? }
+
+        context 'when queue is empty' do
+          it { is_expected.to be(false) }
+        end
+
+        context 'when action without :id exists in queue' do
+          before do
+            channel.request_action(:send_logs, {})
+          end
+          it { is_expected.to be(false) }
+        end
+
+        context 'when action with :id exists in queue' do
+          before do
+            channel.request_action(:send_logs, {id: 1})
+          end
+          it { is_expected.to be(true) }
+        end
+      end
     end
   end
-end    
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-08 00:00:00.000000000 Z
+date: 2017-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client