flydata 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7ec2c4c57c3aa4ddcb5f419474f7558300f0f242
-  data.tar.gz: 6f7148b0c6fc8a114938d1a698a9917fe1ccbe95
+  metadata.gz: 68c50586cde303759f5ac328ff7562286e7480e3
+  data.tar.gz: 83c940bbdccc612ee4fb965d98d28e62b53c8e44
 SHA512:
-  metadata.gz: b89d17e0dd83ccadfa794a3819ae4074336b9b45af56523fcf54349feb115971460e28648f11d3e04e942e6e032e8a4b3fddde891b370c868221d5a8a533adba
-  data.tar.gz: 60e612f837ca719b0da1b60efbde9057576277c15b325fa52e7a0bcc647c9008da91d43bb292ecc59a1eac1eeb426dcef8abce70bba2364cbfe25c28634dc42a
+  metadata.gz: 1031061296b5f598ce63109738fc5b75d83bfa1a754ae5ea5afa71ad1680159205ca28e54d441797204edd2a3865901e5087ac2c4e8207b71f2ddc601dba76fa
+  data.tar.gz: c40898079b909de3f880f9afd0ffbd8622fc5058b10ce5780e8440cca9d5d5c047160583e3a2c330e1864a4c7000163b079520c6e07bf5569a969200ed443343

data/VERSION

@@ -1 +1 @@
-0.8.0
+0.8.1

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.8.0"
+  s.version = "0.8.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2017-02-03"
+  s.date = "2017-02-08"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]
@@ -208,6 +208,7 @@ Gem::Specification.new do |s|
     "lib/flydata/helper/action/check_remote_actions.rb",
     "lib/flydata/helper/action/clear.rb",
     "lib/flydata/helper/action/repair.rb",
+    "lib/flydata/helper/action/reset.rb",
     "lib/flydata/helper/action/restart_agent.rb",
     "lib/flydata/helper/action/resync.rb",
     "lib/flydata/helper/action/send_logs.rb",
@@ -367,6 +368,7 @@ Gem::Specification.new do |s|
     "spec/flydata/helper/action/check_remote_actions_spec.rb",
     "spec/flydata/helper/action/clear_spec.rb",
     "spec/flydata/helper/action/repair_spec.rb",
+    "spec/flydata/helper/action/reset_spec.rb",
     "spec/flydata/helper/action/restart_agent_spec.rb",
     "spec/flydata/helper/action/resync_spec.rb",
     "spec/flydata/helper/action/send_logs_spec.rb",

data/lib/flydata.rb

@@ -35,6 +35,7 @@ module Flydata
   FLYDATA_HELPER_CONF = File.join(FLYDATA_HELPER_HOME, "helper.conf")
   FLYDATA_CMD_PATH = ENV['FLYDATA_CMD_PATH'] || "flydata"
   FLYDATA_CLIENT_BUFFER_DIR = File.join(FLYDATA_HOME, 'buffer')
+  FLYDATA_DEBUG_FILE = File.join(FLYDATA_HOME, '.debug')
 
   VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
   include Flydata::Heroku

data/lib/flydata/command/sync.rb

@@ -128,6 +128,7 @@ EOS
           on 'i', 'init', 'Resets unfinished initial sync'
           on 'force-run', 'Run forcefully, ignoring exclusive run info'
           on 'f', 'force', "Resets tables including append only tables.  Use this option only when you want to stop sync for the append only table permanently."
+          on 'include-unregistered', "Resets tables including unregistered tables.  Use this option only when resetting removed tables."
         end
       end
 
@@ -141,7 +142,7 @@ EOS
           tables = []
           reset_init = false
         end
-        sync_resumed, auto_create = set_current_tables(tables, resume: !opts[:all])
+        sync_resumed, auto_create = set_current_tables(tables, resume: !opts[:all], include_unregistered: opts.include_unregistered?)
         target_tables = opts[:all] ? @full_tables : @input_tables
         target_append_only_tables = target_tables & @append_only_tables
         target_full_sync_tables = target_tables - @append_only_tables
@@ -1552,7 +1553,7 @@ Thank you for using FlyData!
                                             (@unsynced_tables == @full_tables)
         sync_fm.close
 
-        verify_input_tables(@input_tables, @full_tables)
+        verify_input_tables(@input_tables, @full_tables, options)
 
         [sync_resumed, auto_create]
       end
@@ -1607,13 +1608,17 @@ Thank you for using FlyData!
         SyncFileManager.new(de, source)
       end
 
-      def verify_input_tables(input_tables, all_tables)
+      def verify_input_tables(input_tables, all_tables, options = {})
         return unless input_tables
-        inval_table = []
-        input_tables.each do |tab|
-          inval_table << tab unless all_tables.include?(tab)
+        inval_tables = input_tables - all_tables
+        unless inval_tables.empty?
+          msg = "These tables are not registered tables: #{inval_tables.join(", ")}"
+          if options[:include_unregistered]
+            log_warn_stderr(msg)
+          else
+            raise msg
+          end
         end
-        raise "These tables are not registered tables: #{inval_table.join(", ")}" unless inval_table.empty?
       end
 
       def check_master_position_files

data/lib/flydata/helper/action/agent_action.rb

@@ -9,15 +9,28 @@ module Flydata
         def execute(opts = {})
           sub_command = command(opts)
           cmd = FLYDATA_CMD % { sub_command: sub_command }
-          log_debug("Executing `#{cmd}`")
+          log_info("Executing `#{cmd}`")
           o, e, s = Util::Shell.run_cmd(cmd)
-          log_error(e) if not e.to_s.empty?
-          log_error("Could not execute #{cmd}. Response status : #{s}") unless (s.to_i == 0)
+          log_warn(e) if not e.to_s.empty?
+          log_error("[error] Could not execute #{cmd}. Response status: #{s}") unless (s.to_i == 0)
         end
 
         # Override this. Needs to be implemented by the subclass
         #def command(opts = {})
         #end
+
+        def get_tables(opts)
+          tables = config_hash(opts)[:tables]
+          tables_str = case tables
+                       when Array
+                         tables.join(' ')
+                       when String  # --all option
+                         tables
+                       else
+                         raise "Wrong tables parameter - tables:\"#{tables}\""
+                       end
+          tables_str
+        end
       end
     end
   end

data/lib/flydata/helper/action/reset.rb

@@ -0,0 +1,14 @@
+require 'flydata/helper/action/agent_action'
+
+module Flydata
+  module Helper
+    module Action
+      class Reset <  AgentAction
+        def command(opts = {})
+          arg = get_tables(opts)
+          "sync:reset -y --force --include-unregistered %s" % [arg]
+        end
+      end
+    end
+  end
+end

data/lib/flydata/helper/action/resync.rb

@@ -5,15 +5,7 @@ module Flydata
     module Action
       class Resync <  AgentAction
         def command(opts = {})
-          tables = config_hash(opts)[:tables]
-          arg = case tables
-                when Array
-                  tables.join(' ')
-                when String  # --all option
-                  tables
-                else
-                  raise "Wrong tables parameter - tables:\"#{tables}\""
-                end
+          arg = get_tables(opts)
           "sync:resync -y --force %s" % [arg]
         end
       end

data/lib/flydata/helper/action/send_logs.rb

@@ -16,7 +16,6 @@ module Flydata
         end
 
         def execute(opts = {})
-          log_debug("Sending logs")
           num_of_lines = DEFAULT_NUM_OF_LINES
           action_id = opts[:id]
           begin
@@ -24,6 +23,7 @@ module Flydata
           rescue
             # Use default number of lines if config is nil, mal-formed etc
           end
+          log_info("Sending logs - num_of_lines:#{num_of_lines}")
           tailed_lines = tail(FLYDATA_LOG, num_of_lines)
           @api_client.agent.send_logs(action_id, tailed_lines)
         end

data/lib/flydata/helper/action_ownership.rb

@@ -52,6 +52,7 @@ module Flydata
           self.new(:resync, true, Action::Resync),
           self.new(:clear, true, Action::Clear),
           self.new(:stop_helper, true, Action::StopHelper),
+          self.new(:reset, true, Action::Reset),
         ].inject({}) do |h, action|
           h[action.action_name] = action
           h

data/lib/flydata/helper/action_ownership_channel.rb

@@ -22,7 +22,7 @@ module Flydata
         action_ownership = @action_ownership_map[action_name]
         if action_ownership.nil?
           # unknown action.  probably a new action
-          log_warn "unsupported action `#{action_name}'.  Skip."
+          log_error "[error] unsupported action `#{action_name}'. Skip."
           return false
         end
 
@@ -50,21 +50,27 @@ module Flydata
         self.synchronize do
           return nil if @queue.empty?
 
-          # Wait until action will be processed
-          action = @queue.values.first
-          action_ownership = action[:action_ownership]
-          return nil if action_ownership.processing?
-
           # Check resource change flag
-          if action_ownership.resource_change &&
-            @action_ownership_map.any? { |name, act_own|
+          rsc_act_running = @action_ownership_map.any? { |name, act_own|
               (act_own.resource_change && act_own.processing?) }
-            return nil
+
+          target_key, target_action = @queue.find do |key, action|
+            action_ownership = action[:action_ownership]
+
+            # Wait until a same action will be processed
+            next if action_ownership.processing?
+
+            # Skip if another resource change action is running
+            next if action_ownership.resource_change && rsc_act_running
+
+            true
           end
 
-          @queue.shift  # delete last acion from queue
-          action_ownership.owner = new_owner
-          action
+          return nil if target_key.nil? || target_action.nil?
+          @queue.delete(target_key)
+
+          target_action[:action_ownership].owner = new_owner
+          target_action
         end
       end
 
@@ -86,15 +92,24 @@ module Flydata
       end
 
       def dump
-        $log.info "\n" + <<EOT
-========
-action_ownership_channel dump
+        queue_info = @queue.collect{|k,v|
+          "- #{k} #{v[:action_info]}"
+        }.join("\n")
 
-queue:
-#{@queue}
+        map_info = @action_ownership_map.collect{|k,v|
+          "- #{v.action_name} owner:\"#{v.owner}\" " +
+          "retry:#{v.retry_count} " +
+          "last:\"#{v.last_processed_time > 0 ? Time.at(v.last_processed_time).utc : '-'}\""
+        }.join("\n")
 
-action_ownership_map:
-#{@action_ownership_map}
+        <<EOT
+
+========
+action_ownership_channel dump
+# queue:
+#{queue_info}
+# action_ownership_map:
+#{map_info}
 ========
 EOT
       end

data/lib/flydata/helper/scheduler.rb

@@ -7,6 +7,7 @@ module Flydata
 
       RUN_INTERVAL = 1.0  #second
       RUN_GC_INTERVAL = 60.0 * 5  #seconds
+      DEBUG_DUMP_INTERVAL = 60.0  #seconds
 
       def initialize(helper_conf, server)
         @stop_flag = ServerEngine::BlockingFlag.new
@@ -98,10 +99,22 @@ module Flydata
           end
         end
 
+        dump_queue_if_needed
         run_gc_if_needed
         wait_until_next_turn(start_time)
       end
 
+      def dump_queue_if_needed
+        now = Time.now
+        @last_dump_time ||= now
+        if (@last_dump_time + DEBUG_DUMP_INTERVAL) < now
+          if File.exists?(FLYDATA_DEBUG_FILE)
+            log_info(@server.action_ownership_channel.dump) rescue nil
+          end
+          @last_dump_time = now
+        end
+      end
+
       def run_gc_if_needed
         now = Time.now
         @last_gc_time ||= now

data/lib/flydata/helper/worker.rb

@@ -58,6 +58,7 @@ module Flydata
           action_ownership.action_class.new(config).execute(action_info) do |new_action_name, new_action_info = nil|
             # request action if the action requires
             server.action_ownership_channel.request_action(new_action_name, new_action_info)
+            log_info("New action was requested - action:#{new_action_name} action_info:#{new_action_info}")
             #server.action_ownership_channel.dump
           end
           action_ownership.reset_retry_count

data/spec/flydata/command/sync_spec.rb

@@ -454,7 +454,7 @@ module Flydata
           let(:command_options) { ["-y", "--all"] }
           let(:sync_info) { nil }
           it do
-            expect(subject_object).to receive(:set_current_tables).with([], resume: false).and_call_original
+            expect(subject_object).to receive(:set_current_tables).with([], resume: false, include_unregistered: false).and_call_original
             expect(subject_object).to receive(:wait_for_server_buffer).with( {:timeout=>timeout_value, :tables=>arg_tables} )
             expect(subject_object).to receive(:cleanup_sync_server).with( default_data_entry, arg_tables )
             subject
@@ -482,7 +482,7 @@ module Flydata
             let(:sync_info_tables) { ["table1", "table2", "table4"] }
             let(:target_tables_for_api) { [] }
             it do
-              expect(subject_object).to receive(:set_current_tables).with([], resume: true).and_call_original
+              expect(subject_object).to receive(:set_current_tables).with([], resume: true, include_unregistered: false).and_call_original
               expect(subject_object).to receive(:wait_for_server_buffer).with(
                                           {:timeout=>timeout_value, :tables=>target_tables_for_api} )
               expect(subject_object).to receive(:cleanup_sync_server).with(
@@ -496,7 +496,7 @@ module Flydata
             let(:sync_info_tables) { ["table2"] }
             let(:target_tables_for_api) { sync_info_tables }
             it do
-              expect(subject_object).to receive(:set_current_tables).with([], resume: true).and_call_original
+              expect(subject_object).to receive(:set_current_tables).with([], resume: true, include_unregistered: false).and_call_original
               expect(subject_object).to receive(:wait_for_server_buffer).with(
                                           {:timeout=>timeout_value, :tables=>target_tables_for_api} )
               expect(subject_object).to receive(:cleanup_sync_server).with(
@@ -519,7 +519,7 @@ module Flydata
             allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(full_tables, "generated_ddl").and_return([])
           end
           it do
-            expect(subject_object).to receive(:set_current_tables).with(arg_tables, resume: true).and_call_original
+            expect(subject_object).to receive(:set_current_tables).with(arg_tables, resume: true, include_unregistered: false).and_call_original
             expect(subject_object).to receive(:wait_for_server_buffer).with(
                                                 {:timeout=>timeout_value, :tables=>arg_tables} )
             expect(subject_object).to receive(:cleanup_sync_server).with(

data/spec/flydata/helper/action/reset_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+require_relative '../helper_shared_context'
+require 'flydata/helper/action/reset'
+
+module Flydata
+  module Helper
+    module Action
+      describe Reset do
+        include_context 'helper context'
+
+        describe "#command" do
+          subject { described_class.new(config).command(opts) }
+
+          context 'without opts' do
+            let(:opts) { {} }
+            it { expect{subject}.to raise_error(/^Wrong tables parameter/) }
+          end
+
+          context 'with tables array' do
+            let(:opts) { {config_hash: {tables: %w(table_a table_b)}} }
+            it { is_expected.to eq("sync:reset -y --force --include-unregistered table_a table_b") }
+          end
+
+          context 'with tables string' do
+            let(:opts) { {config_hash: {tables: "table_a table_b"}} }
+            it { is_expected.to eq("sync:reset -y --force --include-unregistered table_a table_b") }
+          end
+        end
+      end
+    end
+  end
+end

data/spec/flydata/helper/action_ownership_channel_spec.rb

@@ -1,5 +1,7 @@
 require 'spec_helper'
 require 'flydata/helper/action_ownership_channel'
+require 'flydata/helper/action_ownership'
+require 'flydata/helper/server'
 require 'timecop'
 
 module Flydata
@@ -25,7 +27,7 @@ module Flydata
         context "when action is not valid" do
           let(:action_name) { :start_agent }
           it "should return false and leave a log" do
-            expect(channel).to receive(:log_warn)
+            expect(channel).to receive(:log_error)
             is_expected.to be(false)
           end
         end
@@ -40,6 +42,22 @@ module Flydata
             expect(channel.instance_variable_get(:@queue).size).to be(1)
           end
         end
+
+        context "when the queue already contains the action but different id" do
+          let(:prev_action_info) do
+            { id: 100, config: %Q|{"num_of_lines" : "100"}| }
+          end
+
+          before do
+            channel.request_action(action_name, prev_action_info)
+          end
+          it "should not alter the queue and return false" do
+            expect(channel.instance_variable_get(:@queue).size).to be(1)
+            is_expected.to be(true)
+            expect(channel.instance_variable_get(:@queue).size).to be(2)
+          end
+        end
+
       end
 
       describe '#take_action_ownership' do
@@ -75,6 +93,22 @@ module Flydata
            end
         end
 
+        context "when the action does not need to be taken exclusively  \
+                        but enqueued behind an exclusive actions in progress" do
+           before do
+            channel.request_action(:stop_agent)
+            channel.take_action_ownership("another-owner")
+            channel.request_action(:restart_agent)
+            channel.request_action(:send_logs, action_info)
+           end
+           it "should return a processable non-exclusive action" do
+             action = subject
+             expect(action[:action_ownership].action_name).to be(:send_logs)
+             expect(action[:action_ownership].owner).to be(new_owner)
+             expect(action[:action_info]).to eq(action_info)
+           end
+        end
+
         context "when the action can be taken" do
           before do
             channel.request_action(:send_logs, action_info)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-03 00:00:00.000000000 Z
+date: 2017-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -683,6 +683,7 @@ files:
 - lib/flydata/helper/action/check_remote_actions.rb
 - lib/flydata/helper/action/clear.rb
 - lib/flydata/helper/action/repair.rb
+- lib/flydata/helper/action/reset.rb
 - lib/flydata/helper/action/restart_agent.rb
 - lib/flydata/helper/action/resync.rb
 - lib/flydata/helper/action/send_logs.rb
@@ -842,6 +843,7 @@ files:
 - spec/flydata/helper/action/check_remote_actions_spec.rb
 - spec/flydata/helper/action/clear_spec.rb
 - spec/flydata/helper/action/repair_spec.rb
+- spec/flydata/helper/action/reset_spec.rb
 - spec/flydata/helper/action/restart_agent_spec.rb
 - spec/flydata/helper/action/resync_spec.rb
 - spec/flydata/helper/action/send_logs_spec.rb