flydata 0.5.21 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 516813af4ea0557232dfd85e16c84b3666fa6576
-  data.tar.gz: 8fa4a979e70fde765532a000952f1deab76a0973
+  metadata.gz: 6afa928a6d8e0ee05f33e9a14d81be9659164e8f
+  data.tar.gz: d6323ad8aa4f7d04cee5263b92e19b3208c8c651
 SHA512:
-  metadata.gz: b3eac9c57d1adced1b2657d12797fd076744f8659f032017c6826f92b6f6b1f7b7a6a40bdc476a9fd369402da50eeb92bc9c6022a712b9a6fa70c00648c55a1d
-  data.tar.gz: 25f4a8c298d41f3efdbf9f9560c2d215699af1ebfad086fd8d52ae51a00004a8893bdc288ef0ac7331f7d96030a542e1ec18cf941991907ca8ec8e1daee082b2
+  metadata.gz: c273cafd862d6eab962afe5594b7b7b3f72c5536a39b6e6825c98418ee3589e61005a42b4aa77503f3fcbd3d0011ddf550072550f60ba91c6f34c5e9d18bc52d
+  data.tar.gz: 951b50b74c206488fba05241722c09f7bb5f531b8e2097e2cf810bd6cdd863fb1f83471289287f347d1679e4d8ab4f96d27870f535031ea2e4375c3014b7d9ed

data/VERSION

@@ -1 +1 @@
-0.5.21
+0.6.0

data/lib/flydata.rb

@@ -31,6 +31,7 @@ module Flydata
   FLYDATA_CONF = File.join(FLYDATA_HOME, 'flydata.conf')
   FLYDATA_LOCK = File.join(FLYDATA_HOME, 'flydata_lock.pid')
   FLYDATA_HELPER_HOME = File.join(FLYDATA_HOME, 'helper')
+  FLYDATA_HELPER_CONF = File.join(FLYDATA_HELPER_HOME, "helper.conf")
 
   VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
   include Flydata::Heroku

data/lib/flydata/command/sync.rb

@@ -115,22 +115,42 @@ EOS
         Slop.new do
           on 'c', 'client', 'Resets client only.'
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
+          on 'a', 'all', 'Resets Sync for all tables'
+          on 'i', 'init', 'Resets unfinished initial sync'
         end
       end
 
       # Command: flydata sync:reset
       #  - Entry method
       def reset(*tables)
-        show_purpose_name
+
+        if tables.empty? && !opts[:all] && !opts[:init]
+          log_info_stdout <<EOS
+ERROR!  Argument or option must be specified.
+
+  To reset all tables:              flydata sync:reset --all
+  To reset specific tables:         flydata sync:reset table1 table2 ...
+  To reset unfinished initial sync: flydata sync:reset --init
+EOS
+          return
+        end
 
         # Set instance variables
-        sync_resumed = set_current_tables(tables, resume: true)
+        reset_init = opts[:init]
+        if opts[:all]
+          tables = []
+          reset_init = false
+        end
+        sync_resumed = set_current_tables(tables, resume: !opts[:all])
+
+        return if !sync_resumed && reset_init
 
         message = ''
         if sync_resumed && !tables.empty?
           log_info_stdout <<EOS
-Initial sync is in progress.  In this case, you can only reset the initial sync.  To reset specific table(s), please resume and complete the initial sync by running the 'flydata start' command first.
-If you'd like to reset the initial sync in progress, run the 'flydata reset' command with no arguments.
+ERROR!  You cannot reset tables because the previous initial sync has not been completed.  Reset the unfinished initial sync first with the following command:
+
+  flyudata sync:reset --init
 EOS
           return
         end
@@ -138,6 +158,7 @@ EOS
         # Flush client buffer
         msg_tables = @input_tables.empty? ? '' : " for these tables : #{@input_tables.join(" ")}"
         msg_sync_type = sync_resumed ? "the current initial sync" : "the current sync"
+        show_purpose_name
         return unless ask_yes_no("This resets #{msg_sync_type}#{msg_tables}.  Are you sure?")
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer # TODO We should rather delete buffer files
@@ -164,12 +185,13 @@ EOS
         sync_fm = create_sync_file_manager(de)
 
         # Delete local files
-        tables_for_deleting = tables.empty? ? sync_fm.tables_from_positions_dir : @input_tables
-        sync_fm.delete_table_control_files(*tables_for_deleting)
+        sync_fm.delete_dump_files
+        sync_fm.delete_table_control_files(*@input_tables)
 
         new_tables_after_reset = @unsynced_tables + @input_tables
-        delete_binlog =@input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
-        sync_fm.delete_non_table_control_files(delete_binlog)
+        if @input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
+          sync_fm.delete_master_position_files
+        end
         sync_fm.close
         log_info_stdout("Reset completed successfully.")
       end
@@ -568,11 +590,6 @@ EOS
           Flydata::Command::Conf.new.copy_templates
         end
 
-        # initialize stats
-        target_tables.each do |table_name|
-          sync_fm.save_record_count_stat(table_name, 0)
-        end
-
         generate_mysqldump(de, sync_fm, !opts.dump_stream?, options) do |mysqldump_io, binlog_pos, db_bytesize|
           parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, binlog_pos, db_bytesize)
         end
@@ -779,6 +796,11 @@ EOM
         end
         log_info_stdout("Sending data to FlyData Server...")
 
+        # initialize stats
+        target_tables.each do |table_name|
+          sync_fm.save_record_count_stat(table_name, 0)
+        end
+
         bench_start_time = Time.now
 
         output_thread = QueueableThread.new

data/lib/flydata/helper/action/check_remote_actions.rb

@@ -17,7 +17,7 @@ module Flydata
         end
 
         def execute(opts = {})
-          action_position = ActionPosition.new(config)
+          action_position = ActionPosition.new(config[:helper])
 
           # Get actions from flydata web server
           last_id = action_position.load

data/lib/flydata/helper/action/update_helper_config.rb

@@ -0,0 +1,24 @@
+require 'flydata/helper/base_action'
+
+module Flydata
+  module Helper
+    module Action
+      class UpdateHelperConfig <  BaseAction
+        def execute(opts = {})
+          conf_path = config[:config_path] || FLYDATA_HELPER_CONF
+          helper_conf = opts[:config]
+          helper_conf = "#{helper_conf}\n" if helper_conf && helper_conf.size > 0 && !helper_conf.end_with?("\n")
+          if helper_conf
+            log_info("updating Helper config:#{conf_path}")
+            File.open(conf_path, "w") {|f| f.write helper_conf }
+          else
+            log_info("removing Helper config:#{conf_path}")
+            File.unlink(conf_path) if File.exists?(conf_path)
+          end
+          # reload conf
+          Process.kill("USR2", Process.ppid)
+        end
+      end
+    end
+  end
+end

data/lib/flydata/helper/action_ownership.rb

@@ -46,7 +46,8 @@ module Flydata
           self.new(:check_abnormal_shutdown, false, Action::CheckAbnormalShutdown),
           self.new(:send_logs, false, Action::SendLogs),
           self.new(:stop_agent, true, Action::StopAgent),
-          self.new(:restart_agent, true, Action::RestartAgent)
+          self.new(:restart_agent, true, Action::RestartAgent),
+          self.new(:update_helper_config, false, Action::UpdateHelperConfig),
         ].inject({}) do |h, action|
           h[action.action_name] = action
           h

data/lib/flydata/helper/action_ownership_channel.rb

@@ -1,4 +1,5 @@
 require 'monitor'
+require 'flydata-core/logger'
 
 module Flydata
   module Helper
@@ -7,6 +8,7 @@ module Flydata
     # - Give a action with ownership to workers
     class ActionOwnershipChannel
       include MonitorMixin
+      include FlydataCore::Logger
 
       def initialize
         @action_ownership_map = ActionOwnership.action_ownership_map
@@ -19,7 +21,9 @@ module Flydata
       def request_action(action_name, action_info = {})
         action_ownership = @action_ownership_map[action_name]
         if action_ownership.nil?
-          raise "Received invalid action request:#{action_name}"
+          # unknown action.  probably a new action
+          log_warn "unsupported action `#{action_name}'.  Skip."
+          return false
         end
         self.synchronize do
           if @queue.has_key?(action_name)

data/lib/flydata/helper/base_action.rb

@@ -11,10 +11,6 @@ module Flydata
 
       attr_accessor :config
 
-      def get_service(service_class)
-        service_class.new(config)
-      end
-
       # Override
       #def execute(opts = {}, &block)
       #end

data/lib/flydata/helper/config_parser.rb

@@ -19,7 +19,6 @@ module Flydata
     #       check_interval: 10s
     # ============================================
     class Config < Hash
-      DEFAULT_INTERVAL = 30.0
       DEFAULT_SCHEDULED_ACTIONS = {
         check_remote_actions: {
           check_interval: '30s',
@@ -31,9 +30,6 @@ module Flydata
       DEFAULT_HELPER = {
         scheduled_actions: DEFAULT_SCHEDULED_ACTIONS,
       }
-      DEFAULT_CONFIG = {
-        helper: DEFAULT_HELPER
-      }
 
       def self.create(hash)
         self.new.merge(hash)
@@ -155,6 +151,9 @@ module Flydata
 
     class ConfigParser
       def self.parse_file(config_path = nil)
+        if config_path.nil? && File.exists?(FLYDATA_HELPER_CONF)
+          config_path = FLYDATA_HELPER_CONF
+        end
         config_path.nil? ? { helper: Config.default } : parse(File.open(config_path).read)
       end
 

data/lib/flydata/helper/worker.rb

@@ -55,7 +55,7 @@ module Flydata
         action_id = action_info && action_info[:id] ? "[#{action_info[:id]}]" : "" #scheduled actions (eg:- check_remote_action) don't have id
         set_log_context_item(:prefix, "[worker][#{action_ownership.action_name}]#{action_id}")
         begin
-          action_ownership.action_class.new(helper_conf).execute(action_info) do |new_action_name, new_action_info = nil|
+          action_ownership.action_class.new(config).execute(action_info) do |new_action_name, new_action_info = nil|
             # request action if the action requires
             server.action_ownership_channel.request_action(new_action_name, new_action_info)
           end

data/lib/flydata/sync_file_manager.rb

@@ -300,7 +300,6 @@ module Flydata
     end
 
     def delete_table_control_files(*tables)
-      return if (tables.nil? or tables.empty?)
       files_to_delete = [
           table_position_file_paths(*tables),
           table_binlog_pos_paths(*tables),
@@ -315,7 +314,6 @@ module Flydata
 
     def tables_from_positions_dir
       all_table_control_files = Dir.glob(File.join(table_positions_dir_path, '*.{pos,generated_ddl,init,rev}'))
-      return if all_table_control_files.nil?
       tables = Set.new
       all_table_control_files.each do |control_file|
         file_name = File.basename(control_file)
@@ -325,7 +323,7 @@ module Flydata
       tables.to_a
     end
 
-    def delete_non_table_control_files(delete_binlog= false)
+    def delete_dump_files
       files_to_delete = [
           dump_file_path,
           dump_pos_path,
@@ -333,16 +331,23 @@ module Flydata
           sync_info_file,
           stats_path
       ]
-      if delete_binlog
-        files_to_delete << binlog_path
-        files_to_delete << sent_binlog_path
-        files_to_delete << lock_pid_file
+      files_to_delete.flatten.each do |file_to_delete|
+        FileUtils.rm(file_to_delete) if File.exists?(file_to_delete)
       end
+    end
+
+    def delete_master_position_files
+      files_to_delete = [
+        binlog_path,
+        sent_binlog_path,
+        lock_pid_file,
+      ]
       files_to_delete.flatten.each do |file_to_delete|
         FileUtils.rm(file_to_delete) if File.exists?(file_to_delete)
       end
     end
 
+
     def delete_table_binlog_pos(table_name)
       file = File.join(table_positions_dir_path, table_name + ".binlog.pos")
       if File.exists?(file)

data/spec/flydata/helper/action/check_remote_actions_spec.rb

@@ -19,7 +19,7 @@ module Flydata
 
         let(:check_remote_action) do
           allow(ApiClient).to receive(:instance).and_return(api_client)
-          described_class.new(helper_conf)
+          described_class.new(config)
         end
 
         let(:empty_actions) do

data/spec/flydata/helper/action/restart_agent_spec.rb

@@ -8,7 +8,7 @@ module Flydata
         include_context 'helper context'
 
         describe "#execute" do
-          subject { described_class.new(helper_conf).execute }
+          subject { described_class.new(config).execute }
           it "executes the restart action" do
             expect(Open3).to receive(:capture3).with({}, "flydata restart --skip-helper")
             subject

data/spec/flydata/helper/action/send_logs_spec.rb

@@ -18,7 +18,7 @@ module Flydata
 
         let(:send_log_action) do
           allow(ApiClient).to receive(:instance).and_return(api_client)
-          described_class.new(helper_conf)
+          described_class.new(config)
         end
 
         let(:action_info) do

data/spec/flydata/helper/action/stop_agent_spec.rb

@@ -8,7 +8,7 @@ module Flydata
         include_context 'helper context'
 
         describe "#execute" do
-          subject { described_class.new(helper_conf).execute }
+          subject { described_class.new(config).execute }
           it "executes the stop action" do
             expect(Open3).to receive(:capture3).with({}, "flydata stop")
             subject

data/spec/flydata/helper/action_ownership_channel_spec.rb

@@ -24,8 +24,9 @@ module Flydata
 
         context "when action is not valid" do
           let(:action_name) { :start_agent }
-          it "should raise an error" do
-            expect {subject}.to raise_error('Received invalid action request:start_agent')
+          it "should return false and leave a log" do
+            expect(channel).to receive(:log_warn)
+            is_expected.to be(false)
           end
         end
 

data/spec/flydata/helper/action_ownership_spec.rb

@@ -36,12 +36,13 @@ module Flydata
       describe '.action_ownership_map' do
         subject { described_class.action_ownership_map }
         it do
-          expect(subject.size).to eq(5)
+          expect(subject.size).to eq(6)
           expect(subject.include?(:check_remote_actions)).to be(true)
           expect(subject.include?(:check_abnormal_shutdown)).to be(true)
           expect(subject.include?(:send_logs)).to be(true)
           expect(subject.include?(:stop_agent)).to be(true)
           expect(subject.include?(:restart_agent)).to be(true)
+          expect(subject.include?(:update_helper_config)).to be(true)
         end
       end
     end

data/spec/flydata/helper/config_parser_spec.rb

@@ -105,6 +105,63 @@ EOT
           end
         end
       end
+      describe ".parse_file" do
+        subject { described_class.parse_file(path) }
+
+        let(:expected_result) { double("expected_parse_result") }
+
+        shared_examples "opening a conf file and parse the contents" do
+          let(:conf_contents) { double("conf_contents") }
+          let(:conf_file) { double("conf_file") }
+
+          before do
+            expect(File).to receive(:open).with(expected_path).
+                            and_return(conf_file)
+            expect(conf_file).to receive(:read).and_return(conf_contents)
+            expect(described_class).to receive(:parse).with(conf_contents).
+                                       and_return(expected_result)
+          end
+        end
+
+        context "when path is given" do
+          let(:path) { "/path/to/my/helper.conf" }
+          let(:expected_path) { path }
+
+          before do
+            expect(File).to_not receive(:exists?)
+          end
+          include_examples "opening a conf file and parse the contents"
+          it { is_expected.to eq(expected_result) }
+        end
+        context "when no path is given" do
+          let(:path) { nil }
+          let(:expected_path) { File.join(Flydata::FLYDATA_HELPER_HOME, "helper.conf") }
+
+          context "and default conf file exists" do
+            before do
+              expect(File).to receive(:exists?).with(expected_path).
+                              and_return true
+            end
+            include_examples "opening a conf file and parse the contents"
+            it { is_expected.to eq(expected_result) }
+          end
+          context "and default conf file does not exist" do
+            before do
+              expect(File).to receive(:exists?).with(expected_path).
+                              and_return false
+            end
+            let(:expected_result) { {
+              helper: {
+                scheduled_actions: {
+                  check_remote_actions: { check_interval: "30s"},
+                  check_abnormal_shutdown: {:check_interval=>"60s"}
+                }
+              }
+            } }
+            it { is_expected.to eq(expected_result) }
+          end
+        end
+      end
     end
   end
 end    

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.21
+  version: 0.6.0
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-25 00:00:00.000000000 Z
+date: 2015-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -635,6 +635,7 @@ files:
 - lib/flydata/helper/action/restart_agent.rb
 - lib/flydata/helper/action/send_logs.rb
 - lib/flydata/helper/action/stop_agent.rb
+- lib/flydata/helper/action/update_helper_config.rb
 - lib/flydata/helper/action_ownership.rb
 - lib/flydata/helper/action_ownership_channel.rb
 - lib/flydata/helper/base_action.rb
@@ -743,7 +744,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14.1
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent