flydata 0.3.24 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c6d22e3fb1b51c996297d48f7bd886c250d5d11
-  data.tar.gz: 23ac7e6b2b97a76bf27d538370fb9c73e018e842
+  metadata.gz: 0c46afc6ca25cb5f8969e22aac032415a315eb0e
+  data.tar.gz: a95cdb85e846ae1b56f4ad3d2802fc163ec9df99
 SHA512:
-  metadata.gz: f5455a1b18dcbccb9edae44ec15ac2332710410107687251cba290a99e671b83567c84dde29b7428696750b1732ebae7410093921170ad04ab1f4c938c1da67c
-  data.tar.gz: d72631b42f67a3ae378ed215663149414cf1cd80d02eac97fe97c0517b7813a5341ee93a05daf02774f094daf90c19381d3b1709a428729a99fb2101b6f80468
+  metadata.gz: 56cb5f8d6e58f71598bfc370ea31a24784c1132b5fcadbae263cb29c85a1cd950ff0c2169698722f73058c9d44b16424a57d441c4b98f1ee914ddd57062d91df
+  data.tar.gz: c0b6985cafa887eb2a8aac272c57431f1bb72343b937be7ebb55ef5dfdfec989778acf160119bd0205e4978289fd4134c04890a468bc655022dc99d17f65107f

data/Gemfile

@@ -14,6 +14,7 @@ gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
 gem "io-console", '~> 0.4.2', '>= 0.4.2'
 gem "kodama", '~> 0.1.2', '>= 0.1.5'
+gem "serverengine", '~> 1.5'
 
 group :development do
   gem "jeweler", '~> 1.8', '>= 1.8.8'

data/Gemfile.lock

@@ -107,6 +107,8 @@ GEM
     rspec-support (3.0.3)
     ruby-binlog (1.0.4)
     ruby-prof (0.15.1)
+    serverengine (1.5.10)
+      sigdump (~> 0.2.2)
     sigdump (0.2.2)
     slop (3.6.0)
     sqlite3 (1.3.9)
@@ -140,6 +142,7 @@ DEPENDENCIES
   rest-client (~> 1.6, >= 1.6.7)
   rspec (~> 3.0)
   ruby-prof (~> 0.15, >= 0.15.1)
+  serverengine (~> 1.5)
   slop (~> 3.4, >= 3.4.6)
   sqlite3 (~> 1.3, >= 1.3.9)
   sys-filesystem (~> 1.1, >= 1.1.3)

data/VERSION

@@ -1 +1 @@
-0.3.24
+0.4.0

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.24 ruby lib
+# stub: flydata 0.4.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.24"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-06-01"
+  s.date = "2015-06-17"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -81,6 +81,7 @@ Gem::Specification.new do |s|
     "flydata.gemspec",
     "lib/fly_data_model.rb",
     "lib/flydata.rb",
+    "lib/flydata/api/agent.rb",
     "lib/flydata/api/base.rb",
     "lib/flydata/api/data_entry.rb",
     "lib/flydata/api/data_port.rb",
@@ -91,8 +92,10 @@ Gem::Specification.new do |s|
     "lib/flydata/command/conf.rb",
     "lib/flydata/command/crontab.rb",
     "lib/flydata/command/encrypt.rb",
+    "lib/flydata/command/helper.rb",
     "lib/flydata/command/kill_all.rb",
     "lib/flydata/command/login.rb",
+    "lib/flydata/command/mysql.rb",
     "lib/flydata/command/restart.rb",
     "lib/flydata/command/routine.rb",
     "lib/flydata/command/sender.rb",
@@ -124,6 +127,18 @@ Gem::Specification.new do |s|
     "lib/flydata/fluent-plugins/out_forward_ssl.rb",
     "lib/flydata/fluent-plugins/preference.rb",
     "lib/flydata/flydata_crontab.sh",
+    "lib/flydata/helper/action/agent_action.rb",
+    "lib/flydata/helper/action/check_remote_actions.rb",
+    "lib/flydata/helper/action/restart_agent.rb",
+    "lib/flydata/helper/action/send_logs.rb",
+    "lib/flydata/helper/action/stop_agent.rb",
+    "lib/flydata/helper/action_ownership.rb",
+    "lib/flydata/helper/action_ownership_channel.rb",
+    "lib/flydata/helper/base_action.rb",
+    "lib/flydata/helper/config_parser.rb",
+    "lib/flydata/helper/scheduler.rb",
+    "lib/flydata/helper/server.rb",
+    "lib/flydata/helper/worker.rb",
     "lib/flydata/helpers.rb",
     "lib/flydata/heroku.rb",
     "lib/flydata/heroku/configuration_methods.rb",
@@ -140,6 +155,8 @@ Gem::Specification.new do |s|
     "lib/flydata/proxy.rb",
     "lib/flydata/sync_file_manager.rb",
     "lib/flydata/util/encryptor.rb",
+    "lib/flydata/util/file_util.rb",
+    "lib/flydata/util/shell.rb",
     "spec/fluent_plugins_spec_helper.rb",
     "spec/fly_data_model_spec.rb",
     "spec/flydata/api/data_entry_spec.rb",
@@ -149,6 +166,7 @@ Gem::Specification.new do |s|
     "spec/flydata/command/conf_spec.rb",
     "spec/flydata/command/crontab_spec.rb",
     "spec/flydata/command/encrypt_spec.rb",
+    "spec/flydata/command/helper_spec.rb",
     "spec/flydata/command/kill_all_spec.rb",
     "spec/flydata/command/login_spec.rb",
     "spec/flydata/command/restart_spec.rb",
@@ -171,6 +189,16 @@ Gem::Specification.new do |s|
     "spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb",
     "spec/flydata/fluent-plugins/mysql/table_meta_spec.rb",
     "spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb",
+    "spec/flydata/helper/action/check_remote_actions_spec.rb",
+    "spec/flydata/helper/action/restart_agent_spec.rb",
+    "spec/flydata/helper/action/send_logs_spec.rb",
+    "spec/flydata/helper/action/stop_agent_spec.rb",
+    "spec/flydata/helper/action_ownership_channel_spec.rb",
+    "spec/flydata/helper/action_ownership_spec.rb",
+    "spec/flydata/helper/config_parser_spec.rb",
+    "spec/flydata/helper/helper_shared_context.rb",
+    "spec/flydata/helper/scheduler_spec.rb",
+    "spec/flydata/helper/worker_spec.rb",
     "spec/flydata/heroku_spec.rb",
     "spec/flydata/mysql/binlog_position_spec.rb",
     "spec/flydata/mysql/mysql_util_spec.rb",
@@ -180,6 +208,8 @@ Gem::Specification.new do |s|
     "spec/flydata/parser/mysql/dump_parser_spec.rb",
     "spec/flydata/sync_file_manager_spec.rb",
     "spec/flydata/util/encryptor_spec.rb",
+    "spec/flydata/util/file_util_spec.rb",
+    "spec/flydata/util/shell_spec.rb",
     "spec/flydata_spec.rb",
     "spec/spec_helper.rb",
     "tmpl/redshift_mysql_data_entry.conf.tmpl"
@@ -206,6 +236,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_runtime_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
       s.add_runtime_dependency(%q<kodama>, [">= 0.1.5", "~> 0.1.2"])
+      s.add_runtime_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_development_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_development_dependency(%q<rspec>, ["~> 3.0"])
       s.add_development_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -229,6 +260,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
       s.add_dependency(%q<kodama>, [">= 0.1.5", "~> 0.1.2"])
+      s.add_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_dependency(%q<rspec>, ["~> 3.0"])
       s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])
@@ -253,6 +285,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
     s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
     s.add_dependency(%q<kodama>, [">= 0.1.5", "~> 0.1.2"])
+    s.add_dependency(%q<serverengine>, ["~> 1.5"])
     s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
     s.add_dependency(%q<rspec>, ["~> 3.0"])
     s.add_dependency(%q<timecop>, [">= 0.7.1", "~> 0.7"])

data/lib/flydata.rb

@@ -29,7 +29,15 @@ module Flydata
   FLYDATA_SERVERINFO = File.join(FLYDATA_GEM_BIN, 'serverinfo')
   FLYDATA_LOG = File.join(FLYDATA_HOME, 'flydata.log')
   FLYDATA_CONF = File.join(FLYDATA_HOME, 'flydata.conf')
+  FLYDATA_HELPER_HOME = File.join(FLYDATA_HOME, 'helper')
 
   VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
   include Flydata::Heroku
 end
+
+# Require all helper files -
+# TODO : Check if this should be moved else where or can be more generic (not just helper files)
+lib_dir = File.dirname(File.absolute_path(__FILE__))
+FileUtils.cd(lib_dir) do
+  Dir["flydata/helper/**/*.rb"].each { |file| require file }
+end

data/lib/flydata/api/agent.rb

@@ -0,0 +1,21 @@
+require 'flydata/api/base'
+
+module Flydata
+  module Api
+    class Agent < Base
+      def initialize(api_client)
+        @model_name = 'agent'
+        @url_path = "/#{@model_name}"
+        super
+      end
+
+      def actions(last_id)
+        @client.get("#{@url_path}/actions?last_id=#{last_id}")
+      end
+
+      def send_logs(action_id, logs)
+        @client.post("#{@url_path}/logs", nil, {action_id: action_id, logs: logs})
+      end
+    end
+  end
+end

data/lib/flydata/command/helper.rb

@@ -0,0 +1,154 @@
+require 'flydata/command/base'
+require 'flydata/helper/server'
+require 'flydata/helper/config_parser'
+require 'flydata/util/shell'
+
+module Flydata
+  module Command
+    class Helper < Base
+
+      DEFAULT_OPTIONS = {
+        #daemon
+        daemonize: false,
+        supervisor: true,
+        workers: 2,
+
+        #logger
+        log: nil,
+        log_level: 'info',
+        log_rotate_age: 10,
+        log_rotate_size: 10*1024*1024
+      }
+
+      def self.slop_start
+        Slop.new do
+          on 'c=', 'config=','config file path'
+          on 'n', 'no-daemon', 'Start Helper as a regular program'
+          on 'l=', 'log=', 'log file path'
+          on 'e=', 'level=', 'log level'
+          on 'r=', 'rotate=', 'Number of times logs are rotated before being removed'
+          on 's=', 'size=', 'Size (in bytes) after which log is rotated'
+          #For now, size needs to be specified in bytes (dont think it will be used often)
+        end
+      end
+
+      def initialize(options = Slop.new)
+        super
+        @helper_config = Flydata::Helper::ConfigParser.parse_file(opts[:config])[:helper]
+        create_helper_dirs
+      end
+
+      def start
+        if running?
+          log_info_stdout("Helper is already running.")
+        else
+          raw_start
+        end
+      end
+
+      def stop
+        if running?
+          log_info_stdout("Stopping Helper.")
+          run_command(stop_cmd)
+        else
+          log_info_stdout("Helper is not running.")
+        end
+      end
+
+      def status
+        if running?
+          log_info_stdout("Helper is running.")
+        else
+          log_info_stdout("Helper is not running.")
+        end
+      end
+
+      def restart
+        if running?
+          log_info_stdout("Restarting Helper.")
+          run_command(kill_hup_cmd)
+        else
+          raw_start
+        end
+      end
+
+      private
+
+      def create_config_with_args
+        Hash.new.tap do |c|
+          c[:config_path] = opts[:config]
+          c[:log] = opts[:log] ? opts[:log] : @helper_config.helper_log_path
+          unless opts.no_daemon?
+            c[:pid_path] = @helper_config.helper_pid_path
+            c[:daemonize] = true
+          end
+          c[:log_level] = opts[:level] if opts[:level]
+          c[:log_rotate_age] = opts[:rotate].to_i if opts[:rotate]
+          c[:log_rotate_size] = opts[:size].to_i if opts[:size]
+        end
+      end
+
+      def run_command(cmd)
+        o, e, s = Util::Shell.run_cmd(cmd)
+        log_error_stderr(e) if not e.to_s.empty?
+        success = (s.to_i == 0)
+        if success
+          log_info_stdout("Done")
+        else
+          log_error_stderr("Failed")
+        end
+        success
+      end
+
+      def running?
+        `#{running_cmd}`.to_i > 0
+      end
+
+      def stop_cmd
+        <<EOS
+if #{running_cmd}; then
+  #{kill_cmd}
+  i=0
+  while #{running_cmd}; do
+    sleep 1.0
+    if [ $i -gt 10 ]; then  # 10sec
+      echo "Timeout warning: It failed to stop within 10 seconds. The process will be killed with SIGKILL."
+      kill -9 -$(ps --pid `#{@helper_config.helper_pid_path}` --no-heading -o "%r" | tr -d ' ')
+    else
+      i=`expr $i + 1`
+    fi
+  done
+  rm #{@helper_config.helper_pid_path}
+fi
+EOS
+      end
+
+      def running_cmd
+        "[ -f #{@helper_config.helper_pid_path} ] && pgrep -P \`cat #{@helper_config.helper_pid_path}\`"
+      end
+
+      def kill_hup_cmd
+        kill_cmd("-HUP")
+      end
+
+      def kill_cmd(option = nil)
+        "kill #{option} \`cat #{@helper_config.helper_pid_path}\`"
+      end
+
+      def raw_start
+        log_info_stdout("Starting Helper.")
+        Flydata::Helper::Server.run(
+            DEFAULT_OPTIONS.merge(create_config_with_args))
+        log_info_stdout("Done")
+      end
+
+      def create_helper_dirs
+        #Create helper dirs if they don't exist
+        FileUtils.mkdir_p([@helper_config.helper_home,
+                           @helper_config.helper_pid_dir,
+                           @helper_config.helper_position_dir
+                         ])
+      end
+    end
+  end
+end

data/lib/flydata/command/mysql.rb

@@ -0,0 +1,37 @@
+require 'open3'
+require 'flydata/command/sync'
+
+module Flydata
+  module Command
+
+    class Mysql < Sync
+      def run
+        de = retrieve_sync_data_entry
+        de['mysql_data_entry_preference'].delete('tables')
+        cmd = Flydata::Mysql::MysqlUtil::generate_mysql_cmd(de['mysql_data_entry_preference'])
+        if $stdin.tty?
+          # interactive shell
+          system cmd
+        else
+          # execute queries given to $stdin
+          Open3.popen2e(cmd) do |i, o, wt|
+            $stdin.each_line do |line|
+              i.print line
+            end
+            i.close
+            while line = o.gets
+              print line
+            end
+          end
+        end
+      end
+
+      def flush; end
+      def reset; end
+      def skip; end
+      def generate_table_ddl; end
+      def fix_binlogpos; end
+    end
+
+  end
+end

data/lib/flydata/command/restart.rb

@@ -1,12 +1,23 @@
 require 'flydata/command/base'
 require 'flydata/command/sender'
+require 'flydata/command/helper'
 
 module Flydata
   module Command
     class Restart < Base
+      def self.slop
+        Slop.new do
+          on 'skip-helper', 'Skip restarting the Helper'
+        end
+      end
+
       def run
         sender = Flydata::Command::Sender.new
         sender.restart
+        unless opts.skip_helper?
+          helper = Flydata::Command::Helper.new
+          helper.restart
+        end
       end
     end
   end

data/lib/flydata/command/start.rb

@@ -1,15 +1,25 @@
 require 'flydata/command/base'
 require 'flydata/command/sender'
+require 'flydata/command/helper'
 
 module Flydata
   module Command
     class Start < Base
       def self.slop
-        Flydata::Command::Sender.slop_start # Needs options for Sender#start
+        sender_opts = Flydata::Command::Sender.slop_start # Needs options for Sender#start
+        sender_opts.on 'skip-helper', 'Skip starting the Helper'
+        sender_opts
       end
-      def run
+      def run(*args)
         sender = Flydata::Command::Sender.new(opts)
         sender.start
+        unless opts.skip_helper?
+          helper_opts = Flydata::Command::Helper.slop_start
+          helper_opts.parse!(args)
+          helper = Flydata::Command::Helper.new(helper_opts)
+          helper.stop # kill existing helper process (if any) and start
+          helper.start
+        end
       end
     end
   end