flydata 0.0.1.nc1

data/lib/flydata/command/base.rb

@@ -0,0 +1,35 @@
+module Flydata
+  module Command
+    class Base
+      def initialize
+        @api_client = ApiClient.instance
+      end
+      def flydata; @api_client end
+
+      # retrieve models on servers
+      def retrieve_data_entries
+        data_entries = flydata.get('/data_entries')
+        unless flydata.response.code == 200
+          raise "Failed to retrieve data_entries"
+        end
+        data_entries
+      end
+
+      # print console
+      def newline; puts end
+      def ask_yes_no(message)
+        loop do
+          ans = ask("#{message} (yes/no):  ")
+          if ans.size > 0
+            case ans[0].downcase
+            when 'y'; return true
+            when 'n'; return false
+            end
+          end
+          say(" ! Please answer y[es] or n[o]")
+          newline
+        end
+      end
+    end
+  end
+end

data/lib/flydata/command/crontab.rb

@@ -0,0 +1,9 @@
+module Flydata
+  module Command
+    class Crontab < Base
+      def run(args='')
+        Flydata::Cron.new.update args
+      end
+    end
+  end
+end

data/lib/flydata/command/login.rb

@@ -0,0 +1,39 @@
+module Flydata
+  module Command
+    class Login < Base
+      LOGIN_TRIAL_TIMES=3
+      def run
+        ret = login(LOGIN_TRIAL_TIMES)
+        raise "Login failed #{LOGIN_TRIAL_TIMES} times." unless ret
+        ret
+      end
+
+      private
+      def login(times=3)
+        1.upto(times) do |i|
+          begin
+            return true if login_once
+          rescue Exception => e
+            puts e
+          end
+        end
+        false
+      end
+      def login_once
+        # Ask login info
+        email = ask("FlyData Email:  ")
+        password = ask("FlyData password:  ") {|q| q.echo = false}
+        flydata.credentials = Flydata::Credentials.new(email, password)
+        # Auth request
+        flydata.post('/users/sign_in')
+        if flydata.response.code == 201 # 201: Created
+          say("Login succeeded!")
+          flydata.credentials.authenticate!
+          return true
+        end
+        say("Login failed!")
+        false
+      end
+    end
+  end
+end

data/lib/flydata/command/restart.rb

@@ -0,0 +1,10 @@
+module Flydata
+  module Command
+    class Restart < Base
+      def run
+        sender = Flydata::Command::Sender.new
+        sender.restart
+      end
+    end
+  end
+end

data/lib/flydata/command/routine.rb

@@ -0,0 +1,29 @@
+module Flydata
+  module Command
+    class Routine < Base
+      def run
+        print "#{Time.now} "
+        unless flydata.credentials.authenticated?
+          raise "Authentication error. Please login."
+        end
+        log_paths = retrieve_log_paths
+        log_paths and log_paths.size() > 0 and log_paths.each { |path|
+          if File.exist?(path) and File.writable?(path)
+            puts "Start - #{path}. "
+            Flydata::LogMonitor.new(path).setup.rotate
+          else
+            puts "Skip  - #{path}. System cannot access this path."
+          end
+        }
+      end
+
+      private
+      def retrieve_log_paths
+        data_entries = retrieve_data_entries
+        data_entries.map {|e|
+          e['log_path'] if e['log_deletion']
+        }.compact
+      end
+    end
+  end
+end

data/lib/flydata/command/sender.rb

@@ -0,0 +1,92 @@
+module Flydata
+  module Command
+    class Sender < Base
+      FLYDATA_HOME=Flydata::HOME_DIR
+      def start
+        if process_exist?
+          say("Process exist. Please stop process first.")
+          return
+        end
+
+        retry_count = 10
+        1.upto(retry_count) do |i|
+          break if server_ready?
+          say("Waiting server side active... (#{i}/#{retry_count})")
+          sleep 30
+        end
+
+        say('Starting sender process.')
+        Dir.chdir(FLYDATA_HOME){
+          system("fluentd -d #{FLYDATA_HOME}/flydata.pid -l #{FLYDATA_HOME}/flydata.log -c #{FLYDATA_HOME}/flydata.conf")
+        }
+
+        sleep 5
+
+        data_port = flydata.data_port.get
+        data_port_id = data_port['id']
+
+        retry_count = 10
+        ready = false
+        1.upto(retry_count) do |i|
+          ready ||= client_ready?
+          if ready and uploaded_successfully?(data_port_id)
+            say('Done.')
+            say("Go to your Dashboard! http://#{Flydata::FLYDATA_API_HOST}")
+            return true
+          end
+          say("Waiting client side active... (#{i}/#{retry_count})")
+          sleep 30
+        end
+
+        say('Something wrong..')
+        false
+      end
+      def stop
+        unless process_exist?
+          say("Process doesn't exist.")
+          return true
+        end
+
+        say('Stopping sender process.')
+        if system("kill `cat #{FLYDATA_HOME}/flydata.pid`")
+          say('Done.')
+          return true
+        else
+          say('Something wrong..')
+        end
+        false
+      end
+      def restart
+        if process_exist?
+          say('Restarting sender process.')
+          if system("kill -HUP `cat #{FLYDATA_HOME}/flydata.pid`")
+            say('Done.')
+            return true
+          else
+            say('Something wrong..')
+          end
+        else
+          say("Process doesn't exist.")
+          start
+        end
+      end
+
+      private
+      def server_ready?
+        data_port = flydata.data_port.get
+        data_port['server_status'] == 'active'
+      end
+      def client_ready?
+        process_exist?
+      end
+      def process_exist?
+        process_count = `ps aux|grep -v grep|grep "\\.flydata/flydata\\.conf"|grep fluentd|wc -l`.to_i
+        process_count > 0
+      end
+      def uploaded_successfully?(data_port_id)
+        res = flydata.get("/data_ports/#{data_port_id}/tail.json")
+        res and res['logs'] and res['logs'].size > 0
+      end
+    end
+  end
+end

data/lib/flydata/command/setlogdel.rb

@@ -0,0 +1,73 @@
+module Flydata
+  module Command
+    class Setlogdel < Base
+      def run
+        Flydata::Command::Login.new.run unless flydata.credentials.authenticated?
+        # Choose target data_entry
+        data_entry = choose_data_entry
+        return unless data_entry
+        # Confirm and update
+        if confirm_setting data_entry
+          set_log_deletion data_entry
+        end
+      end
+
+      private
+      def choose_data_entry
+        data_entries = retrieve_data_entries
+        if data_entries.size < 1
+          puts("There are no registered entries.")
+          puts("You need to create a data entry by flydata setup command.")
+          return nil
+        end
+        choices = data_entries.map {|e|
+          "#{e['name']}\t#{e['log_path']}\t -- Log deletion:#{boolToOnOff e['log_deletion']}"
+        }
+        choices << "Cancel"
+
+        choice = nil
+        say('Please select your log path for setting log deletion.')
+        newline
+        choose do |menu|
+          menu.index = :letter
+          menu.index_suffix = ") "
+          menu.prompt = "Select item to change log deletion setting: "
+          menu.choices(*choices) {|item| choice = item.split(' ')[0]}
+        end
+        newline
+        data_entries.select{|e| e['log_path'] == choice}.first
+      end
+      def confirm_setting(data_entry)
+        cur_flag = data_entry['log_deletion']
+        loop do
+          say "Target log path: #{data_entry['log_path']}"
+          say "Now changing log deletion setting: #{boolToOnOff cur_flag} -> #{boolToOnOff !cur_flag}"
+          ans = ask "Are you sure? (yes/no): "
+          if ans.size > 0
+            case ans[0].downcase
+            when 'y'; return true
+            when 'n'; return false
+            end
+          end
+          say " ! Please answer y[es] or n[o]"
+          newline
+        end
+      end
+      def boolToOnOff(bool)
+        bool ? 'ON' : 'OFF'
+      end
+      def set_log_deletion(data_entry)
+        param = {
+          :data_entry => {
+            'log_deletion' => !data_entry['log_deletion']
+          }
+        }
+        url = "/data_entries/#{data_entry['id']}"
+        ret = flydata.put(url, param)
+        raise 'Failed to update the log deletion setting.' unless ret['success']
+        Flydata::Command::Crontab.new.run if !data_entry['log_deletion']
+        puts "Update succeeded!"
+      end
+    end
+  end
+end

data/lib/flydata/command/setup.rb

@@ -0,0 +1,101 @@
+module Flydata
+  module Command
+    class Setup < Base
+      LOG_PATH_EXAMPLES=
+        %w(/var/log/httpd/access_log /var/log/apache2/access.log
+           /var/log/httpd-access.log /var/log/apache2/access_log
+           /var/log/messages /var/log/maillog /var/log/mysql/error.log
+           /home/*/deploy/shared/log/*.log)
+      OTHER = '-- None of above --'
+
+      # readline settings for asking log path
+      Readline.completion_append_character = "/"
+      Readline.completion_proc = Proc.new do |str|
+        Dir[str+'*'].grep( /^#{Regexp.escape(str)}/ )
+      end
+
+      def run
+        # login
+        Flydata::Command::Login.new.run unless flydata.credentials.authenticated?
+
+        # choose entries
+        unless shown = show_registered_entries and not more_entry?
+          begin
+            show_registered_entries unless shown
+            shown = false
+            path = choose_log_path_from_examples
+            case path
+            when OTHER; ask_log_path
+            else; create_log_entry(path, ask_log_deletion)
+            end
+            newline
+          end while more_entry?
+        end
+
+        # start client process
+        Flydata::Command::Sender.new.restart
+      end
+
+      private
+      def show_registered_entries
+        data_entries = retrieve_data_entries
+        @last_fetched_entries = data_entries
+        if data_entries and data_entries.size > 0
+          puts('Registered entries. ')
+          data_entries.each { |data_entry|
+            say(" - #{data_entry['name']}\t#{data_entry['log_path']}")
+          }
+          true
+        else
+          false
+        end
+      end
+      def choose_log_path_from_examples
+        candidates = (`ls #{LOG_PATH_EXAMPLES.join(' ')} 2>/dev/null`).split(/\s+/)
+        candidates = candidates.find_all{|path| File.readable?(path)}
+        if @last_fetched_entries
+           candidates = candidates - @last_fetched_entries.map{|v| v['log_path']}
+        end
+        return OTHER unless candidates.size > 0
+        candidates << OTHER
+        choice = nil
+        say('Please select your log path for sending FlyData')
+        newline
+        choose do |menu|
+          menu.index = :letter
+          menu.index_suffix = ") "
+          menu.prompt = "Your log path:  "
+          menu.choices(*candidates) {|item| choice = item}
+        end
+        newline
+        choice
+      end
+      def ask_log_path
+        path = nil
+        loop do
+          path = Readline.readline("Enter the absolute path of your log (return to cancel):  ")
+          return if path.empty?
+          break if FileTest.file?(path) and FileTest.readable?(path)
+          say(" ! #{path} is not a readable file!")
+          newline
+        end
+        create_log_entry(path, ask_log_deletion)
+      end
+      def ask_log_deletion
+        say("** Log deletion setting **")
+        say("Flydata has a log deletion feature that flydata will delete old log archives uploaded by flydata automatically.")
+        say("Flydata will delete logs whose last modified timestamp is 7 days ago.")
+        ask_yes_no("Set auto log deletion mode?")
+      end
+      def create_log_entry(path, log_deletion)
+        data_port = flydata.data_port.get
+        flydata.data_entry.create(data_port_id: data_port['id'], log_path: path, log_deletion: log_deletion)
+        Flydata::Command::Crontab.new.run if log_deletion
+        say("Process successfuly!") if flydata.response.code == 200
+      end
+      def more_entry?
+        ask_yes_no("Do you want to add more log path?")
+      end
+    end
+  end
+end

data/lib/flydata/command/start.rb

@@ -0,0 +1,10 @@
+module Flydata
+  module Command
+    class Start < Base
+      def run
+        sender = Flydata::Command::Sender.new
+        sender.start
+      end
+    end
+  end
+end

data/lib/flydata/command/stop.rb

@@ -0,0 +1,10 @@
+module Flydata
+  module Command
+    class Stop < Base
+      def run
+        sender = Flydata::Command::Sender.new
+        sender.stop
+      end
+    end
+  end
+end

data/lib/flydata/credentials.rb

@@ -0,0 +1,52 @@
+module Flydata
+  class Credentials
+    include Helpers
+    attr_reader :user, :password
+    def initialize(user=nil, password=nil)
+      read_credentials
+      if user && password
+        @user = user
+        @password = password
+      elsif !(@authenticated)
+        @user =  ENV['FLYDATA_LOGIN']
+        @password = ENV['FLYDATA_PASSWORD']
+      end
+    end
+    def authenticate!
+      @authenticated = true
+      write_credentials
+    end
+    def authenticated?
+      @authenticated
+    end
+    def write_credentials
+      dir = File.dirname(credentials_file)
+      FileUtils.mkdir_p(dir)
+      File.delete(credentials_file) if FileTest.exists?(credentials_file)
+      File.open(credentials_file, 'w', 0400) do |out|
+        out.puts @user
+        out.puts encode(@password)
+      end
+      FileUtils.chmod(0700, dir)
+    end
+    def read_credentials
+      if FileTest.exist?(credentials_file)
+        File.open(credentials_file, 'r') do |f|
+          @user = f.gets.chomp
+          @password = decode(f.gets.chomp)
+          @authenticated = true if @user and @password
+        end
+      end
+    end
+    private
+    def credentials_file
+      File.join(home_directory, '.flydata', 'credentials')
+    end
+    def encode(str)
+      str.unpack('H*') # want to change more complecated? still useful for shoulder hacking.
+    end
+    def decode(str)
+      [str].pack('H*')
+    end
+  end
+end