prometheus-splash 0.7.0 → 0.8.4

data/lib/splash/processes.rb

@@ -6,6 +6,85 @@ module Splash
   # Processes namespace
   module Processes
 
+
+    class ProcessNotifier
+
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_status = Prometheus::Client::Gauge.new(:process_status, docstring: 'SPLASH metric process status', labels: [:process ])
+      @@metric_cpu_percent = Prometheus::Client::Gauge.new(:process_cpu_percent, docstring: 'SPLASH metric process CPU usage in percent', labels: [:process ])
+      @@metric_mem_percent = Prometheus::Client::Gauge.new(:process_mem_percent, docstring: 'SPLASH metric process MEM usage in percent', labels: [:process ])
+      @@registry.register(@@metric_status)
+      @@registry.register(@@metric_cpu_percent)
+      @@registry.register(@@metric_mem_percent)
+
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = options[:name]
+        @status = options[:status]
+        @cpu_percent = options[:cpu_percent]
+        @mem_percent = options[:mem_percent]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+        unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_mem_percent.set(@mem_percent, labels: { process: @name })
+        @@metric_cpu_percent.set(@cpu_percent, labels: { process: @name })
+        @@metric_status.set(@status, labels: { process: @name })
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new("Splash", hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+    class ProcessRecords
+      include Splash::Backends
+      include Splash::Constants
+
+      def initialize(name)
+        @name = name
+        @backend = get_backend :process_trace
+      end
+
+      def clear
+        @backend.del({:key => @name}) if  @backend.exist?({key: @name})
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+    end
+
     # Processes scanner and notifier
     class ProcessScanner
       include Splash::Constants
@@ -17,14 +96,6 @@ module Splash
       def initialize
         @processes_target = Marshal.load(Marshal.dump(get_config.processes))
         @config = get_config
-        @registry = Prometheus::Client::Registry::new
-        @metric_status = Prometheus::Client::Gauge.new(:process_status, docstring: 'SPLASH metric process status', labels: [:process ])
-        @metric_cpu_percent = Prometheus::Client::Gauge.new(:process_cpu_percent, docstring: 'SPLASH metric process CPU usage in percent', labels: [:process ])
-        @metric_mem_percent = Prometheus::Client::Gauge.new(:process_mem_percent, docstring: 'SPLASH metric process MEM usage in percent', labels: [:process ])
-        @registry.register(@metric_status)
-        @registry.register(@metric_cpu_percent)
-        @registry.register(@metric_mem_percent)
-
       end
 
 
@@ -64,17 +135,21 @@ module Splash
         session = (options[:session]) ? options[:session] : log.get_session
         log.info "Sending metrics to Prometheus Pushgateway", session
         @processes_target.each do |item|
+          processrec = ProcessRecords::new item[:process]
           missing = (item[:status] == :missing)? 1 : 0
-          log.item "Sending metrics for #{item[:process]}", session
           val = (item[:status] == :running )? 1 : 0
-          @metric_status.set(val, labels: { process: item[:process] })
-          @metric_cpu_percent.set(item[:cpu], labels: { process: item[:process] })
-          @metric_mem_percent.set(item[:mem], labels: { process: item[:process] })
+          processrec.purge(item[:retention])
+          processrec.add_record :status => item[:status],
+                           :cpu_percent => item[:cpu],
+                           :mem_percent => item[:mem] ,
+                           :process => item[:process]
+          processmonitor = ProcessNotifier::new({name: item[:process], status: val , cpu_percent: item[:cpu], mem_percent: item[:mem]})
+          if processmonitor.notify then
+            log.ok "Sending metrics for process #{item[:process]} to Prometheus Pushgateway", session
+          else
+            log.ko "Failed to send metrics for process #{item[:process]} to Prometheus Pushgateway", session
+          end
         end
-        hostname = Socket.gethostname
-        url = @config.prometheus_pushgateway_url
-        Prometheus::Client::Push.new('Splash',hostname, url).add(@registry)
-        log.ok "Sending to Prometheus PushGateway done.", session
         return {:case => :quiet_exit }
       end
 

data/lib/splash/transfers.rb

@@ -0,0 +1,229 @@
+# coding: utf-8
+
+# base Splash Module
+module Splash
+
+  # Transfers module
+  module Transfers
+
+    include Splash::Config
+    include Splash::Loggers
+    include Splash::Helpers
+
+
+
+    class TxNotifier
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_nbfiles = Prometheus::Client::Gauge.new(:txnbfiles, docstring: 'SPLASH metric transfer number of files')
+      @@metric_nbfiles_failed = Prometheus::Client::Gauge.new(:txnbfilesfailed, docstring: 'SPLASH metric transfer number of failed files')
+      @@metric_time = Prometheus::Client::Gauge.new(:txtime, docstring: 'SPLASH metric transfer execution time')
+      @@registry.register(@@metric_nbfiles)
+      @@registry.register(@@metric_nbfiles_failed)
+      @@registry.register(@@metric_time)
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = "tx_#{options[:name].to_s}"
+        @nbfiles = options[:nbfiles]
+        @nbfiles_failed = options[:nbfiles_failed]
+        @time = options[:time]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+	unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_nbfiles.set(@nbfiles)
+        @@metric_nbfiles_failed.set(@nbfiles_failed)
+        @@metric_time.set(@time)
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new(@name, hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+
+    class TxRecords
+      include Splash::Backends
+      include Splash::Constants
+
+      def initialize(name)
+        @name = name
+        @backend = get_backend :transfers_trace
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+      def check_prepared
+        return :never_run_prepare unless @backend.exist?({key: @name})
+        return :never_prepare unless YAML::load(@backend.get({key: @name})).select {|item|
+          record =item.keys.first
+          value=item[record]
+          value[:status] == :prepared
+        }.count > 0
+        return :prepared
+      end
+
+    end
+
+    def run_txs(options = {})
+      log = get_logger
+      log.info 'Running Transfers'
+      count=0
+      get_config.transfers.each do |record|
+        txrec =  TxRecords::new record[:name]
+        txrec.purge(record[:retention])
+        log.item "Execute : #{record[:name]},  #{record[:desc]}"
+        case txrec.check_prepared
+        when :prepared
+          if record[:type] == :push then
+            unless push record
+              count += 1
+            end
+          elsif record[:type] == :pull then
+            unless pull record
+              count += 1
+            end
+          else
+            log.ko "Transfer type unkown"
+            count += 1
+          end
+        when :never_prepare
+          log.ko "#{record[:name]} : Never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        when :never_run_prepare
+          log.ko "#{record[:name]} : Never Executed and never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        end
+      end
+      return {:case => :error_exit, :more => "#{count} Transfer(s) failed"} if count > 0
+      return {:case => :quiet_exit }
+    end
+
+
+
+    def prepare_tx(name)
+      log = get_logger
+      record = get_config.transfers.select { |item| item[:name] == name.to_sym }.first
+      home  = Etc.getpwnam(record[:local][:user]).dir
+      identity = ::File::readlines("#{home}/.ssh/id_rsa.pub").first.chomp
+      folder  = {:mode => "755",
+                 :owner => record[:local][:user] ,
+                 :group => Etc.getgrgid(Etc.getpwnam(record[:local][:user]).gid).name,
+                 :name => record[:local][:path],
+                 :path => record[:local][:path]}
+      log.info "Ensure local folder : #{record[:local][:path]}"
+      make_folder(folder) unless verify_folder(folder).empty?
+      begin
+        log.info "Ensure RSA Key sharing for local user : #{record[:local][:user]} to remote user : #{record[:remote][:user]}@#{record[:remote][:host]}"
+        ssh = Net::SSH.start(record[:remote][:host],record[:remote][:user])
+        output = ssh.exec!(%[
+          /bin/bash -cl '
+          umask 077;
+          mkdir #{record[:remote][:path]};
+          test -d ~/.ssh || mkdir ~/.ssh;
+          if [ ! -f ~/.ssh/authorized_keys -o `grep "#{identity}" ~/.ssh/authorized_keys 2> /dev/null | wc -l` -eq 0 ]; then              echo "#{identity}" >> ~/.ssh/authorized_keys
+          fi'])
+        log.info "Prepare remote folder : #{record[:remote][:path]}"
+        log.info "Prepare data file for transfer : #{record[:name]}"
+        txrec = TxRecords::new record[:name]
+        txrec.add_record :status => :prepared
+        log.ok "Transfer : #{record[:name]} prepared successfully"
+        return {:case => :quiet_exit }
+      rescue Interrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      rescue TTY::Reader::InputInterrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      end
+    end
+
+
+
+    def save_data
+
+    end
+
+
+
+    def push(record)
+      config = get_config
+      log = get_logger
+      txrec = TxRecords::new record[:name]
+      start = Time.now
+      res = true
+      count = 0
+      done =[]
+      start_date = DateTime.now.to_s
+      list = Dir.glob("#{record[:local][:path]}/#{record[:pattern]}")
+      count = list.count
+      log.arrow "Transfering #{count} file(s)"
+
+      begin
+        scp = Net::SCP.start(record[:remote][:host],record[:remote][:user])
+        list.each do|f|
+          log.arrow "Copy file : #{f} to #{record[:remote][:user]}@#{record[:remote][:host]}:#{record[:remote][:path]}"
+          scp.upload! f, record[:remote][:path]
+          done.push f
+          if record[:backup] then
+            log.arrow "File #{f} backuped"
+            FileUtils::mv(f, "#{f}.#{Time.now.getutc.to_i}")
+          else
+            FileUtils::unlink(f)
+          end
+        end
+
+      rescue
+        res = false
+      end
+
+      end_date = DateTime.now.to_s
+      time = Time.now - start
+      status = (res)? :success : :failure
+      txrec.add_record :status => status,
+                       :end_date => end_date,
+                       :time => time,
+                       :count => count,
+                       :wanted => list,
+                       :done => done
+      count_failed = list.count - done.count
+      txmonitor = TxNotifier::new({name: record[:name], nbfiles: count,nbfiles_failed: count_failed, time: time})
+      if txmonitor.notify then
+        log.ok "Sending metrics to Prometheus Pushgateway"
+      else
+        log.ko "Failed to send metrics to Prometheus Pushgateway"
+      end
+      return res
+    end
+  end
+end

data/lib/splash/webadmin.rb

@@ -21,7 +21,7 @@ module Splash
       # @param [Hash] options
       # @option options [Symbol] :quiet activate quiet mode for log (limit to :fatal)
       # @return [Hash] Exiter Case (:quiet_exit, :already_exist, :unknown_error or other)
-      def startdaemon(options = {})
+      def startweb(options = {})
         require 'splash/webadmin/main'
         config = get_config
         log = get_logger
@@ -64,7 +64,7 @@ module Splash
       # @param [Hash] options
       # @option options [Symbol] :quiet activate quiet mode for log (limit to :fatal)
       # @return [Hash] Exiter Case (:quiet_exit, :not_found, other)
-      def stopdaemon(options = {})
+      def stopweb(options = {})
           config = get_config
           log = get_logger
           log.level = :fatal if options[:quiet]
@@ -86,7 +86,7 @@ module Splash
       # Status of the Splash WebAdmin, display status
       # @param [Hash] options ignored
       # @return [Hash] Exiter Case (:status_ko, :status_ok)
-      def statusdaemon(options = {})
+      def statusweb(options = {})
         log = get_logger
         config = get_config
         pid = realpid = ''

data/lib/splash/webadmin/api/routes/commands.rb

@@ -5,7 +5,7 @@
 WebAdminApp.get '/api/commands/list.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : commands, verb : GET, route : list, format : #{format}"
+  log.call "API : commands, verb : GET, route : list, format : #{format}"
   obj =  splash_return case: :quiet_exit, :more => "Commands list"
   obj[:data] = get_config.commands
   content_type format
@@ -15,7 +15,7 @@ WebAdminApp.get '/api/commands/list.?:format?' do
 WebAdminApp.get '/api/commands/show/:name.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : commands, verb : GET, route : show, item : #{params[:name]} , format : #{format}"
+  log.call "API : commands, verb : GET, route : show, item : #{params[:name]} , format : #{format}"
   commands_recordset = get_config.commands[params[:name].to_sym]
   unless commands_recordset.nil? then
     obj = splash_return case: :quiet_exit

data/lib/splash/webadmin/api/routes/config.rb

@@ -1,10 +1,103 @@
 WebAdminApp.get '/api/config/full.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : config, verb : GET, route : show, format : #{format}"
+  log.call "API : config, verb : GET, route : full, format : #{format}"
   config = get_config.full
-  obj =  splash_return case: :quiet_exit, :more => "logses list"
+  obj =  splash_return case: :quiet_exit, :more => "Show internal Splash Config"
   obj[:data] = config
   content_type format
   format_response(obj, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
 end
+
+WebAdminApp.get '/api/config/fromfile.?:format?' do
+  log = get_logger
+  fromfile = {}
+  format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
+  log.call "API : config, verb : GET, route : fromfile, format : #{format}"
+  config = get_config.config_from_file
+  fromfile =  splash_return case: :quiet_exit, :more => "Show config from file"
+  fromfile[:data] = config
+  content_type format
+  format_response(fromfile, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+end
+
+
+WebAdminApp.post '/api/config/addlog.?:format?' do
+  log = get_logger
+  addlog = {}
+  format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
+  log.call "API : config, verb : POST, route : addlog, format : #{format}"
+  res = get_config.add_record :record => YAML::load(request.body.read), :key => :label, :type => :logs, :clean => true
+  case res[:status]
+  when :success
+    addlog = splash_return case: :quiet_exit, :more => "add log done"
+  when :already_exist
+    addlog = splash_return case: :already_exist, :more => "add log twice nto allowed"
+  when :failure
+    addlog = splash_return case: :configuration_error, :more => "add log failed"
+    addlog[:data] = res
+  end
+  content_type format
+  format_response(addlog, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+end
+
+WebAdminApp.post '/api/config/addprocess.?:format?' do
+  log = get_logger
+  addprocess = {}
+  format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
+  log.call "API : config, verb : POST, route : addprocess, format : #{format}"
+  res = get_config.add_record :record => YAML::load(request.body.read), :type => :processes, :key => :process, :clean => true
+  case res[:status]
+  when :success
+    addprocess = splash_return case: :quiet_exit, :more => "add process done"
+  when :already_exist
+    addprocess = splash_return case: :already_exist, :more => "add process twice not allowed"
+  when :failure
+    addprocess = splash_return case: :configuration_error, :more => "add process failed"
+    addprocess[:data] = res
+  end
+  content_type format
+  format_response(addprocess, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+end
+
+
+
+WebAdminApp.delete '/api/config/deletelog/:label.?:format?' do
+  log = get_logger
+  format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
+  log.call "API : config, verb : DELETE, route : deletelog, format : #{format}"
+  deletelog = {}
+  logsrec = Splash::Logs::LogsRecords::new params[:label].to_sym
+  logsrec.clear
+  res = get_config.delete_record :type => :logs, key: :label, label: params[:label].to_sym
+  case res[:status]
+  when :success
+    deletelog = splash_return case: :quiet_exit, :more => "delete log done"
+  when :not_found
+    deletelog = splash_return case: :not_found, :more => "nothing done for logs"
+  else
+    deletelog = splash_return case: :configuration_error, :more => "delete log failed"
+  end
+  content_type format
+  format_response(deletelog, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+end
+
+WebAdminApp.delete '/api/config/deleteprocess/:process.?:format?' do
+  log = get_logger
+  format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
+  log.call "API : config, verb : DELETE, route : deleteprocess, format : #{format}"
+  deleteprocess = {}
+  procrec = Splash::Processes::ProcessRecords::new params[:process].to_sym
+  procrec.clear
+  res = get_config.delete_record :type => :processes, :key => :process, process: params[:process].to_sym
+  case res[:status]
+  when :success
+    deleteprocess = splash_return case: :quiet_exit, :more => "delete process done"
+  when :not_found
+    deleteprocess = splash_return case: :not_found, :more => "nothing done for processes"
+  else
+    deleteprocess = splash_return case: :configuration_error, :more => "delete process failed"
+  end
+  content_type format
+  format_response(deleteprocess, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+end