prometheus-splash 0.6.1 → 0.8.3

data/lib/splash/daemon/orchestrator/grammar.rb

@@ -16,7 +16,7 @@ module Splash
         include Splash::Loggers
 
         # list of known verbs for Splash orchestrator
-        VERBS=[:ping,:list_commands,:execute_command,:ack_command, :shutdown]
+        VERBS=[:ping,:list_commands,:execute_command,:ack_command, :shutdown, :get_jobs, :reset]
 
         # shutdown verb : stop the Splash daemon gracefully
         # @param [Hash] content message content Hash Structure, ignored
@@ -46,6 +46,21 @@ module Splash
           return execute command: content[:payload][:name], ack: true
         end
 
+
+        # get_jobs verb : return list of scheduled jobs for internal scheduler
+        # @param [Hash] content message content Hash Structure, ignored
+        # @return [String] YAML dataset
+        def get_jobs(content)
+          return @server.jobs.to_yaml
+        end
+
+        # reset verb : reset the internal scheduler
+        # @param [Hash] content message content Hash Structure, ignored
+        # @return [String] "Scheduler reset" static
+        def reset(content)
+          return "Scheduler reset" if reset_orchestrator
+        end
+
         # execute_command verb : execute command specified in payload
         # @param [Hash] content message content Hash Structure, include  mandatory payload[:name]
         # @return [Hash] Exiter case

data/lib/splash/dependencies.rb

@@ -17,8 +17,12 @@ module Splash
 
 
 
+
+
     # Rubygems
     begin
+      require 'net/ssh'
+      require 'net/scp'
       require 'prometheus/client'
       require 'prometheus/client/push'
       require 'thor'
@@ -35,6 +39,7 @@ module Splash
       require 'rest-client'
       require 'kramdown'
       require 'rack/reverse_proxy'
+      require 'tty-table'
 
 
     rescue Gem::GemNotFoundException
@@ -57,9 +62,9 @@ module Splash
 
     require 'splash/commands'
     require 'splash/sequences'
-
     require 'splash/logs'
     require 'splash/processes'
+    require 'splash/transfers'
 
     require 'splash/daemon'
     require 'splash/webadmin'

data/lib/splash/exiter.rb

@@ -60,7 +60,7 @@ module Splash
 
     def splash_return(options = {})
 
-      data = EXIT_MAP[options[:case]]
+      data = EXIT_MAP[options[:case]].clone
       data[:status] = (data[:code]>0)? :failure : :success
       data[:more] = options[:more] if options[:more]
       return data

data/lib/splash/helpers.rb

@@ -252,7 +252,7 @@ module Splash
     # check folder
     # @return [Array] of Symbol with error type : [:inexistant,:mode,:owner,:group]
     # @param [Hash] options
-    # @option options [String] :path folder path (relative or absolute)
+    # @option options [String] :name folder path (relative or absolute)
     # @option options [String] :mode String for OCTAL rights like "644", optionnal
     # @option options [String] :owner file owner for folder, optionnal
     # @option options [String] :group  file group for folder, optionnal
@@ -336,12 +336,11 @@ module Splash
     end
     #!@endgroup
 
-
     def format_response(data, format)
       response = case format
                  when 'application/json' then JSON.pretty_generate(data)
                  when 'text/x-yaml' then data.to_yaml
-                 else JSON.pretty_generate(data)
+                 else data.to_yaml
                  end
       return response
     end
@@ -355,6 +354,16 @@ module Splash
        return result[extension]
     end
 
+    # check if unicode must be used with term ENV
+    # @return [Boolean]
+    def check_unicode_term
+      return false unless ENV.include? "TERM"
+      if ENV.values_at("LC_ALL","LC_CTYPE","LANG").compact.first.include?("UTF-8") and ENV.values_at('TERM').first.include? "xterm" then
+        return true
+      else
+        return false
+      end
+    end
 
   end
 end

data/lib/splash/loggers/cli.rb

@@ -10,6 +10,7 @@ module Splash
     class Cli < Splash::Loggers::LoggerTemplate
 
       include Splash::Config
+      include Splash::Helpers
 
       # mapping of UTf-8 emoji for log levels or alias
       EMOJI =  { :unknown => "\u{1F4A5}",
@@ -82,16 +83,7 @@ module Splash
         get_config.loggers[:cli][:color] = status
       end
 
-      # check if unicode must be used with term ENV
-      # @return [Boolean]
-      def check_unicode_term
-        return false unless ENV.include? "TERM"
-        if ENV.values_at("LC_ALL","LC_CTYPE","LANG").compact.first.include?("UTF-8") and ENV.values_at('TERM').first.include? "xterm" then
-          return true
-        else
-          return false
-        end
-      end
+
 
     end
 

data/lib/splash/logs.rb

@@ -6,6 +6,79 @@ module Splash
   # Logs namespace
   module Logs
 
+    class LogsNotifier
+
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_missing = Prometheus::Client::Gauge.new(:logmissing, docstring: 'SPLASH metric log missing', labels: [:log ])
+      @@metric_count = Prometheus::Client::Gauge.new(:logerrors, docstring: 'SPLASH metric log error', labels: [:log ])
+      @@metric_lines = Prometheus::Client::Gauge.new(:loglines, docstring: 'SPLASH metric log lines numbers', labels: [:log ])
+      @@registry.register(@@metric_count)
+      @@registry.register(@@metric_missing)
+      @@registry.register(@@metric_lines)
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = options[:name]
+        @missing = options[:missing]
+        @lines = options[:lines]
+        @errors = options[:errors]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+        unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_missing.set(@missing, labels: { log: @name })
+        @@metric_count.set(@errors, labels: { log: @name })
+        @@metric_lines.set(@lines, labels: { log: @name })
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new("Splash", hostname, @url).add(@@registry)
+      end
+
+    end
+
+    class LogsRecords
+      include Splash::Backends
+      include Splash::Constants
+      def initialize(name)
+        @name = name
+        @backend = get_backend :logs_trace
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+    end
+
+
+
     # Log scanner and notifier
     class LogScanner
       include Splash::Constants
@@ -17,13 +90,7 @@ module Splash
       def initialize
         @logs_target = Marshal.load(Marshal.dump(get_config.logs))
         @config = get_config
-        @registry = Prometheus::Client::Registry::new
-        @metric_count = Prometheus::Client::Gauge.new(:logerrors, docstring: 'SPLASH metric log error', labels: [:log ])
-        @metric_missing = Prometheus::Client::Gauge.new(:logmissing, docstring: 'SPLASH metric log missing', labels: [:log ])
-        @metric_lines = Prometheus::Client::Gauge.new(:loglines, docstring: 'SPLASH metric log lines numbers', labels: [:log ])
-        @registry.register(@metric_count)
-        @registry.register(@metric_missing)
-        @registry.register(@metric_lines)
+
       end
 
 
@@ -62,17 +129,24 @@ module Splash
         session = (options[:session]) ? options[:session] : log.get_session
         log.info "Sending metrics to Prometheus Pushgateway", session
         @logs_target.each do |item|
-          missing = (item[:status] == :missing)? 1 : 0
-          log.item "Sending metrics for #{item[:log]}", session
-          @metric_count.set(item[:count], labels: { log: item[:log] })
-          @metric_missing.set(missing, labels: { log: item[:log] })
+          logsrec = LogsRecords::new item[:label]
+          errors = (item[:count])? item[:count] : 0
           lines = (item[:lines])? item[:lines] : 0
-          @metric_lines.set(lines, labels: { log: item[:log] })
+          missing = (item[:status] = :missing)? 1 : 0
+          file = item[:log]
+          logsrec.purge(item[:retention])
+          logsrec.add_record :status => item[:status],
+                           :errors => errors,
+                           :lines => lines,
+                           :file => file
+
+          logsmonitor = LogsNotifier::new({name: item[:label], missing: missing, file: file, errors: errors, lines: lines})
+          if logsmonitor.notify then
+            log.ok "Sending metrics for log #{file} to Prometheus Pushgateway", session
+          else
+            log.ko "Failed to send metrics for log #{file} to Prometheus Pushgateway", session
+          end
         end
-        hostname = Socket.gethostname
-        url = @config.prometheus_pushgateway_url
-        Prometheus::Client::Push.new('Splash',hostname, url).add(@registry)
-        log.ok "Sending to Prometheus PushGateway done.", session
         return {:case => :quiet_exit }
       end
 

data/lib/splash/processes.rb

@@ -6,6 +6,81 @@ module Splash
   # Processes namespace
   module Processes
 
+
+    class ProcessNotifier
+
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_status = Prometheus::Client::Gauge.new(:process_status, docstring: 'SPLASH metric process status', labels: [:process ])
+      @@metric_cpu_percent = Prometheus::Client::Gauge.new(:process_cpu_percent, docstring: 'SPLASH metric process CPU usage in percent', labels: [:process ])
+      @@metric_mem_percent = Prometheus::Client::Gauge.new(:process_mem_percent, docstring: 'SPLASH metric process MEM usage in percent', labels: [:process ])
+      @@registry.register(@@metric_status)
+      @@registry.register(@@metric_cpu_percent)
+      @@registry.register(@@metric_mem_percent)
+
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = options[:name]
+        @status = options[:status]
+        @cpu_percent = options[:cpu_percent]
+        @mem_percent = options[:mem_percent]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+        unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_mem_percent.set(@mem_percent, labels: { process: @name })
+        @@metric_cpu_percent.set(@cpu_percent, labels: { process: @name })
+        @@metric_status.set(@status, labels: { process: @name })
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new("Splash", hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+    class ProcessRecords
+      include Splash::Backends
+      include Splash::Constants
+
+      def initialize(name)
+        @name = name
+        @backend = get_backend :process_trace
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+    end
+
     # Processes scanner and notifier
     class ProcessScanner
       include Splash::Constants
@@ -17,14 +92,6 @@ module Splash
       def initialize
         @processes_target = Marshal.load(Marshal.dump(get_config.processes))
         @config = get_config
-        @registry = Prometheus::Client::Registry::new
-        @metric_status = Prometheus::Client::Gauge.new(:process_status, docstring: 'SPLASH metric process status', labels: [:process ])
-        @metric_cpu_percent = Prometheus::Client::Gauge.new(:process_cpu_percent, docstring: 'SPLASH metric process CPU usage in percent', labels: [:process ])
-        @metric_mem_percent = Prometheus::Client::Gauge.new(:process_mem_percent, docstring: 'SPLASH metric process MEM usage in percent', labels: [:process ])
-        @registry.register(@metric_status)
-        @registry.register(@metric_cpu_percent)
-        @registry.register(@metric_mem_percent)
-
       end
 
 
@@ -64,17 +131,21 @@ module Splash
         session = (options[:session]) ? options[:session] : log.get_session
         log.info "Sending metrics to Prometheus Pushgateway", session
         @processes_target.each do |item|
+          processrec = ProcessRecords::new item[:process]
           missing = (item[:status] == :missing)? 1 : 0
-          log.item "Sending metrics for #{item[:process]}", session
           val = (item[:status] == :running )? 1 : 0
-          @metric_status.set(val, labels: { process: item[:process] })
-          @metric_cpu_percent.set(item[:cpu], labels: { process: item[:process] })
-          @metric_mem_percent.set(item[:mem], labels: { process: item[:process] })
+          processrec.purge(item[:retention])
+          processrec.add_record :status => item[:status],
+                           :cpu_percent => item[:cpu],
+                           :mem_percent => item[:mem] ,
+                           :process => item[:process]
+          processmonitor = ProcessNotifier::new({name: item[:process], status: val , cpu_percent: item[:cpu], mem_percent: item[:mem]})
+          if processmonitor.notify then
+            log.ok "Sending metrics for process #{item[:process]} to Prometheus Pushgateway", session
+          else
+            log.ko "Failed to send metrics for process #{item[:process]} to Prometheus Pushgateway", session
+          end
         end
-        hostname = Socket.gethostname
-        url = @config.prometheus_pushgateway_url
-        Prometheus::Client::Push.new('Splash',hostname, url).add(@registry)
-        log.ok "Sending to Prometheus PushGateway done.", session
         return {:case => :quiet_exit }
       end
 

data/lib/splash/transfers.rb

@@ -0,0 +1,229 @@
+# coding: utf-8
+
+# base Splash Module
+module Splash
+
+  # Transfers module
+  module Transfers
+
+    include Splash::Config
+    include Splash::Loggers
+    include Splash::Helpers
+
+
+
+    class TxNotifier
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_nbfiles = Prometheus::Client::Gauge.new(:txnbfiles, docstring: 'SPLASH metric transfer number of files')
+      @@metric_nbfiles_failed = Prometheus::Client::Gauge.new(:txnbfilesfailed, docstring: 'SPLASH metric transfer number of failed files')
+      @@metric_time = Prometheus::Client::Gauge.new(:txtime, docstring: 'SPLASH metric transfer execution time')
+      @@registry.register(@@metric_nbfiles)
+      @@registry.register(@@metric_nbfiles_failed)
+      @@registry.register(@@metric_time)
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = "tx_#{options[:name].to_s}"
+        @nbfiles = options[:nbfiles]
+        @nbfiles_failed = options[:nbfiles_failed]
+        @time = options[:time]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+	unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_nbfiles.set(@nbfiles)
+        @@metric_nbfiles_failed.set(@nbfiles_failed)
+        @@metric_time.set(@time)
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new(@name, hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+
+    class TxRecords
+      include Splash::Backends
+      include Splash::Constants
+
+      def initialize(name)
+        @name = name
+        @backend = get_backend :transfers_trace
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+      def check_prepared
+        return :never_run_prepare unless @backend.exist?({key: @name})
+        return :never_prepare unless YAML::load(@backend.get({key: @name})).select {|item|
+          record =item.keys.first
+          value=item[record]
+          value[:status] == :prepared
+        }.count > 0
+        return :prepared
+      end
+
+    end
+
+    def run_txs(options = {})
+      log = get_logger
+      log.info 'Running Transfers'
+      count=0
+      get_config.transfers.each do |record|
+        txrec =  TxRecords::new record[:name]
+        txrec.purge(record[:retention])
+        log.item "Execute : #{record[:name]},  #{record[:desc]}"
+        case txrec.check_prepared
+        when :prepared
+          if record[:type] == :push then
+            unless push record
+              count += 1
+            end
+          elsif record[:type] == :pull then
+            unless pull record
+              count += 1
+            end
+          else
+            log.ko "Transfer type unkown"
+            count += 1
+          end
+        when :never_prepare
+          log.ko "#{record[:name]} : Never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        when :never_run_prepare
+          log.ko "#{record[:name]} : Never Executed and never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        end
+      end
+      return {:case => :error_exit, :more => "#{count} Transfer(s) failed"} if count > 0
+      return {:case => :quiet_exit }
+    end
+
+
+
+    def prepare_tx(name)
+      log = get_logger
+      record = get_config.transfers.select { |item| item[:name] == name.to_sym }.first
+      home  = Etc.getpwnam(record[:local][:user]).dir
+      identity = ::File::readlines("#{home}/.ssh/id_rsa.pub").first.chomp
+      folder  = {:mode => "755",
+                 :owner => record[:local][:user] ,
+                 :group => Etc.getgrgid(Etc.getpwnam(record[:local][:user]).gid).name,
+                 :name => record[:local][:path],
+                 :path => record[:local][:path]}
+      log.info "Ensure local folder : #{record[:local][:path]}"
+      make_folder(folder) unless verify_folder(folder).empty?
+      begin
+        log.info "Ensure RSA Key sharing for local user : #{record[:local][:user]} to remote user : #{record[:remote][:user]}@#{record[:remote][:host]}"
+        ssh = Net::SSH.start(record[:remote][:host],record[:remote][:user])
+        output = ssh.exec!(%[
+          /bin/bash -cl '
+          umask 077;
+          mkdir #{record[:remote][:path]};
+          test -d ~/.ssh || mkdir ~/.ssh;
+          if [ ! -f ~/.ssh/authorized_keys -o `grep "#{identity}" ~/.ssh/authorized_keys 2> /dev/null | wc -l` -eq 0 ]; then              echo "#{identity}" >> ~/.ssh/authorized_keys
+          fi'])
+        log.info "Prepare remote folder : #{record[:remote][:path]}"
+        log.info "Prepare data file for transfer : #{record[:name]}"
+        txrec = TxRecords::new record[:name]
+        txrec.add_record :status => :prepared
+        log.ok "Transfer : #{record[:name]} prepared successfully"
+        return {:case => :quiet_exit }
+      rescue Interrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      rescue TTY::Reader::InputInterrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      end
+    end
+
+
+
+    def save_data
+
+    end
+
+
+
+    def push(record)
+      config = get_config
+      log = get_logger
+      txrec = TxRecords::new record[:name]
+      start = Time.now
+      res = true
+      count = 0
+      done =[]
+      start_date = DateTime.now.to_s
+      list = Dir.glob("#{record[:local][:path]}/#{record[:pattern]}")
+      count = list.count
+      log.arrow "Transfering #{count} file(s)"
+
+      begin
+        scp = Net::SCP.start(record[:remote][:host],record[:remote][:user])
+        list.each do|f|
+          log.arrow "Copy file : #{f} to #{record[:remote][:user]}@#{record[:remote][:host]}:#{record[:remote][:path]}"
+          scp.upload! f, record[:remote][:path]
+          done.push f
+          if record[:backup] then
+            log.arrow "File #{f} backuped"
+            FileUtils::mv(f, "#{f}.#{Time.now.getutc.to_i}")
+          else
+            FileUtils::unlink(f)
+          end
+        end
+
+      rescue
+        res = false
+      end
+
+      end_date = DateTime.now.to_s
+      time = Time.now - start
+      status = (res)? :success : :failure
+      txrec.add_record :status => status,
+                       :end_date => end_date,
+                       :time => time,
+                       :count => count,
+                       :wanted => list,
+                       :done => done
+      count_failed = list.count - done.count
+      txmonitor = TxNotifier::new({name: record[:name], nbfiles: count,nbfiles_failed: count_failed, time: time})
+      if txmonitor.notify then
+        log.ok "Sending metrics to Prometheus Pushgateway"
+      else
+        log.ko "Failed to send metrics to Prometheus Pushgateway"
+      end
+      return res
+    end
+  end
+end