prometheus-splash 0.7.0 → 0.8.0

data/lib/splash/constants.rb

@@ -7,7 +7,7 @@ module Splash
   module Constants
 
     # Current splash version
-    VERSION = "0.7.0"
+    VERSION = "0.8.0"
 
     # the path to th config file, not overridable by config
     CONFIG_FILE = "/etc/splash.yml"
@@ -43,7 +43,10 @@ module Splash
     LICENSE="BSD-2-Clause"
 
     # the default prometheus pushgateway URL
-    PROMETHEUS_PUSHGATEWAY_URL = 'http://localhost:9090/'
+    PROMETHEUS_PUSHGATEWAY_URL = 'http://localhost:9091/'
+
+    # the default prometheus Alertmanager URL
+    PROMETHEUS_ALERTMANAGER_URL = 'http://localhost:9092/'
 
     # the default prometheus URL
     PROMETHEUS_URL = "http://localhost:9090/"
@@ -84,6 +87,8 @@ module Splash
     # the default sdterr trace file
     WEBADMIN_STDERR_TRACE="stderr.txt"
 
+    # default transfer retention for trace
+    TRANSFER_DEFAULT_RETENTION=1
 
   end
 end

data/lib/splash/daemon/metrics.rb

@@ -17,8 +17,8 @@ module Splash
 
       # metrics manager factory
       # @return [Splash::Daemon::Metrics::Manager]
-      def get_metrics_manager
-        return @@manager ||= Manager::new
+      def get_metrics_manager(session)
+        return @@manager ||= Manager::new(:session => session)
       end
 
       # Metrics Manager (collect and sending to Prometheus)
@@ -32,7 +32,8 @@ module Splash
         attr_reader :monitoring_processes_count
 
         # Constructor prepare prometheus-client, defined metrics and init attributes
-        def initialize
+        def initialize(options ={})
+          @session = options[:session]
           @config = get_config
           @starttime = Time.now
           @execution_count = 0
@@ -75,12 +76,11 @@ module Splash
         # @return [Hash] Exiter case ( :service_dependence_missing , :quiet_exit)
         def notify
           log = get_logger
-          session  = get_session
           unless verify_service url: @config.prometheus_pushgateway_url then
             return  { :case => :service_dependence_missing, :more => "Prometheus Notification not send." }
           end
 
-          log.debug "Sending Splash self metrics to PushGateway." , session
+          log.ok "Sending Splash self metrics to PushGateway." , @session
           @metric_uptime.set uptime
           @metric_execution.set execution_count
           @metric_logs_monitoring.set monitoring_logs_count
@@ -89,7 +89,7 @@ module Splash
           hostname = Socket.gethostname
           url = @config.prometheus_pushgateway_url
           Prometheus::Client::Push.new('Splash',hostname, url).add(@registry)
-          log.debug "Sending to Prometheus PushGateway done.", session
+          log.debug "Sending to Prometheus PushGateway done.", @session
           return {:case => :quiet_exit }
         end
 

data/lib/splash/daemon/orchestrator.rb

@@ -32,20 +32,69 @@ module Splash
         def initialize(options = {})
           @log = get_logger
           self.extend Splash::Daemon::Metrics
-          @metric_manager = get_metrics_manager
+          @session = get_session
+          @metric_manager = get_metrics_manager(@session)
           $stdout.sync = true
           $stderr.sync = true
           @server  = Rufus::Scheduler::new
           @server.extend SchedulerHooks
           @config = get_config
+          @scheduling = options[:scheduling]
 
           @log.info "Splash Orchestrator starting :"
-          if options[:scheduling] then
-            @log.item "Initializing commands Scheduling."
+          if @scheduling then
+            @log.item "Initializing Sequences & commands Scheduling."
             init_commands_scheduling
             init_sequences_scheduling
           end
 
+          init_logs_monitoring_scheduling
+          init_process_monitoring_scheduling
+          init_metrics_scheduling
+          init_daemon_subscriber
+
+        end
+
+
+
+
+
+        # Stop the Splash daemon gracefully
+        # @return [hash] Exiter Case :quiet_exit
+        def terminate
+          @log.info "Splash daemon shutdown"
+          @server.shutdown
+          change_logger logger: :cli
+          splash_exit case: :quiet_exit
+        end
+
+        private
+
+        #prepare main daemon subscriber
+        def init_daemon_subscriber
+          hostname = Socket.gethostname
+          transport = get_default_subscriber queue: "splash.#{hostname}.input"
+          if transport.class == Hash and transport.include? :case then
+            splash_exit transport
+          end
+          transport.subscribe(:block => true) do |delivery_info, properties, body|
+            content = YAML::load(body)
+            session = get_session
+            content[:session] = session
+            if VERBS.include? content[:verb]
+              @log.receive "Valid remote order, verb : #{content[:verb].to_s}", session
+              res = self.send content[:verb], content
+              get_default_client.publish queue: content[:return_to], message: res.to_yaml
+              @log.send "Result to #{content[:return_to]}.", session
+            else
+              @log.receive "INVALID remote order, verb : #{content[:verb].to_s}", session
+              get_default_client.publish queue: content[:return_to], message: "Unkown verb #{content[:verb]}".to_yaml
+            end
+          end
+        end
+
+        #prepare logs monitoring  sheduling
+        def init_logs_monitoring_scheduling
           if @config.logs.empty? then
             @log.item "No logs to monitor"
           else
@@ -64,7 +113,10 @@ module Splash
               end
             end
           end
+        end
 
+        #prepare process monitoring  sheduling
+        def init_process_monitoring_scheduling
           if @config.processes.empty? then
             @log.item "No processes to monitor"
           else
@@ -83,49 +135,24 @@ module Splash
               end
             end
           end
+        end
 
 
+        #prepare metrics sheduling
+        def init_metrics_scheduling
           sched,value = @config.daemon_metrics_scheduling.flatten
           @log.item "Initializing Splash metrics notifications."
           @server.send sched,value do
             begin
+              @log.trigger "Splash Metrics monitoring for Scheduling : #{sched.to_s} #{value.to_s}", @session
               @metric_manager.notify
             rescue Errno::ECONNREFUSED
               @log.error "PushGateway seems to be done, please start it."
             end
           end
-
-          hostname = Socket.gethostname
-          transport = get_default_subscriber queue: "splash.#{hostname}.input"
-          if transport.class == Hash and transport.include? :case then
-            splash_exit transport
-          end
-          transport.subscribe(:block => true) do |delivery_info, properties, body|
-            content = YAML::load(body)
-            session = get_session
-            content[:session] = session
-            if VERBS.include? content[:verb]
-              @log.receive "Valid remote order, verb : #{content[:verb].to_s}", session
-              res = self.send content[:verb], content
-              get_default_client.publish queue: content[:return_to], message: res.to_yaml
-              @log.send "Result to #{content[:return_to]}.", session
-            else
-              @log.receive "INVALID remote order, verb : #{content[:verb].to_s}", session
-              get_default_client.publish queue: content[:return_to], message: "Unkown verb #{content[:verb]}".to_yaml
-            end
-          end
         end
 
-        # Stop the Splash daemon gracefully
-        # @return [hash] Exiter Case :quiet_exit
-        def terminate
-          @log.info "Splash daemon shutdown"
-          @server.shutdown
-          change_logger logger: :cli
-          splash_exit case: :quiet_exit
-        end
 
-        private
         # prepare commands Scheduling
         def init_commands_scheduling
           config = get_config.commands
@@ -139,7 +166,6 @@ module Splash
               execute command: command.to_s, session: session
             end
           end
-
         end
 
 
@@ -156,9 +182,25 @@ module Splash
               run_seq name: sequence.to_s, session: session
             end
           end
-
         end
 
+        # reset the orchestrator
+        # @return [Hash] Exiter case
+        def reset_orchestrator
+          @server.shutdown
+          @server = Rufus::Scheduler::new
+          @server.extend SchedulerHooks
+          @config = rehash_config
+          @log.info "Splash Orchestrator re-hashing :"
+          if @scheduling then
+            @log.item "Re-Initializing Sequences & commands Scheduling."
+            init_commands_scheduling
+            init_sequences_scheduling
+          end
+          init_logs_monitoring_scheduling
+          init_process_monitoring_scheduling
+          init_metrics_scheduling
+        end
 
 
         # execute_command verb : execute command specified in payload
@@ -174,9 +216,7 @@ module Splash
             return command.call_and_notify trace: true, notify: true, callback: true, session: options[:session]
           end
         end
-
       end
-
     end
   end
 end

data/lib/splash/daemon/orchestrator/grammar.rb

@@ -16,7 +16,7 @@ module Splash
         include Splash::Loggers
 
         # list of known verbs for Splash orchestrator
-        VERBS=[:ping,:list_commands,:execute_command,:ack_command, :shutdown]
+        VERBS=[:ping,:list_commands,:execute_command,:ack_command, :shutdown, :get_jobs, :reset]
 
         # shutdown verb : stop the Splash daemon gracefully
         # @param [Hash] content message content Hash Structure, ignored
@@ -46,6 +46,21 @@ module Splash
           return execute command: content[:payload][:name], ack: true
         end
 
+
+        # get_jobs verb : return list of scheduled jobs for internal scheduler
+        # @param [Hash] content message content Hash Structure, ignored
+        # @return [String] YAML dataset
+        def get_jobs(content)
+          return @server.jobs.to_yaml
+        end
+
+        # reset verb : reset the internal scheduler
+        # @param [Hash] content message content Hash Structure, ignored
+        # @return [String] "Scheduler reset" static
+        def reset(content)
+          return "Scheduler reset" if reset_orchestrator
+        end
+
         # execute_command verb : execute command specified in payload
         # @param [Hash] content message content Hash Structure, include  mandatory payload[:name]
         # @return [Hash] Exiter case

data/lib/splash/dependencies.rb

@@ -17,8 +17,12 @@ module Splash
 
 
 
+
+
     # Rubygems
     begin
+      require 'net/ssh'
+      require 'net/scp'
       require 'prometheus/client'
       require 'prometheus/client/push'
       require 'thor'
@@ -35,6 +39,7 @@ module Splash
       require 'rest-client'
       require 'kramdown'
       require 'rack/reverse_proxy'
+      require 'tty-table'
 
 
     rescue Gem::GemNotFoundException
@@ -57,9 +62,9 @@ module Splash
 
     require 'splash/commands'
     require 'splash/sequences'
-
     require 'splash/logs'
     require 'splash/processes'
+    require 'splash/transfers'
 
     require 'splash/daemon'
     require 'splash/webadmin'

data/lib/splash/helpers.rb

@@ -252,7 +252,7 @@ module Splash
     # check folder
     # @return [Array] of Symbol with error type : [:inexistant,:mode,:owner,:group]
     # @param [Hash] options
-    # @option options [String] :path folder path (relative or absolute)
+    # @option options [String] :name folder path (relative or absolute)
     # @option options [String] :mode String for OCTAL rights like "644", optionnal
     # @option options [String] :owner file owner for folder, optionnal
     # @option options [String] :group  file group for folder, optionnal
@@ -336,7 +336,6 @@ module Splash
     end
     #!@endgroup
 
-
     def format_response(data, format)
       response = case format
                  when 'application/json' then JSON.pretty_generate(data)
@@ -355,6 +354,16 @@ module Splash
        return result[extension]
     end
 
+    # check if unicode must be used with term ENV
+    # @return [Boolean]
+    def check_unicode_term
+      return false unless ENV.include? "TERM"
+      if ENV.values_at("LC_ALL","LC_CTYPE","LANG").compact.first.include?("UTF-8") and ENV.values_at('TERM').first.include? "xterm" then
+        return true
+      else
+        return false
+      end
+    end
 
   end
 end

data/lib/splash/loggers/cli.rb

@@ -10,6 +10,7 @@ module Splash
     class Cli < Splash::Loggers::LoggerTemplate
 
       include Splash::Config
+      include Splash::Helpers
 
       # mapping of UTf-8 emoji for log levels or alias
       EMOJI =  { :unknown => "\u{1F4A5}",
@@ -82,16 +83,7 @@ module Splash
         get_config.loggers[:cli][:color] = status
       end
 
-      # check if unicode must be used with term ENV
-      # @return [Boolean]
-      def check_unicode_term
-        return false unless ENV.include? "TERM"
-        if ENV.values_at("LC_ALL","LC_CTYPE","LANG").compact.first.include?("UTF-8") and ENV.values_at('TERM').first.include? "xterm" then
-          return true
-        else
-          return false
-        end
-      end
+
 
     end
 

data/lib/splash/transfers.rb

@@ -0,0 +1,224 @@
+# coding: utf-8
+
+# base Splash Module
+module Splash
+
+  # Transfers module
+  module Transfers
+
+    include Splash::Config
+    include Splash::Loggers
+    include Splash::Helpers
+
+
+
+    class TxNotifier
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_nbfiles = Prometheus::Client::Gauge.new(:txnbfiles, docstring: 'SPLASH metric transfer number of files')
+      @@metric_nbfiles_failed = Prometheus::Client::Gauge.new(:txnbfilesfailed, docstring: 'SPLASH metric transfer number of failed files')
+      @@metric_time = Prometheus::Client::Gauge.new(:txtime, docstring: 'SPLASH metric transfer execution time')
+      @@registry.register(@@metric_nbfiles)
+      @@registry.register(@@metric_nbfiles_failed)
+      @@registry.register(@@metric_time)
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = "tx_#{options[:name].to_s}"
+        @nbfiles = options[:nbfiles]
+        @nbfiles_failed = options[:nbfiles_failed]
+        @time = options[:time]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+	unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_nbfiles.set(@nbfiles)
+        @@metric_nbfiles_failed.set(@nbfiles_failed)
+        @@metric_time.set(@time)
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new(@name, hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+
+    class TxRecords
+      include Splash::Backends
+      def initialize(name)
+        @name = name
+        @backend = get_backend :transferts_trace
+      end
+
+      def purge(retention)
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          retention = TRANSFER_DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item,value|
+          DateTime.parse(item) <= (adjusted_datetime) and value[:status] != :prepared}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data[DateTime.now.to_s] = record
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : {}
+      end
+
+      def check_prepared
+        return :never_run_prepare unless @backend.exist?({key: @name})
+        return :never_prepare unless YAML::load(@backend.get({key: @name})).select {|item,value|
+          value[:status] == :prepared
+        }.count > 0
+        return :prepared
+      end
+
+    end
+
+    def run_txs(options = {})
+      log = get_logger
+      log.info 'Running Transfers'
+      count=0
+      get_config.transfers.each do |record|
+        txrec =  TxRecords::new record[:name]
+        txrec.purge(record[:retention])
+        log.item "Execute : #{record[:name]},  #{record[:desc]}"
+        case txrec.check_prepared
+        when :prepared
+          if record[:type] == :push then
+            unless push record
+              count += 1
+            end
+          elsif record[:type] == :pull then
+            unless pull record
+              count += 1
+            end
+          else
+            log.ko "Transfer type unkown"
+            count += 1
+          end
+        when :never_prepare
+          log.ko "#{record[:name]} : Never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        when :never_run_prepare
+          log.ko "#{record[:name]} : Never Executed and never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        end
+      end
+      return {:case => :error_exit, :more => "#{count} Transfer(s) failed"} if count > 0
+      return {:case => :quiet_exit }
+    end
+
+
+
+    def prepare_tx(name)
+      log = get_logger
+      record = get_config.transfers.select { |item| item[:name] == name.to_sym }.first
+      home  = Etc.getpwnam(record[:local][:user]).dir
+      identity = ::File::readlines("#{home}/.ssh/id_rsa.pub").first.chomp
+      folder  = {:mode => "755",
+                 :owner => record[:local][:user] ,
+                 :group => Etc.getgrgid(Etc.getpwnam(record[:local][:user]).gid).name,
+                 :name => record[:local][:path],
+                 :path => record[:local][:path]}
+      log.info "Ensure local folder : #{record[:local][:path]}"
+      make_folder(folder) unless verify_folder(folder).empty?
+      begin
+        log.info "Ensure RSA Key sharing for local user : #{record[:local][:user]} to remote user : #{record[:remote][:user]}@#{record[:remote][:host]}"
+        ssh = Net::SSH.start(record[:remote][:host],record[:remote][:user])
+        output = ssh.exec!(%[
+          /bin/bash -cl '
+          umask 077;
+          mkdir #{record[:remote][:path]};
+          test -d ~/.ssh || mkdir ~/.ssh;
+          if [ ! -f ~/.ssh/authorized_keys -o `grep "#{identity}" ~/.ssh/authorized_keys 2> /dev/null | wc -l` -eq 0 ]; then              echo "#{identity}" >> ~/.ssh/authorized_keys
+          fi'])
+        log.info "Prepare remote folder : #{record[:remote][:path]}"
+        log.info "Prepare data file for transfer : #{record[:name]}"
+        txrec = TxRecords::new record[:name]
+        txrec.add_record :status => :prepared
+        log.ok "Transfer : #{record[:name]} prepared successfully"
+        return {:case => :quiet_exit }
+      rescue Interrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      rescue TTY::Reader::InputInterrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      end
+    end
+
+
+
+    def save_data
+
+    end
+
+
+
+    def push(record)
+      config = get_config
+      log = get_logger
+      txrec = TxRecords::new record[:name]
+      start = Time.now
+      res = true
+      count = 0
+      done =[]
+      start_date = DateTime.now.to_s
+      list = Dir.glob("#{record[:local][:path]}/#{record[:pattern]}")
+      count = list.count
+      log.arrow "Transfering #{count} file(s)"
+
+      begin
+        scp = Net::SCP.start(record[:remote][:host],record[:remote][:user])
+        list.each do|f|
+          log.arrow "Copy file : #{f} to #{record[:remote][:user]}@#{record[:remote][:host]}:#{record[:remote][:path]}"
+          scp.upload! f, record[:remote][:path]
+          done.push f
+          if record[:backup] then
+            log.arrow "File #{f} backuped"
+            FileUtils::mv(f, "#{f}.#{Time.now.getutc.to_i}")
+          else
+            FileUtils::unlink(f)
+          end
+        end
+
+      rescue
+        res = false
+      end
+
+      end_date = DateTime.now.to_s
+      time = Time.now - start
+      status = (res)? :success : :failure
+      txrec.add_record :status => status,
+                       :end_date => end_date,
+                       :time => time,
+                       :count => count,
+                       :wanted => list,
+                       :done => done
+      count_failed = list.count - done.count
+      txmonitor = TxNotifier::new({name: record[:name], nbfiles: count,nbfiles_failed: count_failed, time: time})
+      if txmonitor.notify then
+        log.ok "Sending metrics to Prometheus Pushgateway"
+      else
+        log.ko "Failed to send metrics to Prometheus Pushgateway"
+      end
+      return res
+    end
+  end
+end