prometheus-splash 0.7.0 → 0.8.4

data/lib/splash/daemon/orchestrator.rb

@@ -32,20 +32,69 @@ module Splash
         def initialize(options = {})
           @log = get_logger
           self.extend Splash::Daemon::Metrics
-          @metric_manager = get_metrics_manager
+          @session = get_session
+          @metric_manager = get_metrics_manager(@session)
           $stdout.sync = true
           $stderr.sync = true
           @server  = Rufus::Scheduler::new
           @server.extend SchedulerHooks
           @config = get_config
+          @scheduling = options[:scheduling]
 
           @log.info "Splash Orchestrator starting :"
-          if options[:scheduling] then
-            @log.item "Initializing commands Scheduling."
+          if @scheduling then
+            @log.item "Initializing Sequences & commands Scheduling."
             init_commands_scheduling
             init_sequences_scheduling
           end
 
+          init_logs_monitoring_scheduling
+          init_process_monitoring_scheduling
+          init_metrics_scheduling
+          init_daemon_subscriber
+
+        end
+
+
+
+
+
+        # Stop the Splash daemon gracefully
+        # @return [hash] Exiter Case :quiet_exit
+        def terminate
+          @log.info "Splash daemon shutdown"
+          @server.shutdown
+          change_logger logger: :cli
+          splash_exit case: :quiet_exit
+        end
+
+        private
+
+        #prepare main daemon subscriber
+        def init_daemon_subscriber
+          hostname = Socket.gethostname
+          transport = get_default_subscriber queue: "splash.#{hostname}.input"
+          if transport.class == Hash and transport.include? :case then
+            splash_exit transport
+          end
+          transport.subscribe(:block => true) do |delivery_info, properties, body|
+            content = YAML::load(body)
+            session = get_session
+            content[:session] = session
+            if VERBS.include? content[:verb]
+              @log.receive "Valid remote order, verb : #{content[:verb].to_s}", session
+              res = self.send content[:verb], content
+              get_default_client.publish queue: content[:return_to], message: res.to_yaml
+              @log.send "Result to #{content[:return_to]}.", session
+            else
+              @log.receive "INVALID remote order, verb : #{content[:verb].to_s}", session
+              get_default_client.publish queue: content[:return_to], message: "Unkown verb #{content[:verb]}".to_yaml
+            end
+          end
+        end
+
+        #prepare logs monitoring  sheduling
+        def init_logs_monitoring_scheduling
           if @config.logs.empty? then
             @log.item "No logs to monitor"
           else
@@ -64,7 +113,10 @@ module Splash
               end
             end
           end
+        end
 
+        #prepare process monitoring  sheduling
+        def init_process_monitoring_scheduling
           if @config.processes.empty? then
             @log.item "No processes to monitor"
           else
@@ -83,49 +135,24 @@ module Splash
               end
             end
           end
+        end
 
 
+        #prepare metrics sheduling
+        def init_metrics_scheduling
           sched,value = @config.daemon_metrics_scheduling.flatten
           @log.item "Initializing Splash metrics notifications."
           @server.send sched,value do
             begin
+              @log.trigger "Splash Metrics monitoring for Scheduling : #{sched.to_s} #{value.to_s}", @session
               @metric_manager.notify
             rescue Errno::ECONNREFUSED
               @log.error "PushGateway seems to be done, please start it."
             end
           end
-
-          hostname = Socket.gethostname
-          transport = get_default_subscriber queue: "splash.#{hostname}.input"
-          if transport.class == Hash and transport.include? :case then
-            splash_exit transport
-          end
-          transport.subscribe(:block => true) do |delivery_info, properties, body|
-            content = YAML::load(body)
-            session = get_session
-            content[:session] = session
-            if VERBS.include? content[:verb]
-              @log.receive "Valid remote order, verb : #{content[:verb].to_s}", session
-              res = self.send content[:verb], content
-              get_default_client.publish queue: content[:return_to], message: res.to_yaml
-              @log.send "Result to #{content[:return_to]}.", session
-            else
-              @log.receive "INVALID remote order, verb : #{content[:verb].to_s}", session
-              get_default_client.publish queue: content[:return_to], message: "Unkown verb #{content[:verb]}".to_yaml
-            end
-          end
         end
 
-        # Stop the Splash daemon gracefully
-        # @return [hash] Exiter Case :quiet_exit
-        def terminate
-          @log.info "Splash daemon shutdown"
-          @server.shutdown
-          change_logger logger: :cli
-          splash_exit case: :quiet_exit
-        end
 
-        private
         # prepare commands Scheduling
         def init_commands_scheduling
           config = get_config.commands
@@ -139,7 +166,6 @@ module Splash
               execute command: command.to_s, session: session
             end
           end
-
         end
 
 
@@ -156,9 +182,25 @@ module Splash
               run_seq name: sequence.to_s, session: session
             end
           end
-
         end
 
+        # reset the orchestrator
+        # @return [Hash] Exiter case
+        def reset_orchestrator
+          @server.shutdown
+          @server = Rufus::Scheduler::new
+          @server.extend SchedulerHooks
+          @config = rehash_config
+          @log.info "Splash Orchestrator re-hashing :"
+          if @scheduling then
+            @log.item "Re-Initializing Sequences & commands Scheduling."
+            init_commands_scheduling
+            init_sequences_scheduling
+          end
+          init_logs_monitoring_scheduling
+          init_process_monitoring_scheduling
+          init_metrics_scheduling
+        end
 
 
         # execute_command verb : execute command specified in payload
@@ -174,9 +216,7 @@ module Splash
             return command.call_and_notify trace: true, notify: true, callback: true, session: options[:session]
           end
         end
-
       end
-
     end
   end
 end

data/lib/splash/daemon/orchestrator/grammar.rb

@@ -16,7 +16,7 @@ module Splash
         include Splash::Loggers
 
         # list of known verbs for Splash orchestrator
-        VERBS=[:ping,:list_commands,:execute_command,:ack_command, :shutdown]
+        VERBS=[:ping,:list_commands,:execute_command,:ack_command, :shutdown, :get_jobs, :reset]
 
         # shutdown verb : stop the Splash daemon gracefully
         # @param [Hash] content message content Hash Structure, ignored
@@ -46,6 +46,21 @@ module Splash
           return execute command: content[:payload][:name], ack: true
         end
 
+
+        # get_jobs verb : return list of scheduled jobs for internal scheduler
+        # @param [Hash] content message content Hash Structure, ignored
+        # @return [String] YAML dataset
+        def get_jobs(content)
+          return @server.jobs.to_yaml
+        end
+
+        # reset verb : reset the internal scheduler
+        # @param [Hash] content message content Hash Structure, ignored
+        # @return [String] "Scheduler reset" static
+        def reset(content)
+          return "Scheduler reset" if reset_orchestrator
+        end
+
         # execute_command verb : execute command specified in payload
         # @param [Hash] content message content Hash Structure, include  mandatory payload[:name]
         # @return [Hash] Exiter case

data/lib/splash/dependencies.rb

@@ -17,8 +17,12 @@ module Splash
 
 
 
+
+
     # Rubygems
     begin
+      require 'net/ssh'
+      require 'net/scp'
       require 'prometheus/client'
       require 'prometheus/client/push'
       require 'thor'
@@ -35,6 +39,7 @@ module Splash
       require 'rest-client'
       require 'kramdown'
       require 'rack/reverse_proxy'
+      require 'tty-table'
 
 
     rescue Gem::GemNotFoundException
@@ -57,9 +62,9 @@ module Splash
 
     require 'splash/commands'
     require 'splash/sequences'
-
     require 'splash/logs'
     require 'splash/processes'
+    require 'splash/transfers'
 
     require 'splash/daemon'
     require 'splash/webadmin'

data/lib/splash/exiter.rb

@@ -60,7 +60,7 @@ module Splash
 
     def splash_return(options = {})
 
-      data = EXIT_MAP[options[:case]]
+      data = EXIT_MAP[options[:case]].clone
       data[:status] = (data[:code]>0)? :failure : :success
       data[:more] = options[:more] if options[:more]
       return data

data/lib/splash/helpers.rb

@@ -252,7 +252,7 @@ module Splash
     # check folder
     # @return [Array] of Symbol with error type : [:inexistant,:mode,:owner,:group]
     # @param [Hash] options
-    # @option options [String] :path folder path (relative or absolute)
+    # @option options [String] :name folder path (relative or absolute)
     # @option options [String] :mode String for OCTAL rights like "644", optionnal
     # @option options [String] :owner file owner for folder, optionnal
     # @option options [String] :group  file group for folder, optionnal
@@ -336,12 +336,11 @@ module Splash
     end
     #!@endgroup
 
-
     def format_response(data, format)
       response = case format
                  when 'application/json' then JSON.pretty_generate(data)
                  when 'text/x-yaml' then data.to_yaml
-                 else JSON.pretty_generate(data)
+                 else data.to_yaml
                  end
       return response
     end
@@ -355,6 +354,16 @@ module Splash
        return result[extension]
     end
 
+    # check if unicode must be used with term ENV
+    # @return [Boolean]
+    def check_unicode_term
+      return false unless ENV.include? "TERM"
+      if ENV.values_at("LC_ALL","LC_CTYPE","LANG").compact.first.include?("UTF-8") and ENV.values_at('TERM').first.include? "xterm" then
+        return true
+      else
+        return false
+      end
+    end
 
   end
 end

data/lib/splash/loggers/cli.rb

@@ -10,6 +10,7 @@ module Splash
     class Cli < Splash::Loggers::LoggerTemplate
 
       include Splash::Config
+      include Splash::Helpers
 
       # mapping of UTf-8 emoji for log levels or alias
       EMOJI =  { :unknown => "\u{1F4A5}",
@@ -82,16 +83,7 @@ module Splash
         get_config.loggers[:cli][:color] = status
       end
 
-      # check if unicode must be used with term ENV
-      # @return [Boolean]
-      def check_unicode_term
-        return false unless ENV.include? "TERM"
-        if ENV.values_at("LC_ALL","LC_CTYPE","LANG").compact.first.include?("UTF-8") and ENV.values_at('TERM').first.include? "xterm" then
-          return true
-        else
-          return false
-        end
-      end
+
 
     end
 

data/lib/splash/logs.rb

@@ -6,6 +6,83 @@ module Splash
   # Logs namespace
   module Logs
 
+    class LogsNotifier
+
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_missing = Prometheus::Client::Gauge.new(:logmissing, docstring: 'SPLASH metric log missing', labels: [:log ])
+      @@metric_count = Prometheus::Client::Gauge.new(:logerrors, docstring: 'SPLASH metric log error', labels: [:log ])
+      @@metric_lines = Prometheus::Client::Gauge.new(:loglines, docstring: 'SPLASH metric log lines numbers', labels: [:log ])
+      @@registry.register(@@metric_count)
+      @@registry.register(@@metric_missing)
+      @@registry.register(@@metric_lines)
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = options[:name]
+        @missing = options[:missing]
+        @lines = options[:lines]
+        @errors = options[:errors]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+        unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_missing.set(@missing, labels: { log: @name })
+        @@metric_count.set(@errors, labels: { log: @name })
+        @@metric_lines.set(@lines, labels: { log: @name })
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new("Splash", hostname, @url).add(@@registry)
+      end
+
+    end
+
+    class LogsRecords
+      include Splash::Backends
+      include Splash::Constants
+      def initialize(name)
+        @name = name
+        @backend = get_backend :logs_trace
+      end
+
+      def clear
+        @backend.del({:key => @name}) if  @backend.exist?({key: @name})
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+    end
+
+
+
     # Log scanner and notifier
     class LogScanner
       include Splash::Constants
@@ -17,13 +94,7 @@ module Splash
       def initialize
         @logs_target = Marshal.load(Marshal.dump(get_config.logs))
         @config = get_config
-        @registry = Prometheus::Client::Registry::new
-        @metric_count = Prometheus::Client::Gauge.new(:logerrors, docstring: 'SPLASH metric log error', labels: [:log ])
-        @metric_missing = Prometheus::Client::Gauge.new(:logmissing, docstring: 'SPLASH metric log missing', labels: [:log ])
-        @metric_lines = Prometheus::Client::Gauge.new(:loglines, docstring: 'SPLASH metric log lines numbers', labels: [:log ])
-        @registry.register(@metric_count)
-        @registry.register(@metric_missing)
-        @registry.register(@metric_lines)
+
       end
 
 
@@ -62,17 +133,24 @@ module Splash
         session = (options[:session]) ? options[:session] : log.get_session
         log.info "Sending metrics to Prometheus Pushgateway", session
         @logs_target.each do |item|
-          missing = (item[:status] == :missing)? 1 : 0
-          log.item "Sending metrics for #{item[:log]}", session
-          @metric_count.set(item[:count], labels: { log: item[:log] })
-          @metric_missing.set(missing, labels: { log: item[:log] })
+          logsrec = LogsRecords::new item[:label]
+          errors = (item[:count])? item[:count] : 0
           lines = (item[:lines])? item[:lines] : 0
-          @metric_lines.set(lines, labels: { log: item[:log] })
+          missing = (item[:status] == :missing)? 1 : 0
+          file = item[:log]
+          logsrec.purge(item[:retention])
+          logsrec.add_record :status => item[:status],
+                           :errors => errors,
+                           :lines => lines,
+                           :file => file
+
+          logsmonitor = LogsNotifier::new({name: item[:label], missing: missing, file: file, errors: errors, lines: lines})
+          if logsmonitor.notify then
+            log.ok "Sending metrics for log #{file} to Prometheus Pushgateway", session
+          else
+            log.ko "Failed to send metrics for log #{file} to Prometheus Pushgateway", session
+          end
         end
-        hostname = Socket.gethostname
-        url = @config.prometheus_pushgateway_url
-        Prometheus::Client::Push.new('Splash',hostname, url).add(@registry)
-        log.ok "Sending to Prometheus PushGateway done.", session
         return {:case => :quiet_exit }
       end