prometheus-splash 0.6.1 → 0.8.3

data/lib/splash/config.rb

@@ -11,56 +11,124 @@ module Splash
     include Splash::ConfigUtilities
 
 
+    class ConfigLinter
+      def initialize
+        @lints_present = {:logs => [:label, :log, :pattern ]}
+        @lints_types = {:logs => {:label => Symbol, :log => String, :pattern => String, :retention => Hash}}
+      end
+
+      def verify(options ={})
+        status = :success
+        missings = []
+        type_errors = []
+        useless = []
+        options[:record].each do |key,value|
+          useless.push key unless @lints_present[options[:type]].include? key or @lints_types[options[:type]].keys.include? key
+          type_errors.push key if @lints_types[options[:type]][key] != value.class and (@lints_present[options[:type]].include? key or @lints_types[options[:type]].keys.include? key)
+        end
+        @lints_present[options[:type]].each do |item|
+          missings.push item unless options[:record].keys.include? item
+        end
+
+        status = :failure if (missings.count > 0) or (type_errors.count > 0)
+        return {:missings => missings, :type_errors => type_errors, :useless => useless, :status => status}
+      end
+
+    end
+
     # Class to manage configuration in Splash from Splash::Constants override by Yaml CONFIG
     class Configuration < Hash
       include Splash::Constants
 
+      attr_accessor :config_from_file
+
       # constructor : read config file and map against Constants
       def initialize(config_file=CONFIG_FILE)
-        config_from_file = readconf config_file
+        @config_file = config_file
+        hash_config_to_default
+
+      end
+
+
+      def hash_config_to_default
+        @config_from_file = readconf @config_file
         self[:version] = VERSION
         self[:author] = "#{AUTHOR} <#{EMAIL}>"
         self[:copyright] = "#{COPYRIGHT} #{LICENSE}"
 
-        self[:prometheus_url] = (config_from_file[:prometheus][:url])? config_from_file[:prometheus][:url] : PROMETHEUS_URL
-        self[:prometheus_pushgateway_url] = (config_from_file[:prometheus][:pushgateway])? config_from_file[:prometheus][:pushgateway] : PROMETHEUS_PUSHGATEWAY_URL
+        self[:prometheus_url] = (@config_from_file[:prometheus][:url])? @config_from_file[:prometheus][:url] : PROMETHEUS_URL
+        self[:prometheus_pushgateway_url] = (@config_from_file[:prometheus][:pushgateway])? @config_from_file[:prometheus][:pushgateway] : PROMETHEUS_PUSHGATEWAY_URL
+        self[:prometheus_alertmanager_url] = (@config_from_file[:prometheus][:alertmanager])? @config_from_file[:prometheus][:alertmanager] : PROMETHEUS_ALERTMANAGER_URL
 
-        self[:daemon_process_name] = (config_from_file[:daemon][:process_name])? config_from_file[:daemon][:process_name] : DAEMON_PROCESS_NAME
-        self[:daemon_logmon_scheduling] = (config_from_file[:daemon][:logmon_scheduling])? config_from_file[:daemon][:logmon_scheduling] : DAEMON_LOGMON_SCHEDULING
-        self[:daemon_metrics_scheduling] = (config_from_file[:daemon][:metrics_scheduling])? config_from_file[:daemon][:metrics_scheduling] : DAEMON_METRICS_SCHEDULING
-        self[:daemon_procmon_scheduling] = (config_from_file[:daemon][:procmon_scheduling])? config_from_file[:daemon][:procmon_scheduling] : DAEMON_PROCMON_SCHEDULING
+        self[:daemon_process_name] = (@config_from_file[:daemon][:process_name])? @config_from_file[:daemon][:process_name] : DAEMON_PROCESS_NAME
+        self[:daemon_logmon_scheduling] = (@config_from_file[:daemon][:logmon_scheduling])? @config_from_file[:daemon][:logmon_scheduling] : DAEMON_LOGMON_SCHEDULING
+        self[:daemon_metrics_scheduling] = (@config_from_file[:daemon][:metrics_scheduling])? @config_from_file[:daemon][:metrics_scheduling] : DAEMON_METRICS_SCHEDULING
+        self[:daemon_procmon_scheduling] = (@config_from_file[:daemon][:procmon_scheduling])? @config_from_file[:daemon][:procmon_scheduling] : DAEMON_PROCMON_SCHEDULING
+        self[:daemon_pid_file] = (@config_from_file[:daemon][:files][:pid_file])? @config_from_file[:daemon][:files][:pid_file] : DAEMON_PID_FILE
+        self[:daemon_stdout_trace] = (@config_from_file[:daemon][:files][:stdout_trace])? @config_from_file[:daemon][:files][:stdout_trace] : DAEMON_STDOUT_TRACE
+        self[:daemon_stderr_trace] = (@config_from_file[:daemon][:files][:stderr_trace])? @config_from_file[:daemon][:files][:stderr_trace] : DAEMON_STDERR_TRACE
 
 
-        self[:webadmin_port] = (config_from_file[:webadmin][:port])? config_from_file[:webadmin][:port] : WEBADMIN_PORT
-        self[:webadmin_ip] = (config_from_file[:webadmin][:ip])? config_from_file[:webadmin][:ip] : WEBADMIN_IP
-        self[:webadmin_proxy] = (config_from_file[:webadmin][:proxy])? config_from_file[:webadmin][:proxy] : WEBADMIN_PROXY
-        self[:webadmin_process_name] = (config_from_file[:webadmin][:process_name])? config_from_file[:webadmin][:process_name] : WEBADMIN_PROCESS_NAME
-        self[:webadmin_pid_file] = (config_from_file[:webadmin][:files][:pid_file])? config_from_file[:webadmin][:files][:pid_file] : WEBADMIN_PID_FILE
-        self[:webadmin_stdout_trace] = (config_from_file[:webadmin][:files][:stdout_trace])? config_from_file[:webadmin][:files][:stdout_trace] : WEBADMIN_STDOUT_TRACE
-        self[:webadmin_stderr_trace] = (config_from_file[:webadmin][:files][:stderr_trace])? config_from_file[:webadmin][:files][:stderr_trace] : WEBADMIN_STDERR_TRACE
+        self[:webadmin_port] = (@config_from_file[:webadmin][:port])? @config_from_file[:webadmin][:port] : WEBADMIN_PORT
+        self[:webadmin_ip] = (@config_from_file[:webadmin][:ip])? @config_from_file[:webadmin][:ip] : WEBADMIN_IP
+        self[:webadmin_proxy] = (@config_from_file[:webadmin][:proxy])? @config_from_file[:webadmin][:proxy] : WEBADMIN_PROXY
+        self[:webadmin_process_name] = (@config_from_file[:webadmin][:process_name])? @config_from_file[:webadmin][:process_name] : WEBADMIN_PROCESS_NAME
+        self[:webadmin_pid_file] = (@config_from_file[:webadmin][:files][:pid_file])? @config_from_file[:webadmin][:files][:pid_file] : WEBADMIN_PID_FILE
+        self[:webadmin_stdout_trace] = (@config_from_file[:webadmin][:files][:stdout_trace])? @config_from_file[:webadmin][:files][:stdout_trace] : WEBADMIN_STDOUT_TRACE
+        self[:webadmin_stderr_trace] = (@config_from_file[:webadmin][:files][:stderr_trace])? @config_from_file[:webadmin][:files][:stderr_trace] : WEBADMIN_STDERR_TRACE
 
 
-        self[:pid_path] = (config_from_file[:daemon][:paths][:pid_path])? config_from_file[:daemon][:paths][:pid_path] : PID_PATH
-        self[:trace_path] = (config_from_file[:daemon][:paths][:trace_path])? config_from_file[:daemon][:paths][:trace_path] : TRACE_PATH
+        self[:pid_path] = (@config_from_file[:paths][:pid_path])? @config_from_file[:paths][:pid_path] : PID_PATH
+        self[:trace_path] = (@config_from_file[:paths][:trace_path])? @config_from_file[:paths][:trace_path] : TRACE_PATH
 
 
         self[:execution_template_tokens] = EXECUTION_TEMPLATE_TOKENS_LIST
-        self[:execution_template_path] = (config_from_file[:templates][:execution][:path])? config_from_file[:templates][:execution][:path] : EXECUTION_TEMPLATE
-        self[:pid_file] = (config_from_file[:daemon][:files][:pid_file])? config_from_file[:daemon][:files][:pid_file] : DAEMON_PID_FILE
-        self[:stdout_trace] = (config_from_file[:daemon][:files][:stdout_trace])? config_from_file[:daemon][:files][:stdout_trace] : DAEMON_STDOUT_TRACE
-        self[:stderr_trace] = (config_from_file[:daemon][:files][:stderr_trace])? config_from_file[:daemon][:files][:stderr_trace] : DAEMON_STDERR_TRACE
+        self[:execution_template_path] = (@config_from_file[:templates][:execution][:path])? @config_from_file[:templates][:execution][:path] : EXECUTION_TEMPLATE
 
-        self[:transports] = {} ; self[:transports].merge! TRANSPORTS_STRUCT ; self[:transports].merge! config_from_file[:transports] if config_from_file[:transports]
-        self[:backends] = {} ; self[:backends].merge! BACKENDS_STRUCT ; self[:backends].merge! config_from_file[:backends] if config_from_file[:backends]
-        self[:loggers] = {} ; self[:loggers].merge! LOGGERS_STRUCT ; self[:loggers].merge! config_from_file[:loggers] if config_from_file[:loggers]
+        self[:transports] = {} ; self[:transports].merge! TRANSPORTS_STRUCT ; self[:transports].merge! @config_from_file[:transports] if @config_from_file[:transports]
+        self[:backends] = {} ; self[:backends].merge! BACKENDS_STRUCT ; self[:backends].merge! @config_from_file[:backends] if @config_from_file[:backends]
+        self[:loggers] = {} ; self[:loggers].merge! LOGGERS_STRUCT ; self[:loggers].merge! @config_from_file[:loggers] if @config_from_file[:loggers]
 
-        self[:processes] = (config_from_file[:processes])? config_from_file[:processes] : {}
-        self[:logs] = (config_from_file[:logs])? config_from_file[:logs] : {}
-        self[:commands] = (config_from_file[:commands])? config_from_file[:commands] : {}
-        self[:sequences] = (config_from_file[:sequences])? config_from_file[:sequences] : {}
+        self[:processes] = (@config_from_file[:processes])? @config_from_file[:processes] : {}
+        self[:logs] = (@config_from_file[:logs])? @config_from_file[:logs] : {}
+        self[:commands] = (@config_from_file[:commands])? @config_from_file[:commands] : {}
+        self[:sequences] = (@config_from_file[:sequences])? @config_from_file[:sequences] : {}
+        self[:transfers] = (@config_from_file[:transfers])? @config_from_file[:transfers] : {}
+      end
 
+
+      def add_log(options = {})
+        @config_from_file = readconf @config_file
+        res = ConfigLinter::new.verify(options)
+        if res[:status] == :success then
+          if @config_from_file[:logs].select{|item| item[:label] == options[:record][:label]}.count > 0 then
+            return {:status => :already_exist}
+          else
+            res[:useless].each {|item| options[:record].delete item} if options[:clean]
+            @config_from_file[:logs].push options[:record]
+            writeconf
+            hash_config_to_default
+            return {:status => :success}
+          end
+        else
+          return res
+        end
       end
 
+
+      def delete_log(options = {})
+        @config_from_file = readconf @config_file
+        unless @config_from_file[:logs].select{|item| item[:label] == options[:label]}.count > 0 then
+          return {:status => :not_found}
+        else
+          @config_from_file[:logs].delete_if {|value| options[:label] == value[:label] }
+          writeconf
+          hash_config_to_default
+          return {:status => :success}
+        end
+      end
+
+
       # @!group accessors on configurations Items
 
       # getter for full Config Hash
@@ -87,6 +155,12 @@ module Splash
         return self[:transports]
       end
 
+      # getter for transfers Hash Config sample
+      # @return [Hash]
+      def transfers
+        return self[:transfers]
+      end
+
       # getter for daemon_logmon_scheduling Hash Config sample
       # @return [Hash]
       def daemon_logmon_scheduling
@@ -105,6 +179,31 @@ module Splash
         return self[:daemon_metrics_scheduling]
       end
 
+      # getter for daemon_process_name Config sample
+      # @return [String]
+      def daemon_process_name
+        return self[:daemon_process_name]
+      end
+
+
+      # getter for daemon_full_pid_path Config sample
+      # @return [String]
+      def daemon_full_pid_path
+        return "#{self[:pid_path]}/#{self[:daemon_pid_file]}"
+      end
+
+      # getter for daemon_full_stdout_trace_path Config sample
+      # @return [String]
+      def daemon_full_stdout_trace_path
+        return "#{self[:trace_path]}/#{self[:daemon_stdout_trace]}"
+      end
+
+      # getter for daemon_full_stderr_trace_path Config sample
+      # @return [String]
+      def daemon_full_stderr_trace_path
+        return "#{self[:trace_path]}/#{self[:daemon_stderr_trace]}"
+      end
+
       # getter for execution_template_path Hash Config sample
       # @return [String]
       def execution_template_path
@@ -215,36 +314,17 @@ module Splash
         return self[:prometheus_pushgateway_url]
       end
 
-      # getter for prometheus_url Config sample
-      # @return [String]
-      def prometheus_url
-        return self[:prometheus_url]
-      end
-
-
-      # getter for daemon_process_name Config sample
-      # @return [String]
-      def daemon_process_name
-        return self[:daemon_process_name]
-      end
-
-
-      # getter for daemon_full_pid_path Config sample
+      # getter for prometheus_alertmanager_url Config sample
       # @return [String]
-      def daemon_full_pid_path
-        return "#{self[:pid_path]}/#{self[:pid_file]}"
+      def prometheus_alertmanager_url
+        return self[:prometheus_alertmanager_url]
       end
 
-      # getter for daemon_full_stdout_trace_path Config sample
-      # @return [String]
-      def daemon_full_stdout_trace_path
-        return "#{self[:trace_path]}/#{self[:stdout_trace]}"
-      end
 
-      # getter for daemon_full_stderr_trace_path Config sample
+      # getter for prometheus_url Config sample
       # @return [String]
-      def daemon_full_stderr_trace_path
-        return "#{self[:trace_path]}/#{self[:stderr_trace]}"
+      def prometheus_url
+        return self[:prometheus_url]
       end
 
       # @!endgroup
@@ -258,6 +338,16 @@ module Splash
         return YAML.load_file(file)[:splash]
       end
 
+      # write config to file from @config_from_file
+      # @param [String] file default from CONFIG_FILE
+      # @return [Bool] if ok
+      def writeconf(file = CONFIG_FILE)
+        File.open(file,"w") do |f|
+          data = {}
+          data[:splash] = @config_from_file
+          f.write(data.to_yaml)
+        end
+      end
 
     end
 

data/lib/splash/constants.rb

@@ -7,8 +7,7 @@ module Splash
   module Constants
 
     # Current splash version
-    VERSION = "0.6.1"
-
+    VERSION = "0.8.3"
     # the path to th config file, not overridable by config
     CONFIG_FILE = "/etc/splash.yml"
     # the default execution trace_path if backend file
@@ -43,7 +42,10 @@ module Splash
     LICENSE="BSD-2-Clause"
 
     # the default prometheus pushgateway URL
-    PROMETHEUS_PUSHGATEWAY_URL = 'http://localhost:9090/'
+    PROMETHEUS_PUSHGATEWAY_URL = 'http://localhost:9091/'
+
+    # the default prometheus Alertmanager URL
+    PROMETHEUS_ALERTMANAGER_URL = 'http://localhost:9092/'
 
     # the default prometheus URL
     PROMETHEUS_URL = "http://localhost:9090/"
@@ -84,6 +86,8 @@ module Splash
     # the default sdterr trace file
     WEBADMIN_STDERR_TRACE="stderr.txt"
 
+    # default retention for trace
+    DEFAULT_RETENTION=1
 
   end
 end

data/lib/splash/daemon/metrics.rb

@@ -17,8 +17,8 @@ module Splash
 
       # metrics manager factory
       # @return [Splash::Daemon::Metrics::Manager]
-      def get_metrics_manager
-        return @@manager ||= Manager::new
+      def get_metrics_manager(session)
+        return @@manager ||= Manager::new(:session => session)
       end
 
       # Metrics Manager (collect and sending to Prometheus)
@@ -32,7 +32,8 @@ module Splash
         attr_reader :monitoring_processes_count
 
         # Constructor prepare prometheus-client, defined metrics and init attributes
-        def initialize
+        def initialize(options ={})
+          @session = options[:session]
           @config = get_config
           @starttime = Time.now
           @execution_count = 0
@@ -75,12 +76,11 @@ module Splash
         # @return [Hash] Exiter case ( :service_dependence_missing , :quiet_exit)
         def notify
           log = get_logger
-          session  = get_session
           unless verify_service url: @config.prometheus_pushgateway_url then
             return  { :case => :service_dependence_missing, :more => "Prometheus Notification not send." }
           end
 
-          log.debug "Sending Splash self metrics to PushGateway." , session
+          log.ok "Sending Splash self metrics to PushGateway." , @session
           @metric_uptime.set uptime
           @metric_execution.set execution_count
           @metric_logs_monitoring.set monitoring_logs_count
@@ -89,7 +89,7 @@ module Splash
           hostname = Socket.gethostname
           url = @config.prometheus_pushgateway_url
           Prometheus::Client::Push.new('Splash',hostname, url).add(@registry)
-          log.debug "Sending to Prometheus PushGateway done.", session
+          log.debug "Sending to Prometheus PushGateway done.", @session
           return {:case => :quiet_exit }
         end
 

data/lib/splash/daemon/orchestrator.rb

@@ -32,20 +32,69 @@ module Splash
         def initialize(options = {})
           @log = get_logger
           self.extend Splash::Daemon::Metrics
-          @metric_manager = get_metrics_manager
+          @session = get_session
+          @metric_manager = get_metrics_manager(@session)
           $stdout.sync = true
           $stderr.sync = true
           @server  = Rufus::Scheduler::new
           @server.extend SchedulerHooks
           @config = get_config
+          @scheduling = options[:scheduling]
 
           @log.info "Splash Orchestrator starting :"
-          if options[:scheduling] then
-            @log.item "Initializing commands Scheduling."
+          if @scheduling then
+            @log.item "Initializing Sequences & commands Scheduling."
             init_commands_scheduling
             init_sequences_scheduling
           end
 
+          init_logs_monitoring_scheduling
+          init_process_monitoring_scheduling
+          init_metrics_scheduling
+          init_daemon_subscriber
+
+        end
+
+
+
+
+
+        # Stop the Splash daemon gracefully
+        # @return [hash] Exiter Case :quiet_exit
+        def terminate
+          @log.info "Splash daemon shutdown"
+          @server.shutdown
+          change_logger logger: :cli
+          splash_exit case: :quiet_exit
+        end
+
+        private
+
+        #prepare main daemon subscriber
+        def init_daemon_subscriber
+          hostname = Socket.gethostname
+          transport = get_default_subscriber queue: "splash.#{hostname}.input"
+          if transport.class == Hash and transport.include? :case then
+            splash_exit transport
+          end
+          transport.subscribe(:block => true) do |delivery_info, properties, body|
+            content = YAML::load(body)
+            session = get_session
+            content[:session] = session
+            if VERBS.include? content[:verb]
+              @log.receive "Valid remote order, verb : #{content[:verb].to_s}", session
+              res = self.send content[:verb], content
+              get_default_client.publish queue: content[:return_to], message: res.to_yaml
+              @log.send "Result to #{content[:return_to]}.", session
+            else
+              @log.receive "INVALID remote order, verb : #{content[:verb].to_s}", session
+              get_default_client.publish queue: content[:return_to], message: "Unkown verb #{content[:verb]}".to_yaml
+            end
+          end
+        end
+
+        #prepare logs monitoring  sheduling
+        def init_logs_monitoring_scheduling
           if @config.logs.empty? then
             @log.item "No logs to monitor"
           else
@@ -64,7 +113,10 @@ module Splash
               end
             end
           end
+        end
 
+        #prepare process monitoring  sheduling
+        def init_process_monitoring_scheduling
           if @config.processes.empty? then
             @log.item "No processes to monitor"
           else
@@ -83,49 +135,24 @@ module Splash
               end
             end
           end
+        end
 
 
+        #prepare metrics sheduling
+        def init_metrics_scheduling
           sched,value = @config.daemon_metrics_scheduling.flatten
           @log.item "Initializing Splash metrics notifications."
           @server.send sched,value do
             begin
+              @log.trigger "Splash Metrics monitoring for Scheduling : #{sched.to_s} #{value.to_s}", @session
               @metric_manager.notify
             rescue Errno::ECONNREFUSED
               @log.error "PushGateway seems to be done, please start it."
             end
           end
-
-          hostname = Socket.gethostname
-          transport = get_default_subscriber queue: "splash.#{hostname}.input"
-          if transport.class == Hash and transport.include? :case then
-            splash_exit transport
-          end
-          transport.subscribe(:block => true) do |delivery_info, properties, body|
-            content = YAML::load(body)
-            session = get_session
-            content[:session] = session
-            if VERBS.include? content[:verb]
-              @log.receive "Valid remote order, verb : #{content[:verb].to_s}", session
-              res = self.send content[:verb], content
-              get_default_client.publish queue: content[:return_to], message: res.to_yaml
-              @log.send "Result to #{content[:return_to]}.", session
-            else
-              @log.receive "INVALID remote order, verb : #{content[:verb].to_s}", session
-              get_default_client.publish queue: content[:return_to], message: "Unkown verb #{content[:verb]}".to_yaml
-            end
-          end
         end
 
-        # Stop the Splash daemon gracefully
-        # @return [hash] Exiter Case :quiet_exit
-        def terminate
-          @log.info "Splash daemon shutdown"
-          @server.shutdown
-          change_logger logger: :cli
-          splash_exit case: :quiet_exit
-        end
 
-        private
         # prepare commands Scheduling
         def init_commands_scheduling
           config = get_config.commands
@@ -139,7 +166,6 @@ module Splash
               execute command: command.to_s, session: session
             end
           end
-
         end
 
 
@@ -156,9 +182,25 @@ module Splash
               run_seq name: sequence.to_s, session: session
             end
           end
-
         end
 
+        # reset the orchestrator
+        # @return [Hash] Exiter case
+        def reset_orchestrator
+          @server.shutdown
+          @server = Rufus::Scheduler::new
+          @server.extend SchedulerHooks
+          @config = rehash_config
+          @log.info "Splash Orchestrator re-hashing :"
+          if @scheduling then
+            @log.item "Re-Initializing Sequences & commands Scheduling."
+            init_commands_scheduling
+            init_sequences_scheduling
+          end
+          init_logs_monitoring_scheduling
+          init_process_monitoring_scheduling
+          init_metrics_scheduling
+        end
 
 
         # execute_command verb : execute command specified in payload
@@ -174,9 +216,7 @@ module Splash
             return command.call_and_notify trace: true, notify: true, callback: true, session: options[:session]
           end
         end
-
       end
-
     end
   end
 end