prometheus-splash 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db8cc49d7a0a733b08a9f744a2ac14b5196b69fc229fcd77c9c635dd9e6c61e8
-  data.tar.gz: c839f4d20b88880a458e92de5b2d10f7d58788645fb7b03c0c8e63d403f5ec78
+  metadata.gz: 60d3aa2e635dbea8022a1804e9886e495648ce777db2293442ae3a16bae7e25a
+  data.tar.gz: 14912be5e4171daae1ee7706f98ac7974c43a96d56e7daa58a1e9a40d46b60ee
 SHA512:
-  metadata.gz: 6d1314705d22d1da16a62b265c0a9967b3fa2cd049db1dd25af3631b144b7872da48a84d1c62d36cc9fa650cb398ab4beaba9523d46f79bde52f3b6382b586e3
-  data.tar.gz: d8d637a2ea309860d988952f87c63b906c1b4c350dfa5327f12f84ddc0cd352c3ca4b7112ac2f180b890d2acc3553db2caaa0abda3fc1a77a0eccf88d6c44f85
+  metadata.gz: d66a48e63fd8940c6343113511e1ad91531037013a042a79348a755de74e6447a791854fa2ec82461c38695370402e30790ac7660fd75c0cc82c0f5ef788701d
+  data.tar.gz: 9d59657c45219e31af8a6c85fc137925ee03ccecc7a98e62848ba28540ec2e39df674fcd391b51334c4bc1252de487a5a55110a286707fee9a38e54bf85a3cfc

data/CHANGELOG.md

@@ -150,3 +150,8 @@
 * orchestrator rebuild
 * reshash config and reset + grammar and Cli
 * refacto config
+
+## V0.8.1
+
+### FEATURE
+* #55 historisation for reports for all

data/README.md

@@ -9,7 +9,7 @@ SPLASH is **Supervision with Prometheus of Logs and Asynchronous tasks orchestra
 * Web : http://www.ultragreen.net
 * Github : https://github.com/Ultragreen/prometheus-splash
 * Rubygems : https://rubygems.org/gems/prometheus-splash
-* DOC yardoc : https://www.rubydoc.info/gems/prometheus-splash/0.8.0
+* DOC yardoc : https://www.rubydoc.info/gems/prometheus-splash/0.8.1
 
 Prometheus Logs and Batchs supervision over PushGateway
 

data/config/splash.yml

@@ -26,9 +26,15 @@
 #        :port: 6379
 #        #:auth: "mykey"
 #        :base: 1
-      :transferts_trace:
+      :transfers_trace:
           :type: :file
-          :path: /var/run/splash/transferts
+          :path: /var/run/splash/transfers
+      :logs_trace:
+          :type: :file
+          :path: /var/run/splash/logs
+      :process_trace:
+          :type: :file
+          :path: /var/run/splash/process
 
   :transports:
     :active: :rabbitmq
@@ -122,15 +128,21 @@
     - :label: :log_app_1
       :log: /tmp/test
       :pattern: ERROR
+      :retention:
+        :hours: 5
     - :label: :log_app_2
       :log: /tmp/test2
       :pattern: ERROR
+      :retention:
+        :hours: 5
 
 ### configuration of monitored processes
   :processes:
     - :process: cron
       :patterns:
         - cron
+      :retention:
+          :hours: 5
 
 
 ###

data/lib/splash/cli/commands.rb

@@ -93,6 +93,9 @@ module CLISplash
     option :at, :type => :string
     option :in, :type => :string
     def schedule(name)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command scheduling"
+      end
       log = get_logger
       log.level = :fatal if options[:quiet]
       hostname = (options[:hostname])? options[:hostname] : Socket.gethostname
@@ -129,6 +132,9 @@ module CLISplash
     LONGDESC
     option :hostname, :type => :string,  :aliases => "-H"
     def treeview(command)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command treeview"
+      end
       depht = 0
       log  = get_logger
       if options[:hostname] then
@@ -179,6 +185,9 @@ module CLISplash
     option :detail, :type => :boolean,  :aliases => "-D"
     option :hostname, :type => :string,  :aliases => "-H"
     def list
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command list"
+      end
       log = get_logger
       list = {}
       if options[:hostname] then
@@ -227,6 +236,9 @@ module CLISplash
     LONGDESC
     option :hostname, :type => :string,  :aliases => "-H"
     def show(command)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command show specifications"
+      end
       log = get_logger
       list = {}
       if options[:hostname] then
@@ -265,6 +277,57 @@ module CLISplash
       end
     end
 
+
+
+    # Thor method : show commands executions  history
+    long_desc <<-LONGDESC
+    show commands executions history for LABEL\n
+    LONGDESC
+    option :table, :type => :boolean,  :aliases => "-t"
+    desc "history LABEL", "show commands executions history"
+    def history(command)
+      if is_root? then
+        log = get_logger
+        log.info "Log : #{command}#"
+        config = get_config
+        if options[:table] then
+          table = TTY::Table.new do |t|
+            t << ["Start Date","Status", "end_date", "Execution time","STDOUT empty ? ", "STDERR empty ? "]
+            t << ['','','','','','']
+            CmdRecords::new(command).get_all_records.each do |item|
+              record =item.keys.first
+              value=item[record]
+              t << [record, value[:status].to_s,
+                            value[:end_date],
+                            value[:exec_time],
+                            value[:stdout].empty?,
+                            value[:stdout].empty?]
+            end
+          end
+          if check_unicode_term  then
+            puts table.render(:unicode)
+          else
+            puts table.render(:ascii)
+          end
+
+        else
+          CmdRecords::new(command).get_all_records.each do |item|
+            record =item.keys.first
+            value=item[record]
+            log.item record
+            log.arrow "Status : #{value[:status].to_s}"
+            log.arrow "End date : #{value[:end_date]}"
+            log.arrow "Execution time : #{value[:exec_time]}"
+            log.arrow "STDOUT empty ?  : #{value[:stdout].empty?}"
+            log.arrow "STDERR empty ?  : #{value[:stderr].empty?}"
+          end
+        end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_root, :more => "Command execution history"
+      end
+    end
+
     # Thor method : getting information on the last execution of a command
     desc "lastrun COMMAND", "Show last running result for specific configured command COMMAND"
     long_desc <<-LONGDESC
@@ -273,6 +336,9 @@ module CLISplash
     LONGDESC
     option :hostname, :type => :string,  :aliases => "-H"
     def lastrun(command)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command last execution report"
+      end
       log = get_logger
       backend = get_backend :execution_trace
       redis = (backend.class == Splash::Backends::Redis)? true : false
@@ -294,7 +360,7 @@ module CLISplash
           tp = Template::new(
               list_token: get_config.execution_template_tokens,
               template_file: get_config.execution_template_path)
-          tp.map YAML::load(res)
+          tp.map YAML::load(res).last.values.first
           log.flat tp.output
         else
           log.ko "Command not already runned."
@@ -305,6 +371,58 @@ module CLISplash
       end
     end
 
+
+    # Thor method : getting information on one specific execution of a command
+    desc "onerun COMMAND", "Show running result for specific configured command COMMAND"
+    long_desc <<-LONGDESC
+    Show specific running result for specific configured command COMMAND\n
+    with --hostname <HOSTNAME>, an other Splash monitored server (only with Redis backend configured)
+    with --date <DATE>, a date format string (same as in history ouput)
+
+    LONGDESC
+    option :hostname, :type => :string,  :aliases => "-H"
+    option :date, :type => :string,  :aliases => "-D", :required => true
+    def onerun(command)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command specific execution report"
+      end
+      log = get_logger
+      backend = get_backend :execution_trace
+      redis = (backend.class == Splash::Backends::Redis)? true : false
+      if not redis and options[:hostname] then
+        splash_exit case: :specific_config_required, :more => "Redis backend is requiered for Remote execution report request"
+      end
+      splash_exit case: :not_root if not is_root? and not redis
+      list = get_config.commands.keys
+      if options[:hostname] then
+        options[:hostname] = Socket.gethostname if options[:hostname] == 'hostname'
+        list = backend.list("*", options[:hostname]).map(&:to_sym)
+      end
+      if list.include? command.to_sym then
+        log.info "Splash command #{command} previous execution report:\n"
+        req  = { :key => command}
+        req[:hostname] = options[:hostname] if options[:hostname]
+        if backend.exist? req then
+          res = backend.get req
+          tp = Template::new(
+              list_token: get_config.execution_template_tokens,
+              template_file: get_config.execution_template_path)
+          prov =  YAML::load(res).select{|key,value| key.keys.first == options[:date]}
+          if prov.empty? then
+            log.ko "Command not runned one this date or date misformatted."
+          else
+            tp.map prov.first.values.first
+            log.flat tp.output
+          end
+        else
+          log.ko "Command not already runned."
+        end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_found, :more => "Command report never runned remotly" if options[:hostname]
+      end
+    end
+
     # Thor method : getting the list of avaibles executions reports
     desc "getreportlist", "list all executions report results "
     long_desc <<-LONGDESC
@@ -320,6 +438,9 @@ module CLISplash
     option :all, :type => :boolean, :negate => false,  :aliases => "-A"
     option :detail, :type => :boolean,  :aliases => "-D"
     def getreportlist
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command execution report list"
+      end
       log = get_logger
       options[:hostname] = Socket.gethostname if options[:hostname] == 'hostname'
       if options[:hostname] and options[:all] then
@@ -355,10 +476,9 @@ module CLISplash
           req = { :key => command }
           req[:hostname] = host if options[:all]
           res = YAML::load(backend.get(req))
-          log.arrow "Status : #{res[:status]}"
-          log.arrow "Start date : #{res[:start_date]}"
-          log.arrow "End date : #{res[:end_date]}"
-          log.arrow "Execution time : #{res[:exec_time]}"
+          res.each do |record|
+            log.arrow "#{record.keys.first} : #{record[record.keys.first][:status]}"
+          end
         end
       end
       splash_exit case: :quiet_exit

data/lib/splash/cli/logs.rb

@@ -12,61 +12,73 @@ module CLISplash
     # Thor method : running Splash configured logs monitors analyse
     desc "analyse", "analyze logs defined in Splash config"
     def analyse
-      log  = get_logger
-      results = LogScanner::new
-      res = results.analyse
-      log.info "SPlash Configured log monitors :"
-      full_status = true
-      results.output.each do |result|
-        if result[:status] == :clean then
-          log.ok "Log : #{result[:log]} with label : #{result[:label]} : no errors"
-          log.item "Detected pattern : #{result[:pattern]}"
-          log.item "Nb lines = #{result[:lines]}"
-        elsif result[:status] == :missing then
-          log.ko "Log : #{result[:log]} with label : #{result[:label]} : missing !"
-          log.item "Detected pattern : #{result[:pattern]}"
-        else
-          log.ko "Log : #{result[:log]} with label : #{result[:label]} : #{result[:count]} errors"
-          log.item "Detected pattern : #{result[:pattern]}"
-          log.item "Nb lines = #{result[:lines]}"
-        end
+      if is_root? then
+        log  = get_logger
+        results = LogScanner::new
+        res = results.analyse
+        log.info "SPlash Configured log monitors :"
+        full_status = true
+        results.output.each do |result|
+          if result[:status] == :clean then
+            log.ok "Log : #{result[:log]} with label : #{result[:label]} : no errors"
+            log.item "Detected pattern : #{result[:pattern]}"
+            log.item "Nb lines = #{result[:lines]}"
+          elsif result[:status] == :missing then
+            log.ko "Log : #{result[:log]} with label : #{result[:label]} : missing !"
+            log.item "Detected pattern : #{result[:pattern]}"
+          else
+            log.ko "Log : #{result[:log]} with label : #{result[:label]} : #{result[:count]} errors"
+            log.item "Detected pattern : #{result[:pattern]}"
+            log.item "Nb lines = #{result[:lines]}"
+          end
 
-        full_status = false unless result[:status] == :clean
-      end
+          full_status = false unless result[:status] == :clean
+        end
 
-      if full_status then
-        log.ok "Global status : no error found"
+        if full_status then
+          log.ok "Global status : no error found"
+        else
+          log.error "Global status : some error found"
+        end
+        splash_exit case: :quiet_exit
       else
-        log.error "Global status : some error found"
+        splash_exit case: :not_root, :more => "Log analysis"
       end
-      splash_exit case: :quiet_exit
     end
 
 
     # Thor method : running Splash configured logs monitors analyse and sending to Prometheus Pushgateway
     desc "monitor", "monitor logs defined in Splash config"
     def monitor
-      log = get_logger
-      log.level = :fatal if options[:quiet]
-      result = LogScanner::new
-      result.analyse
-      splash_exit result.notify
+      if is_root? then
+        log = get_logger
+        log.level = :fatal if options[:quiet]
+        result = LogScanner::new
+        result.analyse
+        splash_exit result.notify
+      else
+        splash_exit case: :not_root, :more => "Log analysis"
+      end
 
     end
 
     # Thor method : display a specific Splash configured log monitor
     desc "show LOG", "show Splash configured log monitoring for LOG"
     def show(logrecord)
-      log = get_logger
-      log_record_set = get_config.logs.select{|item| item[:log] == logrecord or item[:label] == logrecord.to_sym}
-      unless log_record_set.empty? then
-        record = log_record_set.first
-        log.info "Splash log monitor : #{record[:log]}"
-        log.item "pattern : /#{record[:pattern]}/"
-        log.item "label : #{record[:label]}"
-        splash_exit case: :quiet_exit
+      if is_root? then
+        log = get_logger
+        log_record_set = get_config.logs.select{|item| item[:log] == logrecord or item[:label] == logrecord.to_sym}
+        unless log_record_set.empty? then
+          record = log_record_set.first
+          log.info "Splash log monitor : #{record[:log]}"
+          log.item "pattern : /#{record[:pattern]}/"
+          log.item "label : #{record[:label]}"
+          splash_exit case: :quiet_exit
+        else
+          splash_exit case: :not_found, :more => "log not configured"
+        end
       else
-        splash_exit case: :not_found, :more => "log not configured"
+        splash_exit case: :not_root, :more => "Log analysis"
       end
     end
 
@@ -78,17 +90,66 @@ module CLISplash
     LONGDESC
     option :detail, :type => :boolean,  :aliases => "-D"
     def list
-      log = get_logger
-      log.info "Splash configured log monitoring :"
-      log_record_set = get_config.logs
-      log.ko 'No configured commands found' if log_record_set.empty?
-      log_record_set.each do |record|
-        log.item "log monitor : #{record[:log]} label : #{record[:label]}"
-        if options[:detail] then
-          log.arrow "pattern : /#{record[:pattern]}/"
+      if is_root? then
+        log = get_logger
+        log.info "Splash configured log monitoring :"
+        log_record_set = get_config.logs
+        log.ko 'No configured commands found' if log_record_set.empty?
+        log_record_set.each do |record|
+          log.item "log monitor : #{record[:log]} label : #{record[:label]}"
+          if options[:detail] then
+            log.arrow "pattern : /#{record[:pattern]}/"
+          end
+        end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_root, :more => "Log analysis"
+      end
+    end
+
+
+    # Thor method : show logs monitoring history
+    long_desc <<-LONGDESC
+    show logs monitoring history for LABEL\n
+    LONGDESC
+    option :table, :type => :boolean,  :aliases => "-t"
+    desc "history LABEL", "show logs monitoring history"
+    def history(label)
+      if is_root? then
+        log = get_logger
+        log.info "Log : #{label}"
+        config = get_config
+        if options[:table] then
+          table = TTY::Table.new do |t|
+            t << ["Start Date", "File","Status", "Nb errors", "Nb lines"]
+            t << ['','','','','']
+            LogsRecords::new(label).get_all_records.each do |item|
+              record =item.keys.first
+              value=item[record]
+
+              t << [record, value[:file], value[:status].to_s, value[:errors], value[:lines]]
+            end
+          end
+          if check_unicode_term  then
+            puts table.render(:unicode)
+          else
+            puts table.render(:ascii)
+          end
+
+        else
+          LogsRecords::new(label).get_all_records.each do |item|
+            record =item.keys.first
+            value=item[record]
+            log.item record
+            log.arrow "Status : #{value[:status].to_s}"
+            log.arrow "nb files : #{value[:errors]}"
+            log.arrow "nb lines : #{value[:lines]}"
+          end
         end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_root, :more => "Log analysis"
       end
-      splash_exit case: :quiet_exit
     end
 
   end

data/lib/splash/cli/process.rb

@@ -12,65 +12,77 @@ module CLISplash
     # Thor method : unning Splash configured processes monitors analyse
     desc "analyse", "analyze processes defined in Splash config"
     def analyse
-      log  = get_logger
-      results = ProcessScanner::new
-      res = results.analyse
-      log.info "Splash Configured process records :"
-      full_status = true
-      results.output.each do |result|
-        if result[:status] == :running then
-          log.ok "Process : #{result[:process]} : running"
-          log.item "Detected patterns : "
-          result[:patterns].each do |pattern|
-            log.arrow "/#{pattern}/"
+      if is_root? then
+        log  = get_logger
+        results = ProcessScanner::new
+        res = results.analyse
+        log.info "Splash Configured process records :"
+        full_status = true
+        results.output.each do |result|
+          if result[:status] == :running then
+            log.ok "Process : #{result[:process]} : running"
+            log.item "Detected patterns : "
+            result[:patterns].each do |pattern|
+              log.arrow "/#{pattern}/"
+            end
+            log.item "CPU usage in % : #{result[:cpu]} "
+            log.item "Memory usage in % : #{result[:mem]} "
+          else
+            log.ko "Process : #{result[:process]} : inexistant"
+            log.item "Detected patterns : "
+            result[:patterns].each do |pattern|
+              log.arrow "/#{pattern}/"
+            end
           end
-          log.item "CPU usage in % : #{result[:cpu]} "
-          log.item "Memory usage in % : #{result[:mem]} "
-        else
-          log.ko "Process : #{result[:process]} : inexistant"
-          log.item "Detected patterns : "
-          result[:patterns].each do |pattern|
-            log.arrow "/#{pattern}/"
-          end
-        end
 
-        full_status = false unless result[:status] == :running
-      end
+          full_status = false unless result[:status] == :running
+        end
 
-      if full_status then
-        log.ok "Global status : no error found"
+        if full_status then
+          log.ok "Global status : no error found"
+        else
+          log.error "Global status : some error found"
+        end
+        splash_exit case: :quiet_exit
       else
-        log.error "Global status : some error found"
+        splash_exit case: :not_root, :more => "Process analysis"
       end
-      splash_exit case: :quiet_exit
     end
 
 
     # Thor method : running Splash configured processes monitors analyse and sending to Prometheus Pushgateway
     desc "monitor", "monitor processes defined in Splash config"
     def monitor
-      log = get_logger
-      log.level = :fatal if options[:quiet]
-      result = ProcessScanner::new
-      result.analyse
-      splash_exit result.notify
+      if is_root? then
+        log = get_logger
+        log.level = :fatal if options[:quiet]
+        result = ProcessScanner::new
+        result.analyse
+        splash_exit result.notify
+      else
+        splash_exit case: :not_root, :more => "Process analysis"
+      end
     end
 
     # Thor method : display a specific Splash configured process monitor
     desc "show PROCESS", "show Splash configured process record for PROCESS"
     def show(record)
-      log = get_logger
-      process_recordset = get_config.processes.select{|item| item[:process] == record }
-      unless process_recordset.empty? then
-        record = process_recordset.first
-        log.item "Process monitor : #{record[:process]}"
-        log.arrow "patterns :"
-        record[:patterns].each do |pattern|
-          log.flat "   - /#{pattern}/"
+      if is_root? then
+        log = get_logger
+        process_recordset = get_config.processes.select{|item| item[:process] == record }
+        unless process_recordset.empty? then
+          record = process_recordset.first
+          log.item "Process monitor : #{record[:process]}"
+          log.arrow "patterns :"
+          record[:patterns].each do |pattern|
+            log.flat "   - /#{pattern}/"
+          end
+          splash_exit case: :quiet_exit
+        else
+          splash_exit case: :not_found, :more => "Process not configured"
         end
-        splash_exit case: :quiet_exit
       else
-        splash_exit case: :not_found, :more => "Process not configured"
+        splash_exit case: :not_root, :more => "Process analysis"
       end
     end
 
@@ -82,20 +94,68 @@ module CLISplash
     LONGDESC
     option :detail, :type => :boolean,  :aliases => "-D"
     def list
-      log = get_logger
-      log.info "Splash configured process records :"
-      process_recordset = get_config.processes
-      log.ko 'No configured process found' if process_recordset.empty?
-      process_recordset.each do |record|
-        log.item "Process monitor : #{record[:process]}"
-        if options[:detail] then
-          log.arrow "patterns :"
-          record[:patterns].each do |pattern|
-            log.flat "   - /#{pattern}/"
+      if is_root? then
+        log = get_logger
+        log.info "Splash configured process records :"
+        process_recordset = get_config.processes
+        log.ko 'No configured process found' if process_recordset.empty?
+        process_recordset.each do |record|
+          log.item "Process monitor : #{record[:process]}"
+          if options[:detail] then
+            log.arrow "patterns :"
+            record[:patterns].each do |pattern|
+              log.flat "   - /#{pattern}/"
+            end
           end
         end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_root, :more => "Process analysis"
+      end
+    end
+
+
+    # Thor method : show logs monitoring history
+    long_desc <<-LONGDESC
+    show Process monitoring history for LABEL\n
+    LONGDESC
+    option :table, :type => :boolean,  :aliases => "-t"
+    desc "history PROCESS", "show process monitoring history"
+    def history(process)
+      if is_root? then
+        log = get_logger
+        log.info "Process : #{process}"
+        config = get_config
+        if options[:table] then
+          table = TTY::Table.new do |t|
+            t << ["Start Date", "Status", "CPU Percent", "MEM Percent"]
+            t << ['','','','']
+            ProcessRecords::new(process).get_all_records.each do |item|
+              record =item.keys.first
+              value=item[record]
+              t << [record, value[:status].to_s, value[:cpu_percent], value[:mem_percent]]
+            end
+          end
+          if check_unicode_term  then
+            puts table.render(:unicode)
+          else
+            puts table.render(:ascii)
+          end
+
+        else
+          ProcessRecords::new(process).get_all_records.each do |item|
+            record =item.keys.first
+            value=item[record]
+            log.item record
+            log.arrow "Status : #{value[:status].to_s}"
+            log.arrow "CPU Percent : #{value[:cpu_percent]}"
+            log.arrow "MEM Percent : #{value[:mem_percent]}"
+          end
+        end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_root, :more => "Process analysis"
       end
-      splash_exit case: :quiet_exit
     end
 
   end

data/lib/splash/cli/transfers.rb

@@ -44,7 +44,10 @@ module CLISplash
       log = get_logger
       log.item "Transfer : #{name}"
       config = get_config
-      data = TxRecords::new(name).get_all_records.select {|record,value| record == options[:date]}.first
+      data = TxRecords::new(name).get_all_records.select {|item|
+        record =item.keys.first
+        value=item[record]
+        record == options[:date]}.first
       if data.nil? then
         log.ko "Result for #{name} on date #{options[:date]} not found"
         splash_exit case: :not_found, :more => "Result inexistant"
@@ -93,7 +96,9 @@ module CLISplash
         table = TTY::Table.new do |t|
           t << ["Start Date", "End date", "time", "Files count","File count error","Status"]
           t << ['','','','','','']
-          TxRecords::new(name).get_all_records.each do |record,value|
+          TxRecords::new(name).get_all_records.each do |item|
+            record =item.keys.first
+            value=item[record]
             start_date = record
             end_date = (value[:end_date].nil?)? '': value[:end_date]
             time  = (value[:time].nil?)? '': value[:time]
@@ -111,7 +116,9 @@ module CLISplash
         end
 
       else
-        TxRecords::new(name).get_all_records.each do |record,value|
+        TxRecords::new(name).get_all_records.each do |item|
+          record =item.keys.first
+          value=item[record]
           failed = (value[:count].nil? or value[:done].nil?)? 'undef': value[:count].to_i - value[:done].count
           if value[:end_date].nil? then
             log.item "Event : #{record} STATUS : #{value[:status]}"

data/lib/splash/commands.rb

@@ -6,6 +6,78 @@ module Splash
   # Splash Commands module/namespace
   module Commands
 
+
+    class CmdNotifier
+
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_exitcode = Prometheus::Client::Gauge.new(:exitcode, docstring: 'SPLASH metric batch exitcode')
+      @@metric_time = Prometheus::Client::Gauge.new(:exectime, docstring: 'SPLASH metric batch execution time')
+      @@registry.register(@@metric_exitcode)
+      @@registry.register(@@metric_time)
+
+
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = "cmd_#{options[:name].to_s}"
+        @exitcode = options[:exitcode]
+        @time = options[:time]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+        unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_exitcode.set(@exitcode)
+        @@metric_time.set(@time)
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new(@name, hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+    class CmdRecords
+      include Splash::Backends
+      include Splash::Constants
+      def initialize(name)
+        @name = name
+        @backend = get_backend :execution_trace
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+    end
+
+
     # command execution wrapper
     class CommandWrapper
       include Splash::Templates
@@ -16,11 +88,7 @@ module Splash
       include Splash::Transports
 
 
-      @@registry = Prometheus::Client::Registry::new
-      @@metric_exitcode = Prometheus::Client::Gauge.new(:errorcode, docstring: 'SPLASH metric batch errorcode')
-      @@metric_time = Prometheus::Client::Gauge.new(:exectime, docstring: 'SPLASH metric batch execution time')
-      @@registry.register(@@metric_exitcode)
-      @@registry.register(@@metric_time)
+
 
       # Constructor
       # @param [String] name the name of the command
@@ -43,14 +111,17 @@ module Splash
       # @param [String] value numeric.to_s
       # @param [String] time execution time numeric.to_s
       # @return [Hash] Exiter case :quiet_exit
-      def notify(value,time)
+      def notify(value,time, session)
+        log = get_logger
         unless verify_service url: @config.prometheus_pushgateway_url then
           return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
         end
-        @@metric_exitcode.set(value)
-        @@metric_time.set(time)
-        hostname = Socket.gethostname
-        Prometheus::Client::Push.new(@name, hostname, @url).add(@@registry)
+        cmdmonitor = CmdNotifier::new({name: @name, exitcode: value, time: time})
+        if cmdmonitor.notify then
+          log.ok "Sending metrics to Prometheus Pushgateway",session
+        else
+          log.ko "Failed to send metrics to Prometheus Pushgateway",session
+        end
         return { :case => :quiet_exit}
       end
 
@@ -110,9 +181,6 @@ module Splash
               stdout, stderr, status = Open3.capture3(@config.commands[@name.to_sym][:command])
             end
             time = Time.now - start
-            tp = Template::new(
-              list_token: @config.execution_template_tokens,
-              template_file: @config.execution_template_path)
             data = Hash::new
             data[:start_date] = start_date
             data[:end_date] = DateTime.now.to_s
@@ -123,17 +191,15 @@ module Splash
             data[:stdout] = stdout
             data[:stderr] = stderr
             data[:exec_time] = time.to_s
-            backend = get_backend :execution_trace
-            key = @name
-
-            backend.put key: key, value: data.to_yaml
+            cmdrec = CmdRecords::new @name
+            cmdrec.purge(@config.commands[@name.to_sym][:retention])
+            cmdrec.add_record data
             exit_code = status.exitstatus
           end
           log.ok "Command executed", session
           log.arrow "exitcode #{exit_code}", session
           if options[:notify] then
-            acase = notify(exit_code,time.to_i)
-            get_logger.ok "Prometheus Gateway notified.",session
+            acase = notify(exit_code,time.to_i,session)
           else
             log.item "Without Prometheus notification", session
           end

data/lib/splash/constants.rb

@@ -7,7 +7,7 @@ module Splash
   module Constants
 
     # Current splash version
-    VERSION = "0.8.0"
+    VERSION = "0.8.1"
 
     # the path to th config file, not overridable by config
     CONFIG_FILE = "/etc/splash.yml"
@@ -87,8 +87,8 @@ module Splash
     # the default sdterr trace file
     WEBADMIN_STDERR_TRACE="stderr.txt"
 
-    # default transfer retention for trace
-    TRANSFER_DEFAULT_RETENTION=1
+    # default retention for trace
+    DEFAULT_RETENTION=1
 
   end
 end

data/lib/splash/logs.rb

@@ -6,6 +6,79 @@ module Splash
   # Logs namespace
   module Logs
 
+    class LogsNotifier
+
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_missing = Prometheus::Client::Gauge.new(:logmissing, docstring: 'SPLASH metric log missing', labels: [:log ])
+      @@metric_count = Prometheus::Client::Gauge.new(:logerrors, docstring: 'SPLASH metric log error', labels: [:log ])
+      @@metric_lines = Prometheus::Client::Gauge.new(:loglines, docstring: 'SPLASH metric log lines numbers', labels: [:log ])
+      @@registry.register(@@metric_count)
+      @@registry.register(@@metric_missing)
+      @@registry.register(@@metric_lines)
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = options[:name]
+        @missing = options[:missing]
+        @lines = options[:lines]
+        @errors = options[:errors]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+        unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_missing.set(@missing, labels: { log: @name })
+        @@metric_count.set(@errors, labels: { log: @name })
+        @@metric_lines.set(@lines, labels: { log: @name })
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new("Splash", hostname, @url).add(@@registry)
+      end
+
+    end
+
+    class LogsRecords
+      include Splash::Backends
+      include Splash::Constants
+      def initialize(name)
+        @name = name
+        @backend = get_backend :logs_trace
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+    end
+
+
+
     # Log scanner and notifier
     class LogScanner
       include Splash::Constants
@@ -17,13 +90,7 @@ module Splash
       def initialize
         @logs_target = Marshal.load(Marshal.dump(get_config.logs))
         @config = get_config
-        @registry = Prometheus::Client::Registry::new
-        @metric_count = Prometheus::Client::Gauge.new(:logerrors, docstring: 'SPLASH metric log error', labels: [:log ])
-        @metric_missing = Prometheus::Client::Gauge.new(:logmissing, docstring: 'SPLASH metric log missing', labels: [:log ])
-        @metric_lines = Prometheus::Client::Gauge.new(:loglines, docstring: 'SPLASH metric log lines numbers', labels: [:log ])
-        @registry.register(@metric_count)
-        @registry.register(@metric_missing)
-        @registry.register(@metric_lines)
+
       end
 
 
@@ -62,17 +129,24 @@ module Splash
         session = (options[:session]) ? options[:session] : log.get_session
         log.info "Sending metrics to Prometheus Pushgateway", session
         @logs_target.each do |item|
-          missing = (item[:status] == :missing)? 1 : 0
-          log.item "Sending metrics for #{item[:log]}", session
-          @metric_count.set(item[:count], labels: { log: item[:log] })
-          @metric_missing.set(missing, labels: { log: item[:log] })
+          logsrec = LogsRecords::new item[:label]
+          errors = (item[:count])? item[:count] : 0
           lines = (item[:lines])? item[:lines] : 0
-          @metric_lines.set(lines, labels: { log: item[:log] })
+          missing = (item[:status] = :missing)? 1 : 0
+          file = item[:log]
+          logsrec.purge(item[:retention])
+          logsrec.add_record :status => item[:status],
+                           :errors => errors,
+                           :lines => lines,
+                           :file => file
+
+          logsmonitor = LogsNotifier::new({name: item[:label], missing: missing, file: file, errors: errors, lines: lines})
+          if logsmonitor.notify then
+            log.ok "Sending metrics for log #{file} to Prometheus Pushgateway", session
+          else
+            log.ko "Failed to send metrics for log #{file} to Prometheus Pushgateway", session
+          end
         end
-        hostname = Socket.gethostname
-        url = @config.prometheus_pushgateway_url
-        Prometheus::Client::Push.new('Splash',hostname, url).add(@registry)
-        log.ok "Sending to Prometheus PushGateway done.", session
         return {:case => :quiet_exit }
       end
 

data/lib/splash/processes.rb

@@ -6,6 +6,81 @@ module Splash
   # Processes namespace
   module Processes
 
+
+    class ProcessNotifier
+
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_status = Prometheus::Client::Gauge.new(:process_status, docstring: 'SPLASH metric process status', labels: [:process ])
+      @@metric_cpu_percent = Prometheus::Client::Gauge.new(:process_cpu_percent, docstring: 'SPLASH metric process CPU usage in percent', labels: [:process ])
+      @@metric_mem_percent = Prometheus::Client::Gauge.new(:process_mem_percent, docstring: 'SPLASH metric process MEM usage in percent', labels: [:process ])
+      @@registry.register(@@metric_status)
+      @@registry.register(@@metric_cpu_percent)
+      @@registry.register(@@metric_mem_percent)
+
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = options[:name]
+        @status = options[:status]
+        @cpu_percent = options[:cpu_percent]
+        @mem_percent = options[:mem_percent]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+        unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_mem_percent.set(@mem_percent, labels: { process: @name })
+        @@metric_cpu_percent.set(@cpu_percent, labels: { process: @name })
+        @@metric_status.set(@status, labels: { process: @name })
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new("Splash", hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+    class ProcessRecords
+      include Splash::Backends
+      include Splash::Constants
+
+      def initialize(name)
+        @name = name
+        @backend = get_backend :process_trace
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+    end
+
     # Processes scanner and notifier
     class ProcessScanner
       include Splash::Constants
@@ -17,14 +92,6 @@ module Splash
       def initialize
         @processes_target = Marshal.load(Marshal.dump(get_config.processes))
         @config = get_config
-        @registry = Prometheus::Client::Registry::new
-        @metric_status = Prometheus::Client::Gauge.new(:process_status, docstring: 'SPLASH metric process status', labels: [:process ])
-        @metric_cpu_percent = Prometheus::Client::Gauge.new(:process_cpu_percent, docstring: 'SPLASH metric process CPU usage in percent', labels: [:process ])
-        @metric_mem_percent = Prometheus::Client::Gauge.new(:process_mem_percent, docstring: 'SPLASH metric process MEM usage in percent', labels: [:process ])
-        @registry.register(@metric_status)
-        @registry.register(@metric_cpu_percent)
-        @registry.register(@metric_mem_percent)
-
       end
 
 
@@ -64,17 +131,21 @@ module Splash
         session = (options[:session]) ? options[:session] : log.get_session
         log.info "Sending metrics to Prometheus Pushgateway", session
         @processes_target.each do |item|
+          processrec = ProcessRecords::new item[:process]
           missing = (item[:status] == :missing)? 1 : 0
-          log.item "Sending metrics for #{item[:process]}", session
           val = (item[:status] == :running )? 1 : 0
-          @metric_status.set(val, labels: { process: item[:process] })
-          @metric_cpu_percent.set(item[:cpu], labels: { process: item[:process] })
-          @metric_mem_percent.set(item[:mem], labels: { process: item[:process] })
+          processrec.purge(item[:retention])
+          processrec.add_record :status => item[:status],
+                           :cpu_percent => item[:cpu],
+                           :mem_percent => item[:mem] ,
+                           :process => item[:process]
+          processmonitor = ProcessNotifier::new({name: item[:process], status: val , cpu_percent: item[:cpu], mem_percent: item[:mem]})
+          if processmonitor.notify then
+            log.ok "Sending metrics for process #{item[:process]} to Prometheus Pushgateway", session
+          else
+            log.ko "Failed to send metrics for process #{item[:process]} to Prometheus Pushgateway", session
+          end
         end
-        hostname = Socket.gethostname
-        url = @config.prometheus_pushgateway_url
-        Prometheus::Client::Push.new('Splash',hostname, url).add(@registry)
-        log.ok "Sending to Prometheus PushGateway done.", session
         return {:case => :quiet_exit }
       end
 

data/lib/splash/transfers.rb

@@ -49,40 +49,45 @@ module Splash
 
     class TxRecords
       include Splash::Backends
+      include Splash::Constants
+
       def initialize(name)
         @name = name
-        @backend = get_backend :transferts_trace
+        @backend = get_backend :transfers_trace
       end
 
       def purge(retention)
+        retention = {} if retention.nil?
         if retention.include? :hours then
           adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
         elsif retention.include? :hours then
           adjusted_datetime = DateTime.now - retention[:days].to_i
         else
-          retention = TRANSFER_DEFAULT_RETENTION
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
         end
 
         data = get_all_records
 
-        data.delete_if { |item,value|
-          DateTime.parse(item) <= (adjusted_datetime) and value[:status] != :prepared}
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
         @backend.put key: @name, value: data.to_yaml
       end
 
       def add_record(record)
         data = get_all_records
-        data[DateTime.now.to_s] = record
+        data.push({ DateTime.now.to_s => record })
         @backend.put key: @name, value: data.to_yaml
       end
 
-      def get_all_records
-        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : {}
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
       end
 
       def check_prepared
         return :never_run_prepare unless @backend.exist?({key: @name})
-        return :never_prepare unless YAML::load(@backend.get({key: @name})).select {|item,value|
+        return :never_prepare unless YAML::load(@backend.get({key: @name})).select {|item|
+          record =item.keys.first
+          value=item[record]
           value[:status] == :prepared
         }.count > 0
         return :prepared

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-splash
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Romain GEORGES
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2020-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -508,7 +508,7 @@ homepage: https://github.com/Ultragreen/prometheus-splash
 licenses:
 - BSD-2-Clause
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -523,8 +523,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.0.6
+signing_key: 
 specification_version: 4
 summary: Supervision with Prometheus of Logs and Asynchronous tasks orchestration
   for Services or Hosts