prometheus-splash 0.6.1 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3048edcf69c8bac285eab790a497fddbb9a4420a5cee8738ee1397c0a7c39587
-  data.tar.gz: adb8f52b4e3b2d808728e092960e648768a45d8fbb66876ca7c585f492516c86
+  metadata.gz: 7da11971391d77ffc6bf37228938bba227880e8748ce605cfa3c33ab16855121
+  data.tar.gz: f3a1b196f64ea586a2d604c799ca1a11f9f966471e8938ba56803b1067852b85
 SHA512:
-  metadata.gz: fd9ed4dc543ea0a81cd163c73988424a87b76a5e094f548900e30ff71db7e8525798d9c824113ecf8972d2e26b6a616d2393a1afb3a1e3caaa8f62ebb331f3dc
-  data.tar.gz: 23635fa7711d83e15f8b859d47bdcea3fa4dadc268cb899b46282555b73a2f998bf38614a48ec54997e9fe9986a78098fc2a9c818183be37e7dbd913bc7cfe8a
+  metadata.gz: 729ffa39b3dcafb9c70631d6c0c02dc3ba67ce78c1d97fff9b665bff3b1ff311aaa24ed79f43cfb414550da022f8cc7f7cc40de96928c1c44731bd9211e36932
+  data.tar.gz: 85dd4b338e46736d8bc68d3b33b5325f18c6276bf2a1e90ac3731e0bb86741d25f1574f333f1a77f23889bb0de8bcf5b2f763bb75bf1c0e4f75c5b00910ddd86

data/CHANGELOG.md

@@ -124,4 +124,46 @@
 ### FEATURE
 
 * processes monitoring #23
- 
+
+## V 0.6.0 2020/10/10
+
+### FEATURE
+
+* read only WebAdmin (Major update)
+* API REST
+* refactoring
+
+## V 0.7.0
+
+### FEATURE
+
+* sequences
+* API sequences
+* UI sequences
+* Transfers
+* API Transfers
+* UI Transfers
+
+## V 0.8.0
+
+### FEATURE
+* orchestrator rebuild
+* reshash config and reset + grammar and Cli
+* refacto config
+
+## V 0.8.1
+
+### FEATURE
+* full Web UI features for Logs (R/W)
+* API Logs Full
+
+## V 0.8.2
+
+### FIX
+* WebAdmin UI bugs and cosmetics
+* API (constants CLONE for exiter)
+
+## V 0.8.3
+
+### SECURITY
+* kramdown dependencies update

data/README.md

@@ -9,7 +9,7 @@ SPLASH is **Supervision with Prometheus of Logs and Asynchronous tasks orchestra
 * Web : http://www.ultragreen.net
 * Github : https://github.com/Ultragreen/prometheus-splash
 * Rubygems : https://rubygems.org/gems/prometheus-splash
-* DOC yardoc : https://www.rubydoc.info/gems/prometheus-splash/0.6.0
+* DOC yardoc : https://www.rubydoc.info/gems/prometheus-splash/0.8.3
 
 Prometheus Logs and Batchs supervision over PushGateway
 

data/config/splash.yml

@@ -2,6 +2,9 @@
 :splash:
 
 ### Main Configuration
+  :paths:
+    :pid_path: /var/run/splash
+    :trace_path: /var/run/splash/traces
   :loggers:
     :level: :info
     :daemon:
@@ -23,6 +26,16 @@
 #        :port: 6379
 #        #:auth: "mykey"
 #        :base: 1
+      :transfers_trace:
+          :type: :file
+          :path: /var/run/splash/transfers
+      :logs_trace:
+          :type: :file
+          :path: /var/run/splash/logs
+      :process_trace:
+          :type: :file
+          :path: /var/run/splash/process
+
   :transports:
     :active: :rabbitmq
     :rabbitmq:
@@ -39,16 +52,14 @@
     :procmon_scheduling:
       :every: 20s
     :process_name: "Splash : daemon."
-    :paths:
-      :pid_path: /var/run/splash
-      :trace_path: /var/run/splash/traces
     :files:
       :stdout_trace: stdout.txt
       :stderr_trace: stderr.txt
       :pid_file: splash.pid
   :prometheus:
-    :pushgateway: 'http://localhost:9091/'
-    :url: 'http://localhost:9090/'
+    :pushgateway: 'http://localhost:9091'
+    :url: 'http://localhost:9090'
+    :alertmanager: 'http://localhost:9093'
   :webadmin:
     :port: 9234
     :ip: 127.0.0.1
@@ -117,15 +128,21 @@
     - :label: :log_app_1
       :log: /tmp/test
       :pattern: ERROR
+      :retention:
+        :hours: 5
     - :label: :log_app_2
       :log: /tmp/test2
       :pattern: ERROR
+      :retention:
+        :hours: 5
 
 ### configuration of monitored processes
   :processes:
     - :process: cron
       :patterns:
         - cron
+      :retention:
+          :hours: 5
 
 
 ###
@@ -175,4 +192,27 @@
         - :step: exec echo3
           :command: :echo3
 
+### Transfers
+  :transfers:
+    - :name: :transfer_text
+      :desc: "List of text files"
+      :pattern: '*.txt'
+      :backup: true
+      :type: :push
+      :retention:
+        :hours: 1
+      :local:
+        :path: /tmp/transferts
+        :user: root
+      :remote:
+        :user: root
+        :host: localhost
+        :path: /tmp/
+      :post:
+        :remote_command: 'ls'
+
+
+
+
+
 ###

data/lib/splash/backends.rb

@@ -13,6 +13,7 @@ module Splash
     # @param [Symbol] store the name of the store actually in [:execution_trace]
     # @return [Splash::Backends::<Type>|Hash] with type in [:redis,:file] or Exiter case :configuration_error 
     def get_backend(store)
+      splash_exit case: :configuration_error, more: "backend definition failure" if get_config[:backends][:stores][store].nil?
       backend = get_config[:backends][:stores][store][:type].to_s
       aclass = "Splash::Backends::#{backend.capitalize}"
       begin

data/lib/splash/cli.rb

@@ -43,6 +43,7 @@ class CLI < Thor
   subcommand "documentation", Documentation
   desc "webadmin SUBCOMMAND ...ARGS", "Splash Webadmin daemon controller"
   subcommand "webadmin", WebAdmin
-
+  desc "transfers SUBCOMMAND ...ARGS", "Managing transfers"
+  subcommand "transfers", Transfers
 
 end

data/lib/splash/cli/commands.rb

@@ -93,6 +93,9 @@ module CLISplash
     option :at, :type => :string
     option :in, :type => :string
     def schedule(name)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command scheduling"
+      end
       log = get_logger
       log.level = :fatal if options[:quiet]
       hostname = (options[:hostname])? options[:hostname] : Socket.gethostname
@@ -129,6 +132,9 @@ module CLISplash
     LONGDESC
     option :hostname, :type => :string,  :aliases => "-H"
     def treeview(command)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command treeview"
+      end
       depht = 0
       log  = get_logger
       if options[:hostname] then
@@ -179,6 +185,9 @@ module CLISplash
     option :detail, :type => :boolean,  :aliases => "-D"
     option :hostname, :type => :string,  :aliases => "-H"
     def list
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command list"
+      end
       log = get_logger
       list = {}
       if options[:hostname] then
@@ -227,6 +236,9 @@ module CLISplash
     LONGDESC
     option :hostname, :type => :string,  :aliases => "-H"
     def show(command)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command show specifications"
+      end
       log = get_logger
       list = {}
       if options[:hostname] then
@@ -265,6 +277,57 @@ module CLISplash
       end
     end
 
+
+
+    # Thor method : show commands executions  history
+    long_desc <<-LONGDESC
+    show commands executions history for LABEL\n
+    LONGDESC
+    option :table, :type => :boolean,  :aliases => "-t"
+    desc "history LABEL", "show commands executions history"
+    def history(command)
+      if is_root? then
+        log = get_logger
+        log.info "Log : #{command}#"
+        config = get_config
+        if options[:table] then
+          table = TTY::Table.new do |t|
+            t << ["Start Date","Status", "end_date", "Execution time","STDOUT empty ? ", "STDERR empty ? "]
+            t << ['','','','','','']
+            CmdRecords::new(command).get_all_records.each do |item|
+              record =item.keys.first
+              value=item[record]
+              t << [record, value[:status].to_s,
+                            value[:end_date],
+                            value[:exec_time],
+                            value[:stdout].empty?,
+                            value[:stdout].empty?]
+            end
+          end
+          if check_unicode_term  then
+            puts table.render(:unicode)
+          else
+            puts table.render(:ascii)
+          end
+
+        else
+          CmdRecords::new(command).get_all_records.each do |item|
+            record =item.keys.first
+            value=item[record]
+            log.item record
+            log.arrow "Status : #{value[:status].to_s}"
+            log.arrow "End date : #{value[:end_date]}"
+            log.arrow "Execution time : #{value[:exec_time]}"
+            log.arrow "STDOUT empty ?  : #{value[:stdout].empty?}"
+            log.arrow "STDERR empty ?  : #{value[:stderr].empty?}"
+          end
+        end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_root, :more => "Command execution history"
+      end
+    end
+
     # Thor method : getting information on the last execution of a command
     desc "lastrun COMMAND", "Show last running result for specific configured command COMMAND"
     long_desc <<-LONGDESC
@@ -273,6 +336,9 @@ module CLISplash
     LONGDESC
     option :hostname, :type => :string,  :aliases => "-H"
     def lastrun(command)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command last execution report"
+      end
       log = get_logger
       backend = get_backend :execution_trace
       redis = (backend.class == Splash::Backends::Redis)? true : false
@@ -294,7 +360,7 @@ module CLISplash
           tp = Template::new(
               list_token: get_config.execution_template_tokens,
               template_file: get_config.execution_template_path)
-          tp.map YAML::load(res)
+          tp.map YAML::load(res).last.values.first
           log.flat tp.output
         else
           log.ko "Command not already runned."
@@ -305,6 +371,58 @@ module CLISplash
       end
     end
 
+
+    # Thor method : getting information on one specific execution of a command
+    desc "onerun COMMAND", "Show running result for specific configured command COMMAND"
+    long_desc <<-LONGDESC
+    Show specific running result for specific configured command COMMAND\n
+    with --hostname <HOSTNAME>, an other Splash monitored server (only with Redis backend configured)
+    with --date <DATE>, a date format string (same as in history ouput)
+
+    LONGDESC
+    option :hostname, :type => :string,  :aliases => "-H"
+    option :date, :type => :string,  :aliases => "-D", :required => true
+    def onerun(command)
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command specific execution report"
+      end
+      log = get_logger
+      backend = get_backend :execution_trace
+      redis = (backend.class == Splash::Backends::Redis)? true : false
+      if not redis and options[:hostname] then
+        splash_exit case: :specific_config_required, :more => "Redis backend is requiered for Remote execution report request"
+      end
+      splash_exit case: :not_root if not is_root? and not redis
+      list = get_config.commands.keys
+      if options[:hostname] then
+        options[:hostname] = Socket.gethostname if options[:hostname] == 'hostname'
+        list = backend.list("*", options[:hostname]).map(&:to_sym)
+      end
+      if list.include? command.to_sym then
+        log.info "Splash command #{command} previous execution report:\n"
+        req  = { :key => command}
+        req[:hostname] = options[:hostname] if options[:hostname]
+        if backend.exist? req then
+          res = backend.get req
+          tp = Template::new(
+              list_token: get_config.execution_template_tokens,
+              template_file: get_config.execution_template_path)
+          prov =  YAML::load(res).select{|key,value| key.keys.first == options[:date]}
+          if prov.empty? then
+            log.ko "Command not runned one this date or date misformatted."
+          else
+            tp.map prov.first.values.first
+            log.flat tp.output
+          end
+        else
+          log.ko "Command not already runned."
+        end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_found, :more => "Command report never runned remotly" if options[:hostname]
+      end
+    end
+
     # Thor method : getting the list of avaibles executions reports
     desc "getreportlist", "list all executions report results "
     long_desc <<-LONGDESC
@@ -320,6 +438,9 @@ module CLISplash
     option :all, :type => :boolean, :negate => false,  :aliases => "-A"
     option :detail, :type => :boolean,  :aliases => "-D"
     def getreportlist
+      unless is_root? then
+        splash_exit case: :not_root, :more => "Command execution report list"
+      end
       log = get_logger
       options[:hostname] = Socket.gethostname if options[:hostname] == 'hostname'
       if options[:hostname] and options[:all] then
@@ -355,10 +476,9 @@ module CLISplash
           req = { :key => command }
           req[:hostname] = host if options[:all]
           res = YAML::load(backend.get(req))
-          log.arrow "Status : #{res[:status]}"
-          log.arrow "Start date : #{res[:start_date]}"
-          log.arrow "End date : #{res[:end_date]}"
-          log.arrow "Execution time : #{res[:exec_time]}"
+          res.each do |record|
+            log.arrow "#{record.keys.first} : #{record[record.keys.first][:status]}"
+          end
         end
       end
       splash_exit case: :quiet_exit

data/lib/splash/cli/daemon.rb

@@ -74,7 +74,47 @@ module CLISplash
           splash_exit case: :quiet_exit
         end
       rescue Interrupt
-        splash_exit status: :error, case: :interrupt, more: "Ping Command"
+        splash_exit status: :error, case: :interrupt, more: "ping Command"
+      end
+    end
+
+    # Thor method : sending get_jobs verb over transport in the input queue of Splashd
+    desc "getjobs", "send a get_jobs verb to HOSTNAME daemon over transport (need an active tranport), Typicallly RabbitMQ"
+    def getjobs(hostname=Socket.gethostname)
+      log = get_logger
+      log.info "ctrl+c for interrupt"
+      begin
+        transport = get_default_client
+        if transport.class == Hash  and transport.include? :case then
+          splash_exit transport
+        else
+          log.receive transport.execute({ :verb => :get_jobs,
+                                  :return_to => "splash.#{Socket.gethostname}.returncli",
+                                  :queue => "splash.#{hostname}.input" })
+          splash_exit case: :quiet_exit
+        end
+      rescue Interrupt
+        splash_exit status: :error, case: :interrupt, more: "getjobs Command"
+      end
+    end
+
+    # Thor method : sending reset verb over transport in the input queue of Splashd
+    desc "getjobs", "send a reset verb to HOSTNAME daemon over transport (need an active tranport), Typicallly RabbitMQ"
+    def reset(hostname=Socket.gethostname)
+      log = get_logger
+      log.info "ctrl+c for interrupt"
+      begin
+        transport = get_default_client
+        if transport.class == Hash  and transport.include? :case then
+          splash_exit transport
+        else
+          log.receive transport.execute({ :verb => :reset,
+                                  :return_to => "splash.#{Socket.gethostname}.returncli",
+                                  :queue => "splash.#{hostname}.input" })
+          splash_exit case: :quiet_exit
+        end
+      rescue Interrupt
+        splash_exit status: :error, case: :interrupt, more: "reset Command"
       end
     end
 

data/lib/splash/cli/logs.rb

@@ -12,61 +12,76 @@ module CLISplash
     # Thor method : running Splash configured logs monitors analyse
     desc "analyse", "analyze logs defined in Splash config"
     def analyse
-      log  = get_logger
-      results = LogScanner::new
-      res = results.analyse
-      log.info "SPlash Configured log monitors :"
-      full_status = true
-      results.output.each do |result|
-        if result[:status] == :clean then
-          log.ok "Log : #{result[:log]} with label : #{result[:label]} : no errors"
-          log.item "Detected pattern : #{result[:pattern]}"
-          log.item "Nb lines = #{result[:lines]}"
-        elsif result[:status] == :missing then
-          log.ko "Log : #{result[:log]} with label : #{result[:label]} : missing !"
-          log.item "Detected pattern : #{result[:pattern]}"
-        else
-          log.ko "Log : #{result[:log]} with label : #{result[:label]} : #{result[:count]} errors"
-          log.item "Detected pattern : #{result[:pattern]}"
-          log.item "Nb lines = #{result[:lines]}"
-        end
+      if is_root? then
+        log  = get_logger
+        results = LogScanner::new
+        res = results.analyse
+        log.info "SPlash Configured log monitors :"
+        full_status = true
+        results.output.each do |result|
+          if result[:status] == :clean then
+            log.ok "Log : #{result[:log]} with label : #{result[:label]} : no errors"
+            log.item "Detected pattern : #{result[:pattern]}"
+            log.item "Nb lines = #{result[:lines]}"
+          elsif result[:status] == :missing then
+            log.ko "Log : #{result[:log]} with label : #{result[:label]} : missing !"
+            log.item "Detected pattern : #{result[:pattern]}"
+          else
+            log.ko "Log : #{result[:log]} with label : #{result[:label]} : #{result[:count]} errors"
+            log.item "Detected pattern : #{result[:pattern]}"
+            log.item "Nb lines = #{result[:lines]}"
+          end
 
-        full_status = false unless result[:status] == :clean
-      end
+          full_status = false unless result[:status] == :clean
+        end
 
-      if full_status then
-        log.ok "Global status : no error found"
+        if full_status then
+          log.ok "Global status : no error found"
+        else
+          log.error "Global status : some error found"
+        end
+        splash_exit case: :quiet_exit
       else
-        log.error "Global status : some error found"
+        splash_exit case: :not_root, :more => "Log analysis"
       end
-      splash_exit case: :quiet_exit
     end
 
 
     # Thor method : running Splash configured logs monitors analyse and sending to Prometheus Pushgateway
     desc "monitor", "monitor logs defined in Splash config"
     def monitor
-      log = get_logger
-      log.level = :fatal if options[:quiet]
-      result = LogScanner::new
-      result.analyse
-      splash_exit result.notify
+      if is_root? then
+        log = get_logger
+        log.level = :fatal if options[:quiet]
+        result = LogScanner::new
+        result.analyse
+        splash_exit result.notify
+      else
+        splash_exit case: :not_root, :more => "Log analysis"
+      end
 
     end
 
+
+    
+
     # Thor method : display a specific Splash configured log monitor
     desc "show LOG", "show Splash configured log monitoring for LOG"
     def show(logrecord)
-      log = get_logger
-      log_record_set = get_config.logs.select{|item| item[:log] == logrecord or item[:label] == logrecord.to_sym}
-      unless log_record_set.empty? then
-        record = log_record_set.first
-        log.info "Splash log monitor : #{record[:log]}"
-        log.item "pattern : /#{record[:pattern]}/"
-        log.item "label : #{record[:label]}"
-        splash_exit case: :quiet_exit
+      if is_root? then
+        log = get_logger
+        log_record_set = get_config.logs.select{|item| item[:log] == logrecord or item[:label] == logrecord.to_sym}
+        unless log_record_set.empty? then
+          record = log_record_set.first
+          log.info "Splash log monitor : #{record[:log]}"
+          log.item "pattern : /#{record[:pattern]}/"
+          log.item "label : #{record[:label]}"
+          splash_exit case: :quiet_exit
+        else
+          splash_exit case: :not_found, :more => "log not configured"
+        end
       else
-        splash_exit case: :not_found, :more => "log not configured"
+        splash_exit case: :not_root, :more => "Log analysis"
       end
     end
 
@@ -78,17 +93,66 @@ module CLISplash
     LONGDESC
     option :detail, :type => :boolean,  :aliases => "-D"
     def list
-      log = get_logger
-      log.info "Splash configured log monitoring :"
-      log_record_set = get_config.logs
-      log.ko 'No configured commands found' if log_record_set.empty?
-      log_record_set.each do |record|
-        log.item "log monitor : #{record[:log]} label : #{record[:label]}"
-        if options[:detail] then
-          log.arrow "pattern : /#{record[:pattern]}/"
+      if is_root? then
+        log = get_logger
+        log.info "Splash configured log monitoring :"
+        log_record_set = get_config.logs
+        log.ko 'No configured commands found' if log_record_set.empty?
+        log_record_set.each do |record|
+          log.item "log monitor : #{record[:log]} label : #{record[:label]}"
+          if options[:detail] then
+            log.arrow "pattern : /#{record[:pattern]}/"
+          end
         end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_root, :more => "Log analysis"
+      end
+    end
+
+
+    # Thor method : show logs monitoring history
+    long_desc <<-LONGDESC
+    show logs monitoring history for LABEL\n
+    LONGDESC
+    option :table, :type => :boolean,  :aliases => "-t"
+    desc "history LABEL", "show logs monitoring history"
+    def history(label)
+      if is_root? then
+        log = get_logger
+        log.info "Log : #{label}"
+        config = get_config
+        if options[:table] then
+          table = TTY::Table.new do |t|
+            t << ["Start Date", "File","Status", "Nb errors", "Nb lines"]
+            t << ['','','','','']
+            LogsRecords::new(label).get_all_records.each do |item|
+              record =item.keys.first
+              value=item[record]
+
+              t << [record, value[:file], value[:status].to_s, value[:errors], value[:lines]]
+            end
+          end
+          if check_unicode_term  then
+            puts table.render(:unicode)
+          else
+            puts table.render(:ascii)
+          end
+
+        else
+          LogsRecords::new(label).get_all_records.each do |item|
+            record =item.keys.first
+            value=item[record]
+            log.item record
+            log.arrow "Status : #{value[:status].to_s}"
+            log.arrow "nb files : #{value[:errors]}"
+            log.arrow "nb lines : #{value[:lines]}"
+          end
+        end
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_root, :more => "Log analysis"
       end
-      splash_exit case: :quiet_exit
     end
 
   end