prometheus-splash 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e97bc4cce28c28f041ef5699bf6d3236001e84517bdefe8449cc572952997d2
-  data.tar.gz: c8e67cdc8089ff1db9e19ea4b86b84b52a64515d167b11df4f70e0e85e42f9f8
+  metadata.gz: db8cc49d7a0a733b08a9f744a2ac14b5196b69fc229fcd77c9c635dd9e6c61e8
+  data.tar.gz: c839f4d20b88880a458e92de5b2d10f7d58788645fb7b03c0c8e63d403f5ec78
 SHA512:
-  metadata.gz: 6642f9127b2f422e839d0b19c10c4625eb8b86bb1feeb7cd8ff2ba00369905fc1bb8a3f078a896cf3cf5e661273fc355c6194dde625673e9a17dc4bba14c3bc9
-  data.tar.gz: 10d052291ebe57a916f4079c23934fda5dac4e44d3ec45856982ba2b827c2764a57dd3990bd4819847278d7f6a5b22996f7e24dd5a4bcfc3936091b71d599645
+  metadata.gz: 6d1314705d22d1da16a62b265c0a9967b3fa2cd049db1dd25af3631b144b7872da48a84d1c62d36cc9fa650cb398ab4beaba9523d46f79bde52f3b6382b586e3
+  data.tar.gz: d8d637a2ea309860d988952f87c63b906c1b4c350dfa5327f12f84ddc0cd352c3ca4b7112ac2f180b890d2acc3553db2caaa0abda3fc1a77a0eccf88d6c44f85

data/CHANGELOG.md

@@ -124,4 +124,29 @@
 ### FEATURE
 
 * processes monitoring #23
- 
+
+## V 0.6.0 2020/10/10
+
+### FEATURE
+
+* read only WebAdmin (Major update)
+* API REST
+* refactoring
+
+## V 0.7.0
+
+### FEATURE
+
+* sequences
+* API sequences
+* UI sequences
+* Transfers
+* API Transfers
+* UI Transfers
+
+## V 0.8.0
+
+### FEATURE
+* orchestrator rebuild
+* reshash config and reset + grammar and Cli
+* refacto config

data/README.md

@@ -9,7 +9,7 @@ SPLASH is **Supervision with Prometheus of Logs and Asynchronous tasks orchestra
 * Web : http://www.ultragreen.net
 * Github : https://github.com/Ultragreen/prometheus-splash
 * Rubygems : https://rubygems.org/gems/prometheus-splash
-* DOC yardoc : https://www.rubydoc.info/gems/prometheus-splash/0.6.0
+* DOC yardoc : https://www.rubydoc.info/gems/prometheus-splash/0.8.0
 
 Prometheus Logs and Batchs supervision over PushGateway
 

data/config/splash.yml

@@ -2,6 +2,9 @@
 :splash:
 
 ### Main Configuration
+  :paths:
+    :pid_path: /var/run/splash
+    :trace_path: /var/run/splash/traces
   :loggers:
     :level: :info
     :daemon:
@@ -23,6 +26,10 @@
 #        :port: 6379
 #        #:auth: "mykey"
 #        :base: 1
+      :transferts_trace:
+          :type: :file
+          :path: /var/run/splash/transferts
+
   :transports:
     :active: :rabbitmq
     :rabbitmq:
@@ -39,16 +46,14 @@
     :procmon_scheduling:
       :every: 20s
     :process_name: "Splash : daemon."
-    :paths:
-      :pid_path: /var/run/splash
-      :trace_path: /var/run/splash/traces
     :files:
       :stdout_trace: stdout.txt
       :stderr_trace: stderr.txt
       :pid_file: splash.pid
   :prometheus:
-    :pushgateway: 'http://localhost:9091/'
-    :url: 'http://localhost:9090/'
+    :pushgateway: 'http://localhost:9091'
+    :url: 'http://localhost:9090'
+    :alertmanager: 'http://localhost:9093'
   :webadmin:
     :port: 9234
     :ip: 127.0.0.1
@@ -175,4 +180,27 @@
         - :step: exec echo3
           :command: :echo3
 
+### Transfers
+  :transfers:
+    - :name: :transfer_text
+      :desc: "List of text files"
+      :pattern: '*.txt'
+      :backup: true
+      :type: :push
+      :retention:
+        :hours: 1
+      :local:
+        :path: /tmp/transferts
+        :user: root
+      :remote:
+        :user: root
+        :host: localhost
+        :path: /tmp/
+      :post:
+        :remote_command: 'ls'
+
+
+
+
+
 ###

data/lib/splash/backends.rb

@@ -13,6 +13,7 @@ module Splash
     # @param [Symbol] store the name of the store actually in [:execution_trace]
     # @return [Splash::Backends::<Type>|Hash] with type in [:redis,:file] or Exiter case :configuration_error 
     def get_backend(store)
+      splash_exit case: :configuration_error, more: "backend definition failure" if get_config[:backends][:stores][store].nil?
       backend = get_config[:backends][:stores][store][:type].to_s
       aclass = "Splash::Backends::#{backend.capitalize}"
       begin

data/lib/splash/cli.rb

@@ -43,6 +43,7 @@ class CLI < Thor
   subcommand "documentation", Documentation
   desc "webadmin SUBCOMMAND ...ARGS", "Splash Webadmin daemon controller"
   subcommand "webadmin", WebAdmin
-
+  desc "transfers SUBCOMMAND ...ARGS", "Managing transfers"
+  subcommand "transfers", Transfers
 
 end

data/lib/splash/cli/daemon.rb

@@ -74,7 +74,47 @@ module CLISplash
           splash_exit case: :quiet_exit
         end
       rescue Interrupt
-        splash_exit status: :error, case: :interrupt, more: "Ping Command"
+        splash_exit status: :error, case: :interrupt, more: "ping Command"
+      end
+    end
+
+    # Thor method : sending get_jobs verb over transport in the input queue of Splashd
+    desc "getjobs", "send a get_jobs verb to HOSTNAME daemon over transport (need an active tranport), Typicallly RabbitMQ"
+    def getjobs(hostname=Socket.gethostname)
+      log = get_logger
+      log.info "ctrl+c for interrupt"
+      begin
+        transport = get_default_client
+        if transport.class == Hash  and transport.include? :case then
+          splash_exit transport
+        else
+          log.receive transport.execute({ :verb => :get_jobs,
+                                  :return_to => "splash.#{Socket.gethostname}.returncli",
+                                  :queue => "splash.#{hostname}.input" })
+          splash_exit case: :quiet_exit
+        end
+      rescue Interrupt
+        splash_exit status: :error, case: :interrupt, more: "getjobs Command"
+      end
+    end
+
+    # Thor method : sending reset verb over transport in the input queue of Splashd
+    desc "getjobs", "send a reset verb to HOSTNAME daemon over transport (need an active tranport), Typicallly RabbitMQ"
+    def reset(hostname=Socket.gethostname)
+      log = get_logger
+      log.info "ctrl+c for interrupt"
+      begin
+        transport = get_default_client
+        if transport.class == Hash  and transport.include? :case then
+          splash_exit transport
+        else
+          log.receive transport.execute({ :verb => :reset,
+                                  :return_to => "splash.#{Socket.gethostname}.returncli",
+                                  :queue => "splash.#{hostname}.input" })
+          splash_exit case: :quiet_exit
+        end
+      rescue Interrupt
+        splash_exit status: :error, case: :interrupt, more: "reset Command"
       end
     end
 

data/lib/splash/cli/transfers.rb

@@ -0,0 +1,206 @@
+# coding: utf-8
+
+# module for all Thor subcommands
+module CLISplash
+
+  # Thor inherited class for transfers management
+  class Transfers < Thor
+    include Splash::Transfers
+    include Splash::Helpers
+    include Splash::Exiter
+    include Splash::Loggers
+    include Splash::Transfers
+
+
+    # Thor method : running transfer prepare
+    long_desc <<-LONGDESC
+    Prepare transfer with RSA Public key for NAME\n
+    Warning : interactive command only (prompt for passwd)
+    LONGDESC
+    desc "prepare", "Prepare transfers with RSA Public key"
+    def prepare(name)
+      acase = run_as_root :prepare_tx, name
+      splash_exit acase
+    end
+
+    # Thor method : Execute all transfers
+    long_desc <<-LONGDESC
+    Execute all transfers\n
+    Warning : interactive command only (prompt for passwd)
+    LONGDESC
+    desc "full_execute", "Execute all transfers"
+    def full_execute
+      acase = run_as_root :run_txs
+      splash_exit acase
+    end
+
+    # Thor method : Get specific result for a transfers
+    long_desc <<-LONGDESC
+    Get specific result for a transfers\n
+    LONGDESC
+    option :date, :type => :string,  :aliases => "-d"
+    desc "get_result TRANSFER", "Get specific result for a transfers "
+    def get_result(name)
+      log = get_logger
+      log.item "Transfer : #{name}"
+      config = get_config
+      data = TxRecords::new(name).get_all_records.select {|record,value| record == options[:date]}.first
+      if data.nil? then
+        log.ko "Result for #{name} on date #{options[:date]} not found"
+        splash_exit case: :not_found, :more => "Result inexistant"
+      else
+        record = options[:date]
+        value = data[record]
+        failed = (value[:count].nil? or value[:done].nil?)? 'undef': value[:count].to_i - value[:done].count
+        if value[:end_date].nil? then
+          log.item "Event : #{record} STATUS : #{value[:status]}"
+        else
+          log.item "Tx Begin : #{record} => end : #{value[:end_date]} STATUS : #{value[:status]}"
+        end
+        log.arrow "Tx Time : #{value[:time]}" unless value[:time].nil?
+        log.arrow "nb files : #{value[:count]}" unless value[:count].nil?
+        unless value[:wanted].nil?
+          log.arrow "Files wanted :" unless value[:wanted].empty?
+          value[:wanted].each do |file|
+            log.flat  "    * #{file}"
+          end
+        end
+        unless value[:done].nil?
+          log.arrow "Files done :" unless value[:done].empty?
+          value[:done].each do |file|
+            log.flat  "    * #{file}"
+          end
+        end
+        unless failed then
+          log.arrow "Nb failure : #{failed}"
+        end
+
+      end
+      splash_exit case: :quiet_exit
+    end
+
+    # Thor method : show specfic transfers history
+    long_desc <<-LONGDESC
+    show transfers history for transfer NAME\n
+    LONGDESC
+    option :table, :type => :boolean,  :aliases => "-t"
+    desc "history", "Show transfers history"
+    def history(name)
+      log = get_logger
+      log.item "Transfer : #{name}"
+      config = get_config
+      if options[:table] then
+        table = TTY::Table.new do |t|
+          t << ["Start Date", "End date", "time", "Files count","File count error","Status"]
+          t << ['','','','','','']
+          TxRecords::new(name).get_all_records.each do |record,value|
+            start_date = record
+            end_date = (value[:end_date].nil?)? '': value[:end_date]
+            time  = (value[:time].nil?)? '': value[:time]
+            count = (value[:count].nil?)? '': value[:count]
+            failed = (value[:count].nil? or value[:done].nil?)? '': value[:count].to_i - value[:done].count
+            status = value[:status]
+            t << [start_date, end_date, time, count, failed, status]
+
+          end
+        end
+        if check_unicode_term  then
+          puts table.render(:unicode)
+        else
+          puts table.render(:ascii)
+        end
+
+      else
+        TxRecords::new(name).get_all_records.each do |record,value|
+          failed = (value[:count].nil? or value[:done].nil?)? 'undef': value[:count].to_i - value[:done].count
+          if value[:end_date].nil? then
+            log.item "Event : #{record} STATUS : #{value[:status]}"
+          else
+            log.item "Tx Begin : #{record} => end : #{value[:end_date]} STATUS : #{value[:status]}"
+          end
+          log.arrow "Tx Time : #{value[:time]}" unless value[:time].nil?
+          log.arrow "nb files : #{value[:count]}" unless value[:count].nil?
+          unless value[:wanted].nil?
+            log.arrow "Files wanted :" unless value[:wanted].empty?
+            value[:wanted].each do |file|
+              log.flat  "    * #{file}"
+            end
+          end
+          unless value[:done].nil?
+            log.arrow "Files done :" unless value[:done].empty?
+            value[:done].each do |file|
+              log.flat  "    * #{file}"
+            end
+          end
+          unless failed then
+            log.arrow "Nb failure : #{failed}"
+          end
+
+        end
+      end
+      splash_exit case: :quiet_exit
+    end
+
+
+    # Thor method : display a specific Splash configured transfer
+    desc "show TRANSFER", "show Splash configured transfer TRANSFER"
+    def show(transfer)
+      log = get_logger
+      transfer_record_set = get_config.transfers.select{|item| item[:name] == transfer.to_sym }
+      unless transfer_record_set.empty? then
+        record = transfer_record_set.first
+        log.info "Splash transfer : #{record[:name]}"
+        log.item "Description : /#{record[:desc]}/"
+        log.item "Type : #{record[:type].to_s}"
+        log.item "Backup file after copy : #{record[:backup].to_s}"
+        log.item "Local spool"
+        log.arrow "Path : #{record[:local][:path]}"
+        log.arrow "User : #{record[:local][:user]}"
+        log.item "Remote spool"
+        log.arrow "Path : #{record[:remote][:path]}"
+        log.arrow "User : #{record[:remote][:user]}"
+        log.arrow "Host : #{record[:remote][:host]}"
+        log.item "Post execution"
+        log.arrow "Remote command : #{record[:post][:remote_command]}" unless record[:post][:remote_command].nil?
+        log.arrow "Local command : #{record[:post][:local_command]}" unless record[:post][:local_command].nil?
+        splash_exit case: :quiet_exit
+      else
+        splash_exit case: :not_found, :more => "log not configured"
+      end
+    end
+
+    # Thor method : display the full list of Splash configured transfers
+    desc "list", "List all Splash configured transfers"
+    long_desc <<-LONGDESC
+    Show configured transfers\n
+    with --detail, show transfers details
+    LONGDESC
+    option :detail, :type => :boolean,  :aliases => "-D"
+    def list
+      log = get_logger
+      log.info "Splash configured transfer :"
+      tx_record_set = get_config.transfers
+      log.ko 'No configured transfers found' if tx_record_set.empty?
+      tx_record_set.each do |record|
+        log.item "Transfer : #{record[:name]} Description : #{record[:desc]}"
+        if options[:detail] then
+          log.arrow "Type : #{record[:type].to_s}"
+          log.arrow "Backup file after copy : #{record[:backup].to_s}"
+          log.arrow "Local spool"
+          log.flat "   * Path : #{record[:local][:path]}"
+          log.flat "   * User : #{record[:local][:user]}"
+          log.arrow "Remote spool"
+          log.flat "   * Path : #{record[:remote][:path]}"
+          log.flat "   * User : #{record[:remote][:user]}"
+          log.flat "   * Host : #{record[:remote][:host]}"
+          log.arrow "Post execution"
+          log.flat "   * Remote command : #{record[:post][:remote_command]}" unless record[:post][:remote_command].nil?
+          log.flat "   * Local command : #{record[:post][:local_command]}" unless record[:post][:local_command].nil?
+        end
+      end
+      splash_exit case: :quiet_exit
+    end
+
+  end
+
+end

data/lib/splash/cli/webadmin.rb

@@ -14,14 +14,14 @@ module CLISplash
     # Thor method : stopping Splash Webadmin
     desc "stop", "Stopping Splash Webadmin Daemon"
     def stop
-      acase = run_as_root :stopdaemon, options
+      acase = run_as_root :stopweb, options
       splash_exit acase
     end
 
     # Thor method : getting execution status of Splashd
     desc "status", "Splash WebAdmin Daemon status"
     def status
-      acase = run_as_root :statusdaemon, options
+      acase = run_as_root :statusweb, options
       splash_exit acase
     end
 
@@ -37,7 +37,7 @@ module CLISplash
     LONGDESC
     desc "start", "Splash WebAdmin Daemon status"
     def start
-      acase = run_as_root :startdaemon
+      acase = run_as_root :startweb
       splash_exit acase
     end
 

data/lib/splash/config.rb

@@ -24,11 +24,15 @@ module Splash
 
         self[:prometheus_url] = (config_from_file[:prometheus][:url])? config_from_file[:prometheus][:url] : PROMETHEUS_URL
         self[:prometheus_pushgateway_url] = (config_from_file[:prometheus][:pushgateway])? config_from_file[:prometheus][:pushgateway] : PROMETHEUS_PUSHGATEWAY_URL
+        self[:prometheus_alertmanager_url] = (config_from_file[:prometheus][:alertmanager])? config_from_file[:prometheus][:alertmanager] : PROMETHEUS_ALERTMANAGER_URL
 
         self[:daemon_process_name] = (config_from_file[:daemon][:process_name])? config_from_file[:daemon][:process_name] : DAEMON_PROCESS_NAME
         self[:daemon_logmon_scheduling] = (config_from_file[:daemon][:logmon_scheduling])? config_from_file[:daemon][:logmon_scheduling] : DAEMON_LOGMON_SCHEDULING
         self[:daemon_metrics_scheduling] = (config_from_file[:daemon][:metrics_scheduling])? config_from_file[:daemon][:metrics_scheduling] : DAEMON_METRICS_SCHEDULING
         self[:daemon_procmon_scheduling] = (config_from_file[:daemon][:procmon_scheduling])? config_from_file[:daemon][:procmon_scheduling] : DAEMON_PROCMON_SCHEDULING
+        self[:daemon_pid_file] = (config_from_file[:daemon][:files][:pid_file])? config_from_file[:daemon][:files][:pid_file] : DAEMON_PID_FILE
+        self[:daemon_stdout_trace] = (config_from_file[:daemon][:files][:stdout_trace])? config_from_file[:daemon][:files][:stdout_trace] : DAEMON_STDOUT_TRACE
+        self[:daemon_stderr_trace] = (config_from_file[:daemon][:files][:stderr_trace])? config_from_file[:daemon][:files][:stderr_trace] : DAEMON_STDERR_TRACE
 
 
         self[:webadmin_port] = (config_from_file[:webadmin][:port])? config_from_file[:webadmin][:port] : WEBADMIN_PORT
@@ -40,15 +44,12 @@ module Splash
         self[:webadmin_stderr_trace] = (config_from_file[:webadmin][:files][:stderr_trace])? config_from_file[:webadmin][:files][:stderr_trace] : WEBADMIN_STDERR_TRACE
 
 
-        self[:pid_path] = (config_from_file[:daemon][:paths][:pid_path])? config_from_file[:daemon][:paths][:pid_path] : PID_PATH
-        self[:trace_path] = (config_from_file[:daemon][:paths][:trace_path])? config_from_file[:daemon][:paths][:trace_path] : TRACE_PATH
+        self[:pid_path] = (config_from_file[:paths][:pid_path])? config_from_file[:paths][:pid_path] : PID_PATH
+        self[:trace_path] = (config_from_file[:paths][:trace_path])? config_from_file[:paths][:trace_path] : TRACE_PATH
 
 
         self[:execution_template_tokens] = EXECUTION_TEMPLATE_TOKENS_LIST
         self[:execution_template_path] = (config_from_file[:templates][:execution][:path])? config_from_file[:templates][:execution][:path] : EXECUTION_TEMPLATE
-        self[:pid_file] = (config_from_file[:daemon][:files][:pid_file])? config_from_file[:daemon][:files][:pid_file] : DAEMON_PID_FILE
-        self[:stdout_trace] = (config_from_file[:daemon][:files][:stdout_trace])? config_from_file[:daemon][:files][:stdout_trace] : DAEMON_STDOUT_TRACE
-        self[:stderr_trace] = (config_from_file[:daemon][:files][:stderr_trace])? config_from_file[:daemon][:files][:stderr_trace] : DAEMON_STDERR_TRACE
 
         self[:transports] = {} ; self[:transports].merge! TRANSPORTS_STRUCT ; self[:transports].merge! config_from_file[:transports] if config_from_file[:transports]
         self[:backends] = {} ; self[:backends].merge! BACKENDS_STRUCT ; self[:backends].merge! config_from_file[:backends] if config_from_file[:backends]
@@ -58,6 +59,7 @@ module Splash
         self[:logs] = (config_from_file[:logs])? config_from_file[:logs] : {}
         self[:commands] = (config_from_file[:commands])? config_from_file[:commands] : {}
         self[:sequences] = (config_from_file[:sequences])? config_from_file[:sequences] : {}
+        self[:transfers] = (config_from_file[:transfers])? config_from_file[:transfers] : {}
 
       end
 
@@ -87,6 +89,12 @@ module Splash
         return self[:transports]
       end
 
+      # getter for transfers Hash Config sample
+      # @return [Hash]
+      def transfers
+        return self[:transfers]
+      end
+
       # getter for daemon_logmon_scheduling Hash Config sample
       # @return [Hash]
       def daemon_logmon_scheduling
@@ -105,6 +113,31 @@ module Splash
         return self[:daemon_metrics_scheduling]
       end
 
+      # getter for daemon_process_name Config sample
+      # @return [String]
+      def daemon_process_name
+        return self[:daemon_process_name]
+      end
+
+
+      # getter for daemon_full_pid_path Config sample
+      # @return [String]
+      def daemon_full_pid_path
+        return "#{self[:pid_path]}/#{self[:daemon_pid_file]}"
+      end
+
+      # getter for daemon_full_stdout_trace_path Config sample
+      # @return [String]
+      def daemon_full_stdout_trace_path
+        return "#{self[:trace_path]}/#{self[:daemon_stdout_trace]}"
+      end
+
+      # getter for daemon_full_stderr_trace_path Config sample
+      # @return [String]
+      def daemon_full_stderr_trace_path
+        return "#{self[:trace_path]}/#{self[:daemon_stderr_trace]}"
+      end
+
       # getter for execution_template_path Hash Config sample
       # @return [String]
       def execution_template_path
@@ -215,36 +248,17 @@ module Splash
         return self[:prometheus_pushgateway_url]
       end
 
-      # getter for prometheus_url Config sample
+      # getter for prometheus_alertmanager_url Config sample
       # @return [String]
-      def prometheus_url
-        return self[:prometheus_url]
+      def prometheus_alertmanager_url
+        return self[:prometheus_alertmanager_url]
       end
 
 
-      # getter for daemon_process_name Config sample
-      # @return [String]
-      def daemon_process_name
-        return self[:daemon_process_name]
-      end
-
-
-      # getter for daemon_full_pid_path Config sample
-      # @return [String]
-      def daemon_full_pid_path
-        return "#{self[:pid_path]}/#{self[:pid_file]}"
-      end
-
-      # getter for daemon_full_stdout_trace_path Config sample
-      # @return [String]
-      def daemon_full_stdout_trace_path
-        return "#{self[:trace_path]}/#{self[:stdout_trace]}"
-      end
-
-      # getter for daemon_full_stderr_trace_path Config sample
+      # getter for prometheus_url Config sample
       # @return [String]
-      def daemon_full_stderr_trace_path
-        return "#{self[:trace_path]}/#{self[:stderr_trace]}"
+      def prometheus_url
+        return self[:prometheus_url]
       end
 
       # @!endgroup