prometheus-splash 0.6.0 → 0.8.2

data/lib/splash/transfers.rb

@@ -0,0 +1,229 @@
+# coding: utf-8
+
+# base Splash Module
+module Splash
+
+  # Transfers module
+  module Transfers
+
+    include Splash::Config
+    include Splash::Loggers
+    include Splash::Helpers
+
+
+
+    class TxNotifier
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_nbfiles = Prometheus::Client::Gauge.new(:txnbfiles, docstring: 'SPLASH metric transfer number of files')
+      @@metric_nbfiles_failed = Prometheus::Client::Gauge.new(:txnbfilesfailed, docstring: 'SPLASH metric transfer number of failed files')
+      @@metric_time = Prometheus::Client::Gauge.new(:txtime, docstring: 'SPLASH metric transfer execution time')
+      @@registry.register(@@metric_nbfiles)
+      @@registry.register(@@metric_nbfiles_failed)
+      @@registry.register(@@metric_time)
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = "tx_#{options[:name].to_s}"
+        @nbfiles = options[:nbfiles]
+        @nbfiles_failed = options[:nbfiles_failed]
+        @time = options[:time]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+	unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_nbfiles.set(@nbfiles)
+        @@metric_nbfiles_failed.set(@nbfiles_failed)
+        @@metric_time.set(@time)
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new(@name, hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+
+    class TxRecords
+      include Splash::Backends
+      include Splash::Constants
+
+      def initialize(name)
+        @name = name
+        @backend = get_backend :transfers_trace
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+      def check_prepared
+        return :never_run_prepare unless @backend.exist?({key: @name})
+        return :never_prepare unless YAML::load(@backend.get({key: @name})).select {|item|
+          record =item.keys.first
+          value=item[record]
+          value[:status] == :prepared
+        }.count > 0
+        return :prepared
+      end
+
+    end
+
+    def run_txs(options = {})
+      log = get_logger
+      log.info 'Running Transfers'
+      count=0
+      get_config.transfers.each do |record|
+        txrec =  TxRecords::new record[:name]
+        txrec.purge(record[:retention])
+        log.item "Execute : #{record[:name]},  #{record[:desc]}"
+        case txrec.check_prepared
+        when :prepared
+          if record[:type] == :push then
+            unless push record
+              count += 1
+            end
+          elsif record[:type] == :pull then
+            unless pull record
+              count += 1
+            end
+          else
+            log.ko "Transfer type unkown"
+            count += 1
+          end
+        when :never_prepare
+          log.ko "#{record[:name]} : Never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        when :never_run_prepare
+          log.ko "#{record[:name]} : Never Executed and never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        end
+      end
+      return {:case => :error_exit, :more => "#{count} Transfer(s) failed"} if count > 0
+      return {:case => :quiet_exit }
+    end
+
+
+
+    def prepare_tx(name)
+      log = get_logger
+      record = get_config.transfers.select { |item| item[:name] == name.to_sym }.first
+      home  = Etc.getpwnam(record[:local][:user]).dir
+      identity = ::File::readlines("#{home}/.ssh/id_rsa.pub").first.chomp
+      folder  = {:mode => "755",
+                 :owner => record[:local][:user] ,
+                 :group => Etc.getgrgid(Etc.getpwnam(record[:local][:user]).gid).name,
+                 :name => record[:local][:path],
+                 :path => record[:local][:path]}
+      log.info "Ensure local folder : #{record[:local][:path]}"
+      make_folder(folder) unless verify_folder(folder).empty?
+      begin
+        log.info "Ensure RSA Key sharing for local user : #{record[:local][:user]} to remote user : #{record[:remote][:user]}@#{record[:remote][:host]}"
+        ssh = Net::SSH.start(record[:remote][:host],record[:remote][:user])
+        output = ssh.exec!(%[
+          /bin/bash -cl '
+          umask 077;
+          mkdir #{record[:remote][:path]};
+          test -d ~/.ssh || mkdir ~/.ssh;
+          if [ ! -f ~/.ssh/authorized_keys -o `grep "#{identity}" ~/.ssh/authorized_keys 2> /dev/null | wc -l` -eq 0 ]; then              echo "#{identity}" >> ~/.ssh/authorized_keys
+          fi'])
+        log.info "Prepare remote folder : #{record[:remote][:path]}"
+        log.info "Prepare data file for transfer : #{record[:name]}"
+        txrec = TxRecords::new record[:name]
+        txrec.add_record :status => :prepared
+        log.ok "Transfer : #{record[:name]} prepared successfully"
+        return {:case => :quiet_exit }
+      rescue Interrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      rescue TTY::Reader::InputInterrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      end
+    end
+
+
+
+    def save_data
+
+    end
+
+
+
+    def push(record)
+      config = get_config
+      log = get_logger
+      txrec = TxRecords::new record[:name]
+      start = Time.now
+      res = true
+      count = 0
+      done =[]
+      start_date = DateTime.now.to_s
+      list = Dir.glob("#{record[:local][:path]}/#{record[:pattern]}")
+      count = list.count
+      log.arrow "Transfering #{count} file(s)"
+
+      begin
+        scp = Net::SCP.start(record[:remote][:host],record[:remote][:user])
+        list.each do|f|
+          log.arrow "Copy file : #{f} to #{record[:remote][:user]}@#{record[:remote][:host]}:#{record[:remote][:path]}"
+          scp.upload! f, record[:remote][:path]
+          done.push f
+          if record[:backup] then
+            log.arrow "File #{f} backuped"
+            FileUtils::mv(f, "#{f}.#{Time.now.getutc.to_i}")
+          else
+            FileUtils::unlink(f)
+          end
+        end
+
+      rescue
+        res = false
+      end
+
+      end_date = DateTime.now.to_s
+      time = Time.now - start
+      status = (res)? :success : :failure
+      txrec.add_record :status => status,
+                       :end_date => end_date,
+                       :time => time,
+                       :count => count,
+                       :wanted => list,
+                       :done => done
+      count_failed = list.count - done.count
+      txmonitor = TxNotifier::new({name: record[:name], nbfiles: count,nbfiles_failed: count_failed, time: time})
+      if txmonitor.notify then
+        log.ok "Sending metrics to Prometheus Pushgateway"
+      else
+        log.ko "Failed to send metrics to Prometheus Pushgateway"
+      end
+      return res
+    end
+  end
+end

data/lib/splash/transports/rabbitmq.rb

@@ -100,7 +100,6 @@ module Splash
           condition = ConditionVariable.new
           get_default_subscriber(queue: queue).subscribe do |delivery_info, properties, payload|
             res = YAML::load(payload)
-
             lock.synchronize { condition.signal }
           end
           get_logger.send "Verb : #{order[:verb].to_s} to queue : #{order[:queue]}."

data/lib/splash/webadmin.rb

@@ -21,7 +21,7 @@ module Splash
       # @param [Hash] options
       # @option options [Symbol] :quiet activate quiet mode for log (limit to :fatal)
       # @return [Hash] Exiter Case (:quiet_exit, :already_exist, :unknown_error or other)
-      def startdaemon(options = {})
+      def startweb(options = {})
         require 'splash/webadmin/main'
         config = get_config
         log = get_logger
@@ -29,7 +29,7 @@ module Splash
         realpid = get_processes pattern: get_config.webadmin_process_name
 
 
-        unless File::exist? config.full_pid_path then
+        unless File::exist? config.webadmin_full_pid_path then
           unless realpid.empty? then
             return {:case => :already_exist, :more => "Splash WebAdmin Process already launched "}
           end
@@ -64,7 +64,7 @@ module Splash
       # @param [Hash] options
       # @option options [Symbol] :quiet activate quiet mode for log (limit to :fatal)
       # @return [Hash] Exiter Case (:quiet_exit, :not_found, other)
-      def stopdaemon(options = {})
+      def stopweb(options = {})
           config = get_config
           log = get_logger
           log.level = :fatal if options[:quiet]
@@ -86,7 +86,7 @@ module Splash
       # Status of the Splash WebAdmin, display status
       # @param [Hash] options ignored
       # @return [Hash] Exiter Case (:status_ko, :status_ok)
-      def statusdaemon(options = {})
+      def statusweb(options = {})
         log = get_logger
         config = get_config
         pid = realpid = ''

data/lib/splash/webadmin/api/routes/commands.rb

@@ -5,7 +5,7 @@
 WebAdminApp.get '/api/commands/list.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : commands, verb : GET, route : list, format : #{format}"
+  log.call "API : commands, verb : GET, route : list, format : #{format}"
   obj =  splash_return case: :quiet_exit, :more => "Commands list"
   obj[:data] = get_config.commands
   content_type format
@@ -15,7 +15,7 @@ WebAdminApp.get '/api/commands/list.?:format?' do
 WebAdminApp.get '/api/commands/show/:name.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : commands, verb : GET, route : show, item : #{params[:name]} , format : #{format}"
+  log.call "API : commands, verb : GET, route : show, item : #{params[:name]} , format : #{format}"
   commands_recordset = get_config.commands[params[:name].to_sym]
   unless commands_recordset.nil? then
     obj = splash_return case: :quiet_exit

data/lib/splash/webadmin/api/routes/config.rb

@@ -1,10 +1,61 @@
 WebAdminApp.get '/api/config/full.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : config, verb : GET, route : show, format : #{format}"
+  log.call "API : config, verb : GET, route : full, format : #{format}"
   config = get_config.full
-  obj =  splash_return case: :quiet_exit, :more => "logses list"
+  obj =  splash_return case: :quiet_exit, :more => "Show internal Splash Config"
   obj[:data] = config
   content_type format
   format_response(obj, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
 end
+
+WebAdminApp.get '/api/config/fromfile.?:format?' do
+  log = get_logger
+  fromfile = {}
+  format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
+  log.call "API : config, verb : GET, route : fromfile, format : #{format}"
+  config = get_config.config_from_file
+  fromfile =  splash_return case: :quiet_exit, :more => "Show config from file"
+  fromfile[:data] = config
+  content_type format
+  format_response(fromfile, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+end
+
+
+WebAdminApp.post '/api/config/addlog.?:format?' do
+  log = get_logger
+  addlog = {}
+  format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
+  log.call "API : config, verb : POST, route : addlog, format : #{format}"
+  res = get_config.add_log :record => YAML::load(request.body.read), :type => :logs, :clean => true
+  case res[:status]
+  when :success
+    addlog = splash_return case: :quiet_exit, :more => "add logs"
+  when :already_exist
+    addlog = splash_return case: :already_exist, :more => "add logs"
+  when :failure
+    addlog = splash_return case: :configuration_error, :more => "add logs"
+    addlog[:data] = res
+  end
+  content_type format
+  format_response(addlog, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+end
+
+
+WebAdminApp.delete '/api/config/deletelog/:label.?:format?' do
+  log = get_logger
+  format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
+  log.call "API : config, verb : DELETE, route : deletelog, format : #{format}"
+  res = get_config.delete_log label: params[:label].to_sym
+  deletelog = {}
+  case res[:status]
+  when :success
+    deletelog = splash_return case: :quiet_exit, :more => "delete logs"
+  when :not_found
+    deletelog = splash_return case: :not_found, :more => "delete logs"
+  else
+    deletelog = splash_return case: :configuration_error, :more => "delete logs"
+  end
+  content_type format
+  format_response(deletelog, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+end

data/lib/splash/webadmin/api/routes/logs.rb

@@ -4,56 +4,71 @@
 
 WebAdminApp.get '/api/logs/list.?:format?' do
   log = get_logger
+  list = {}
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : logs, verb : GET, route : list, format : #{format}"
+  log.call "API : logs, verb : GET, route : list, format : #{format}"
   logs_recordset = get_config.logs
-  obj =  splash_return case: :quiet_exit, :more => "logses list"
-  obj[:data] = logs_recordset
+  list =  splash_return case: :quiet_exit, :more => "logs list"
+  list[:data] = logs_recordset
   content_type format
-  format_response(obj, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+  format_response(list, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
   end
 
 WebAdminApp.get '/api/logs/show/:name.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : logs, verb : GET, route : show, item : #{params[:name]} , format : #{format}"
+  log.call "API : logs, verb : GET, route : show, item : #{params[:name]} , format : #{format}"
   logs_recordset = get_config.logs.select{|item| item[:label] == params[:name].to_sym }
+  show = {}
   unless logs_recordset.empty? then
     record = logs_recordset.first
-    obj = splash_return case: :quiet_exit
-    obj[:data] = record
+    show = splash_return case: :quiet_exit
+    show[:data] = record
+    status 201
   else
-    obj = splash_return case: :not_found, :more => "logs not configured"
+    show = splash_return case: :not_found, :more => "logs not configured"
   end
-  format_response(obj, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+  format_response(show, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
 end
 
 WebAdminApp.post '/api/logs/analyse.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : logs, verb : POST, route : analyse, format : #{format}"
+  log.call "API : logs, verb : POST, route : analyse, format : #{format}"
   results = Splash::Logs::LogScanner::new
   results.analyse
   res = results.output
-  obj =  splash_return case: :quiet_exit, :more => "logs analyse report"
-  obj[:data] = res
+  analyse =  splash_return case: :quiet_exit, :more => "logs analyse report"
+  analyse[:data] = res
   status 201
   content_type format
-  format_response(obj, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+  format_response(analyse, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
 end
 
 WebAdminApp.post '/api/logs/monitor.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : logs, verb : POST, route : monitor, format : #{format}"
+  log.call "API : logs, verb : POST, route : monitor, format : #{format}"
   results = Splash::Logs::LogScanner::new
   results.analyse
-  res = splash_return results.notify
-  if res[:status] == :failure then
+  monitor = splash_return results.notify
+  if monitor[:status] == :failure then
     status 503
   else
     status 201
   end
   content_type format
-  format_response(res, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+  format_response(monitor, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
+end
+
+WebAdminApp.get '/api/logs/history/:label.?:format?' do
+  log = get_logger
+  format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
+  log.call "API : logs, verb : GET, route : history, format : #{format}"
+  record = Splash::Logs::LogsRecords::new(params[:label]).get_all_records
+  history =  splash_return case: :quiet_exit, :more => "log monitoring history"
+  history[:data] = record
+  content_type format
+  status 201
+  format_response(history, (params[:format])? format_by_extensions(params[:format]): request.accept.first)
 end

data/lib/splash/webadmin/api/routes/process.rb

@@ -5,7 +5,7 @@
 WebAdminApp.get '/api/process/list.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : process, verb : GET, route : list, format : #{format}"
+  log.call "API : process, verb : GET, route : list, format : #{format}"
   process_recordset = get_config.processes
   obj =  splash_return case: :quiet_exit, :more => "Processes list"
   obj[:data] = process_recordset
@@ -16,7 +16,7 @@ WebAdminApp.get '/api/process/list.?:format?' do
 WebAdminApp.get '/api/process/show/:name.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : process, verb : GET, route : show, item : #{params[:name]} , format : #{format}"
+  log.call "API : process, verb : GET, route : show, item : #{params[:name]} , format : #{format}"
   process_recordset = get_config.processes.select{|item| item[:process] == params[:name] }
   unless process_recordset.empty? then
     record = process_recordset.first
@@ -32,7 +32,7 @@ end
 WebAdminApp.post '/api/process/analyse.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : process, verb : POST, route : analyse, format : #{format}"
+  log.call "API : process, verb : POST, route : analyse, format : #{format}"
   results = Splash::Processes::ProcessScanner::new
   results.analyse
   res = results.output
@@ -46,7 +46,7 @@ end
 WebAdminApp.post '/api/process/monitor.?:format?' do
   log = get_logger
   format = (params[:format])? format_by_extensions(params[:format]) : format_by_extensions('json')
-  log.call "api : process, verb : POST, route : monitor, format : #{format}"
+  log.call "API : process, verb : POST, route : monitor, format : #{format}"
   results = Splash::Processes::ProcessScanner::new
   results.analyse
   res = splash_return results.notify