prometheus-splash 0.5.2 → 0.8.0

data/lib/splash/templates.rb

@@ -1,12 +1,19 @@
 # coding: utf-8
+
+# base Splash Module
 module Splash
+
+  # Templates namespace
   module Templates
 
     # KISS template Engine
     class Template
 
+      # getter of the list of token
       attr_reader :list_token
+      # getter of the template file
       attr_reader :template_file
+      # getter of the flat content of the template
       attr_reader :content
 
       # constructor : generate the pseudo accessor for template Class from token list
@@ -72,8 +79,11 @@ module Splash
 
     end
 
+    # Exception for an invalid Token list
     class InvalidTokenList < Exception; end
+    # Exception for an malformed token
     class NotAToken < Exception; end
+    # Exception for an invalid template file
     class NoTemplateFile < Exception; end
 
   end

data/lib/splash/transfers.rb

@@ -0,0 +1,224 @@
+# coding: utf-8
+
+# base Splash Module
+module Splash
+
+  # Transfers module
+  module Transfers
+
+    include Splash::Config
+    include Splash::Loggers
+    include Splash::Helpers
+
+
+
+    class TxNotifier
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_nbfiles = Prometheus::Client::Gauge.new(:txnbfiles, docstring: 'SPLASH metric transfer number of files')
+      @@metric_nbfiles_failed = Prometheus::Client::Gauge.new(:txnbfilesfailed, docstring: 'SPLASH metric transfer number of failed files')
+      @@metric_time = Prometheus::Client::Gauge.new(:txtime, docstring: 'SPLASH metric transfer execution time')
+      @@registry.register(@@metric_nbfiles)
+      @@registry.register(@@metric_nbfiles_failed)
+      @@registry.register(@@metric_time)
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = "tx_#{options[:name].to_s}"
+        @nbfiles = options[:nbfiles]
+        @nbfiles_failed = options[:nbfiles_failed]
+        @time = options[:time]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+	unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_nbfiles.set(@nbfiles)
+        @@metric_nbfiles_failed.set(@nbfiles_failed)
+        @@metric_time.set(@time)
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new(@name, hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+
+    class TxRecords
+      include Splash::Backends
+      def initialize(name)
+        @name = name
+        @backend = get_backend :transferts_trace
+      end
+
+      def purge(retention)
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          retention = TRANSFER_DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item,value|
+          DateTime.parse(item) <= (adjusted_datetime) and value[:status] != :prepared}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data[DateTime.now.to_s] = record
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : {}
+      end
+
+      def check_prepared
+        return :never_run_prepare unless @backend.exist?({key: @name})
+        return :never_prepare unless YAML::load(@backend.get({key: @name})).select {|item,value|
+          value[:status] == :prepared
+        }.count > 0
+        return :prepared
+      end
+
+    end
+
+    def run_txs(options = {})
+      log = get_logger
+      log.info 'Running Transfers'
+      count=0
+      get_config.transfers.each do |record|
+        txrec =  TxRecords::new record[:name]
+        txrec.purge(record[:retention])
+        log.item "Execute : #{record[:name]},  #{record[:desc]}"
+        case txrec.check_prepared
+        when :prepared
+          if record[:type] == :push then
+            unless push record
+              count += 1
+            end
+          elsif record[:type] == :pull then
+            unless pull record
+              count += 1
+            end
+          else
+            log.ko "Transfer type unkown"
+            count += 1
+          end
+        when :never_prepare
+          log.ko "#{record[:name]} : Never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        when :never_run_prepare
+          log.ko "#{record[:name]} : Never Executed and never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        end
+      end
+      return {:case => :error_exit, :more => "#{count} Transfer(s) failed"} if count > 0
+      return {:case => :quiet_exit }
+    end
+
+
+
+    def prepare_tx(name)
+      log = get_logger
+      record = get_config.transfers.select { |item| item[:name] == name.to_sym }.first
+      home  = Etc.getpwnam(record[:local][:user]).dir
+      identity = ::File::readlines("#{home}/.ssh/id_rsa.pub").first.chomp
+      folder  = {:mode => "755",
+                 :owner => record[:local][:user] ,
+                 :group => Etc.getgrgid(Etc.getpwnam(record[:local][:user]).gid).name,
+                 :name => record[:local][:path],
+                 :path => record[:local][:path]}
+      log.info "Ensure local folder : #{record[:local][:path]}"
+      make_folder(folder) unless verify_folder(folder).empty?
+      begin
+        log.info "Ensure RSA Key sharing for local user : #{record[:local][:user]} to remote user : #{record[:remote][:user]}@#{record[:remote][:host]}"
+        ssh = Net::SSH.start(record[:remote][:host],record[:remote][:user])
+        output = ssh.exec!(%[
+          /bin/bash -cl '
+          umask 077;
+          mkdir #{record[:remote][:path]};
+          test -d ~/.ssh || mkdir ~/.ssh;
+          if [ ! -f ~/.ssh/authorized_keys -o `grep "#{identity}" ~/.ssh/authorized_keys 2> /dev/null | wc -l` -eq 0 ]; then              echo "#{identity}" >> ~/.ssh/authorized_keys
+          fi'])
+        log.info "Prepare remote folder : #{record[:remote][:path]}"
+        log.info "Prepare data file for transfer : #{record[:name]}"
+        txrec = TxRecords::new record[:name]
+        txrec.add_record :status => :prepared
+        log.ok "Transfer : #{record[:name]} prepared successfully"
+        return {:case => :quiet_exit }
+      rescue Interrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      rescue TTY::Reader::InputInterrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      end
+    end
+
+
+
+    def save_data
+
+    end
+
+
+
+    def push(record)
+      config = get_config
+      log = get_logger
+      txrec = TxRecords::new record[:name]
+      start = Time.now
+      res = true
+      count = 0
+      done =[]
+      start_date = DateTime.now.to_s
+      list = Dir.glob("#{record[:local][:path]}/#{record[:pattern]}")
+      count = list.count
+      log.arrow "Transfering #{count} file(s)"
+
+      begin
+        scp = Net::SCP.start(record[:remote][:host],record[:remote][:user])
+        list.each do|f|
+          log.arrow "Copy file : #{f} to #{record[:remote][:user]}@#{record[:remote][:host]}:#{record[:remote][:path]}"
+          scp.upload! f, record[:remote][:path]
+          done.push f
+          if record[:backup] then
+            log.arrow "File #{f} backuped"
+            FileUtils::mv(f, "#{f}.#{Time.now.getutc.to_i}")
+          else
+            FileUtils::unlink(f)
+          end
+        end
+
+      rescue
+        res = false
+      end
+
+      end_date = DateTime.now.to_s
+      time = Time.now - start
+      status = (res)? :success : :failure
+      txrec.add_record :status => status,
+                       :end_date => end_date,
+                       :time => time,
+                       :count => count,
+                       :wanted => list,
+                       :done => done
+      count_failed = list.count - done.count
+      txmonitor = TxNotifier::new({name: record[:name], nbfiles: count,nbfiles_failed: count_failed, time: time})
+      if txmonitor.notify then
+        log.ok "Sending metrics to Prometheus Pushgateway"
+      else
+        log.ko "Failed to send metrics to Prometheus Pushgateway"
+      end
+      return res
+    end
+  end
+end

data/lib/splash/transports.rb

@@ -1,9 +1,16 @@
 # coding: utf-8
 
+# base Splash module
 module Splash
+
+  # Splash Transports namespace
   module Transports
     include Splash::Config
 
+
+    # factory for Splash::Transports::Rabbitmq::Subscriber
+    # @param [Hash] options
+    # @return [Splash::Transports::Rabbitmq::Subscriber|Hash] Subscriber or Exiter case :configuration_error
     def get_default_subscriber(options)
       config = get_config.transports
       transport = config[:active]
@@ -20,6 +27,8 @@ module Splash
       end
     end
 
+    # factory for Splash::Transports::Rabbitmq::Client
+    # @return [Splash::Transports::Rabbitmq::Client|Hash] Client or Exiter case :configuration_error
     def get_default_client
       config = get_config.transports
       transport = config[:active]

data/lib/splash/transports/rabbitmq.rb

@@ -1,14 +1,24 @@
 # coding: utf-8
+
+# base Splash module
 module Splash
+
+  # Splash Transports namespace
   module Transports
+
+    # RabbitMQ Transport
     module Rabbitmq
 
+      # Subscriber Mode RabbitMQ Client
       class Subscriber
         include Splash::Config
         extend Forwardable
 
         def_delegators :@queue, :subscribe
 
+        # Constructor Forward subscribe method and initialize a Bunny Client atribute @queue
+        # @param [Hash] options
+        # @option options [String] :queue the name of the subscribed queue
         def initialize(options = {})
           @config = get_config.transports
 
@@ -32,12 +42,13 @@ module Splash
 
       end
 
-
+      # publish / get Mode RabbitMQ Client
       class Client
         include Splash::Config
         include Splash::Transports
         include Splash::Loggers
 
+        # Constructor initialize a Bunny Client
         def initialize
           @config = get_config.transports
           host = @config[:rabbitmq][:host]
@@ -48,7 +59,7 @@ module Splash
           conf  = { :host => host, :vhost => vhost, :user => user, :password => passwd, :port => port.to_i}
 
           begin
-            @connection = Bunny.new conf 
+            @connection = Bunny.new conf
             @connection.start
             @channel = @connection.create_channel
           rescue Bunny::Exception
@@ -56,19 +67,32 @@ module Splash
           end
         end
 
-
+        # purge a queue
+        # @param [Hash] options
+        # @option options [String] :queue the name of the queue to purge
         def purge(options)
           @channel.queue(options[:queue]).purge
         end
 
+        # publish to a queue
+        # @param [Hash] options
+        # @option options [String] :queue the name of the queue to purge
+        # @option options [String] :message the message to send
         def publish(options ={})
           return @channel.default_exchange.publish(options[:message], :routing_key => options[:queue])
         end
 
+        # ack a specific message for manual ack with a delivery tag to a queue
+        # @param [String] ack
+        # @return [Boolean]
         def ack(ack)
           return @channel.acknowledge(ack, false)
         end
 
+
+        # send an execution order message (verb+payload) via RabbitMQ to an slash input queue
+        # @param [Hash] order
+        # @return [Void] unserialized Void object from YAML
         def execute(order)
           queue = order[:return_to]
           lock = Mutex.new
@@ -76,7 +100,6 @@ module Splash
           condition = ConditionVariable.new
           get_default_subscriber(queue: queue).subscribe do |delivery_info, properties, payload|
             res = YAML::load(payload)
-
             lock.synchronize { condition.signal }
           end
           get_logger.send "Verb : #{order[:verb].to_s} to queue : #{order[:queue]}."
@@ -85,6 +108,11 @@ module Splash
           return res
         end
 
+        # Get a message from a RabbitMQ queue
+        # @param [Hash] options
+        # @option options [String] :queue the name of the queue to query
+        # @option options [String] :manual_ack flag to inhibit ack
+        # @return [Hash] Payload + ack tag if :manual_ack
         def get(options ={})
           queue = @channel.queue(options[:queue])
           opt = {}; opt[:manual_ack] = (options[:manual_ack])? true : false
@@ -94,6 +122,7 @@ module Splash
           return res
         end
 
+        # close the RabbitMQ connection
         def close
           @connection.close
         end

data/lib/splash/webadmin.rb

@@ -0,0 +1,122 @@
+# coding: utf-8
+
+
+
+# base Splash module
+module Splash
+
+  # global daemon module
+  module WebAdmin
+
+    # Daemon Controller Module
+    module Controller
+      include Splash::Constants
+      include Splash::Helpers
+      include Splash::Config
+      include Splash::Exiter
+      include Splash::Loggers
+
+
+      # Start the Splash Daemon
+      # @param [Hash] options
+      # @option options [Symbol] :quiet activate quiet mode for log (limit to :fatal)
+      # @return [Hash] Exiter Case (:quiet_exit, :already_exist, :unknown_error or other)
+      def startweb(options = {})
+        require 'splash/webadmin/main'
+        config = get_config
+        log = get_logger
+        log.level = :fatal if options[:quiet]
+        realpid = get_processes pattern: get_config.webadmin_process_name
+
+
+        unless File::exist? config.webadmin_full_pid_path then
+          unless realpid.empty? then
+            return {:case => :already_exist, :more => "Splash WebAdmin Process already launched "}
+          end
+
+          daemon_config = {:description => config.webadmin_process_name,
+              :pid_file => config.webadmin_full_pid_path,
+              :stdout_trace => config.webadmin_full_stdout_trace_path,
+              :stderr_trace => config.webadmin_full_stderr_trace_path
+            }
+
+          ["int","term","hup"].each do |type| daemon_config["sig#{type}_handler".to_sym] = Proc::new {  WebAdminApp.quit! } end
+          res = daemonize daemon_config do
+            log = get_logger logger: :web, force: true
+            log.info "Starting Splash WebAdmin"
+            WebAdminApp.run!
+          end
+          sleep 1
+          if res == 0 then
+            pid = `cat #{config.webadmin_full_pid_path}`.to_i
+            log.ok "Splash WebAdmin Started, with PID : #{pid}"
+            return {:case => :quiet_exit, :more => "Splash WebAdmin successfully loaded."}
+          else
+            return {:case => :unknown_error, :more => "Splash WebAdmin loading error, see logs for more details."}
+          end
+
+        else
+          return {:case => :already_exist, :more => "Pid File, please verify if Splash WebAdmin is running."}
+        end
+      end
+
+      # Stop the Splash WebAdmin
+      # @param [Hash] options
+      # @option options [Symbol] :quiet activate quiet mode for log (limit to :fatal)
+      # @return [Hash] Exiter Case (:quiet_exit, :not_found, other)
+      def stopweb(options = {})
+          config = get_config
+          log = get_logger
+          log.level = :fatal if options[:quiet]
+          if File.exist?(config.webadmin_full_pid_path) then
+            begin
+              pid = `cat #{config.webadmin_full_pid_path}`.to_i
+              Process.kill("TERM", pid)
+              acase = {:case => :quiet_exit, :more => 'Splash WebAdmin stopped succesfully'}
+            rescue Errno::ESRCH
+              acase =  {:case => :not_found, :more => "Process of PID : #{pid} not found"}
+            end
+            FileUtils::rm config.webadmin_full_pid_path if File::exist? config.webadmin_full_pid_path
+          else
+            acase =  {:case => :not_found, :more => "Splash WebAdmin is not running"}
+          end
+          return acase
+      end
+
+      # Status of the Splash WebAdmin, display status
+      # @param [Hash] options ignored
+      # @return [Hash] Exiter Case (:status_ko, :status_ok)
+      def statusweb(options = {})
+        log = get_logger
+        config = get_config
+        pid = realpid = ''
+        pid = `cat #{config.webadmin_full_pid_path}`.to_s if File.exist?(config.webadmin_full_pid_path)
+        listpid = get_processes({ :pattern => get_config.webadmin_process_name})
+        pid.chomp!
+        if listpid.empty? then
+          realpid = ''
+        else
+          realpid = listpid.first
+        end
+        unless realpid.empty? then
+          log.item "Splash  WebAdmin Process is running with PID #{realpid} "
+        else
+          log.item 'Splash  WebAdminProcess not found '
+        end
+        unless pid.empty? then
+          log.item "and PID file exist with PID #{pid}"
+        else
+          log.item "and PID file don't exist"
+        end
+        if pid == realpid then
+          return {:case => :status_ok }
+        elsif pid.empty? then
+          return {:case => :status_ko, :more => "PID File error, you have to kill process manualy, with : '(sudo )kill -TERM #{realpid}'"}
+        elsif realpid.empty? then
+          return {:case => :status_ko, :more => "Process Splash WebAdmin missing, run 'splash webadmin stop' before reload properly"}
+        end
+      end
+
+    end
+  end
+end