prometheus-splash 0.5.3 → 0.8.1

data/lib/splash/sequences.rb

@@ -0,0 +1,105 @@
+# coding: utf-8
+
+# base Splash module
+module Splash
+
+  # Splash Commands module/namespace
+  module Sequences
+
+    include Splash::Commands
+    include Splash::Config
+    include Splash::Loggers
+    include Splash::Exiter
+    include Splash::Transports
+
+    def run_seq(params = {})
+      list = get_config.sequences
+      name = params[:name].to_sym
+      acase = {}
+      seq_res = []
+      log = get_logger
+      log.info "beginnning Sequence execution : #{name}"
+      session = (params[:session])? params[:session] : get_session
+      if list.include? name then
+        seq_options = (list[name][:options].nil?)? {} : list[name][:options]
+        list[name][:definition].each do |step|
+          log.info "STEP : #{step[:step]}",session
+          if step[:on_host].nil? then
+            command =  CommandWrapper::new(step[:command].to_s)
+              step[:callback] = true if step[:callback].nil?
+              step[:trace] = true if step[:trace].nil?
+              step[:notify] = true if step[:notify].nil?
+              step[:session] = session
+              acase = command.call_and_notify step
+          else
+            log.info "Remote execution of #{step[:command]} on #{step[:on_host]}", session
+            begin
+              transport = get_default_client
+              if transport.class == Hash  and transport.include? :case then
+                return transport
+              else
+                acase = transport.execute({ :verb => :execute_command,
+                                          payload: {:name => step[:command].to_s},
+                                         :return_to => "splash.#{Socket.gethostname}.return",
+                                         :queue => "splash.#{step[:on_host]}.input" })
+                log.receive "return with exitcode #{acase[:exit_code]}", session
+              end
+            rescue Interrupt
+              acase = { case: :interrupt, more: "Remote command exection", exit_code: 33}
+            end
+
+           end
+           seq_res.push acase[:exit_code]
+           continue = (seq_options[:continue].nil?)? true : seq_options[:continue]
+           if acase[:exit_code] > 0 and continue == false then
+             acase[:more] = "Break execution on error, continue = false"
+             break
+           end
+        end
+      else
+        return  { :case => :not_found, :more => "Sequence #{name} not configured" }
+      end
+
+      if seq_res.select {|res| res > 0}.count == seq_res.count then
+        acase =  { case: :status_ko, more: "all execution failed" }
+      elsif seq_res.select {|res| res > 0}.count > 0 then
+        acase =  { case: :status_ko, more: "some execution failed" }
+      else
+        acase =  { case: :status_ok, more: "all execution successed" }
+      end
+      acase[:more] << " with remote call interrupt" if seq_res.include?(33)
+      return acase
+    end
+
+
+    def list_seq(options = {})
+      list = get_config.sequences
+      unless list.empty?
+        acase = { :case =>  :quiet_exit , :more => "List configured sequences"}
+        acase[:data] = list
+        return acase
+      else
+        return { :case => :not_found, :more => 'No sequences configured' }
+      end
+    end
+
+    def show_seq(options = {})
+      list = get_config.sequences
+      name = options[:name].to_sym
+      if list.include? name then
+        acase = { :case => :quiet_exit, :more => "Show specific sequence : #{name}"}
+        acase[:data] = list[name]
+      else
+        return { :case => :not_found, :more => "Sequence #{name} not configured" }
+      end
+      return acase
+    end
+
+    def schedule_seq(options = {})
+      acase = { :case => :quiet_exit,  :more => "schedule" }
+      return acase
+
+    end
+
+  end
+end

data/lib/splash/templates.rb

@@ -1,12 +1,19 @@
 # coding: utf-8
+
+# base Splash Module
 module Splash
+
+  # Templates namespace
   module Templates
 
     # KISS template Engine
     class Template
 
+      # getter of the list of token
       attr_reader :list_token
+      # getter of the template file
       attr_reader :template_file
+      # getter of the flat content of the template
       attr_reader :content
 
       # constructor : generate the pseudo accessor for template Class from token list
@@ -72,8 +79,11 @@ module Splash
 
     end
 
+    # Exception for an invalid Token list
     class InvalidTokenList < Exception; end
+    # Exception for an malformed token
     class NotAToken < Exception; end
+    # Exception for an invalid template file
     class NoTemplateFile < Exception; end
 
   end

data/lib/splash/transfers.rb

@@ -0,0 +1,229 @@
+# coding: utf-8
+
+# base Splash Module
+module Splash
+
+  # Transfers module
+  module Transfers
+
+    include Splash::Config
+    include Splash::Loggers
+    include Splash::Helpers
+
+
+
+    class TxNotifier
+      @@registry = Prometheus::Client::Registry::new
+      @@metric_nbfiles = Prometheus::Client::Gauge.new(:txnbfiles, docstring: 'SPLASH metric transfer number of files')
+      @@metric_nbfiles_failed = Prometheus::Client::Gauge.new(:txnbfilesfailed, docstring: 'SPLASH metric transfer number of failed files')
+      @@metric_time = Prometheus::Client::Gauge.new(:txtime, docstring: 'SPLASH metric transfer execution time')
+      @@registry.register(@@metric_nbfiles)
+      @@registry.register(@@metric_nbfiles_failed)
+      @@registry.register(@@metric_time)
+
+      def initialize(options={})
+        @config = get_config
+        @url = @config.prometheus_pushgateway_url
+        @name = "tx_#{options[:name].to_s}"
+        @nbfiles = options[:nbfiles]
+        @nbfiles_failed = options[:nbfiles_failed]
+        @time = options[:time]
+      end
+
+      # send metrics to Prometheus PushGateway
+      # @return [Bool]
+      def notify
+	unless verify_service url: @url then
+          return { :case => :service_dependence_missing, :more => "Prometheus Notification not send."}
+        end
+        @@metric_nbfiles.set(@nbfiles)
+        @@metric_nbfiles_failed.set(@nbfiles_failed)
+        @@metric_time.set(@time)
+        hostname = Socket.gethostname
+        return Prometheus::Client::Push.new(@name, hostname, @url).add(@@registry)
+      end
+
+    end
+
+
+
+    class TxRecords
+      include Splash::Backends
+      include Splash::Constants
+
+      def initialize(name)
+        @name = name
+        @backend = get_backend :transfers_trace
+      end
+
+      def purge(retention)
+        retention = {} if retention.nil?
+        if retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:hours].to_f / 24
+        elsif retention.include? :hours then
+          adjusted_datetime = DateTime.now - retention[:days].to_i
+        else
+          adjusted_datetime = DateTime.now - DEFAULT_RETENTION
+        end
+
+        data = get_all_records
+
+        data.delete_if { |item|
+          DateTime.parse(item.keys.first) <= (adjusted_datetime)}
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def add_record(record)
+        data = get_all_records
+        data.push({ DateTime.now.to_s => record })
+        @backend.put key: @name, value: data.to_yaml
+      end
+
+      def get_all_records(options={})
+        return (@backend.exist?({key: @name}))? YAML::load(@backend.get({key: @name})) : []
+      end
+
+      def check_prepared
+        return :never_run_prepare unless @backend.exist?({key: @name})
+        return :never_prepare unless YAML::load(@backend.get({key: @name})).select {|item|
+          record =item.keys.first
+          value=item[record]
+          value[:status] == :prepared
+        }.count > 0
+        return :prepared
+      end
+
+    end
+
+    def run_txs(options = {})
+      log = get_logger
+      log.info 'Running Transfers'
+      count=0
+      get_config.transfers.each do |record|
+        txrec =  TxRecords::new record[:name]
+        txrec.purge(record[:retention])
+        log.item "Execute : #{record[:name]},  #{record[:desc]}"
+        case txrec.check_prepared
+        when :prepared
+          if record[:type] == :push then
+            unless push record
+              count += 1
+            end
+          elsif record[:type] == :pull then
+            unless pull record
+              count += 1
+            end
+          else
+            log.ko "Transfer type unkown"
+            count += 1
+          end
+        when :never_prepare
+          log.ko "#{record[:name]} : Never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        when :never_run_prepare
+          log.ko "#{record[:name]} : Never Executed and never prepared, ignored"
+          txrec.add_record :status => :never_prepared
+          count += 1
+        end
+      end
+      return {:case => :error_exit, :more => "#{count} Transfer(s) failed"} if count > 0
+      return {:case => :quiet_exit }
+    end
+
+
+
+    def prepare_tx(name)
+      log = get_logger
+      record = get_config.transfers.select { |item| item[:name] == name.to_sym }.first
+      home  = Etc.getpwnam(record[:local][:user]).dir
+      identity = ::File::readlines("#{home}/.ssh/id_rsa.pub").first.chomp
+      folder  = {:mode => "755",
+                 :owner => record[:local][:user] ,
+                 :group => Etc.getgrgid(Etc.getpwnam(record[:local][:user]).gid).name,
+                 :name => record[:local][:path],
+                 :path => record[:local][:path]}
+      log.info "Ensure local folder : #{record[:local][:path]}"
+      make_folder(folder) unless verify_folder(folder).empty?
+      begin
+        log.info "Ensure RSA Key sharing for local user : #{record[:local][:user]} to remote user : #{record[:remote][:user]}@#{record[:remote][:host]}"
+        ssh = Net::SSH.start(record[:remote][:host],record[:remote][:user])
+        output = ssh.exec!(%[
+          /bin/bash -cl '
+          umask 077;
+          mkdir #{record[:remote][:path]};
+          test -d ~/.ssh || mkdir ~/.ssh;
+          if [ ! -f ~/.ssh/authorized_keys -o `grep "#{identity}" ~/.ssh/authorized_keys 2> /dev/null | wc -l` -eq 0 ]; then              echo "#{identity}" >> ~/.ssh/authorized_keys
+          fi'])
+        log.info "Prepare remote folder : #{record[:remote][:path]}"
+        log.info "Prepare data file for transfer : #{record[:name]}"
+        txrec = TxRecords::new record[:name]
+        txrec.add_record :status => :prepared
+        log.ok "Transfer : #{record[:name]} prepared successfully"
+        return {:case => :quiet_exit }
+      rescue Interrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      rescue TTY::Reader::InputInterrupt
+        splash_exit case: :interrupt, more: "Remote command exection"
+      end
+    end
+
+
+
+    def save_data
+
+    end
+
+
+
+    def push(record)
+      config = get_config
+      log = get_logger
+      txrec = TxRecords::new record[:name]
+      start = Time.now
+      res = true
+      count = 0
+      done =[]
+      start_date = DateTime.now.to_s
+      list = Dir.glob("#{record[:local][:path]}/#{record[:pattern]}")
+      count = list.count
+      log.arrow "Transfering #{count} file(s)"
+
+      begin
+        scp = Net::SCP.start(record[:remote][:host],record[:remote][:user])
+        list.each do|f|
+          log.arrow "Copy file : #{f} to #{record[:remote][:user]}@#{record[:remote][:host]}:#{record[:remote][:path]}"
+          scp.upload! f, record[:remote][:path]
+          done.push f
+          if record[:backup] then
+            log.arrow "File #{f} backuped"
+            FileUtils::mv(f, "#{f}.#{Time.now.getutc.to_i}")
+          else
+            FileUtils::unlink(f)
+          end
+        end
+
+      rescue
+        res = false
+      end
+
+      end_date = DateTime.now.to_s
+      time = Time.now - start
+      status = (res)? :success : :failure
+      txrec.add_record :status => status,
+                       :end_date => end_date,
+                       :time => time,
+                       :count => count,
+                       :wanted => list,
+                       :done => done
+      count_failed = list.count - done.count
+      txmonitor = TxNotifier::new({name: record[:name], nbfiles: count,nbfiles_failed: count_failed, time: time})
+      if txmonitor.notify then
+        log.ok "Sending metrics to Prometheus Pushgateway"
+      else
+        log.ko "Failed to send metrics to Prometheus Pushgateway"
+      end
+      return res
+    end
+  end
+end

data/lib/splash/transports.rb

@@ -1,9 +1,16 @@
 # coding: utf-8
 
+# base Splash module
 module Splash
+
+  # Splash Transports namespace
   module Transports
     include Splash::Config
 
+
+    # factory for Splash::Transports::Rabbitmq::Subscriber
+    # @param [Hash] options
+    # @return [Splash::Transports::Rabbitmq::Subscriber|Hash] Subscriber or Exiter case :configuration_error
     def get_default_subscriber(options)
       config = get_config.transports
       transport = config[:active]
@@ -20,6 +27,8 @@ module Splash
       end
     end
 
+    # factory for Splash::Transports::Rabbitmq::Client
+    # @return [Splash::Transports::Rabbitmq::Client|Hash] Client or Exiter case :configuration_error
     def get_default_client
       config = get_config.transports
       transport = config[:active]

data/lib/splash/transports/rabbitmq.rb

@@ -1,14 +1,24 @@
 # coding: utf-8
+
+# base Splash module
 module Splash
+
+  # Splash Transports namespace
   module Transports
+
+    # RabbitMQ Transport
     module Rabbitmq
 
+      # Subscriber Mode RabbitMQ Client
       class Subscriber
         include Splash::Config
         extend Forwardable
 
         def_delegators :@queue, :subscribe
 
+        # Constructor Forward subscribe method and initialize a Bunny Client atribute @queue
+        # @param [Hash] options
+        # @option options [String] :queue the name of the subscribed queue
         def initialize(options = {})
           @config = get_config.transports
 
@@ -32,12 +42,13 @@ module Splash
 
       end
 
-
+      # publish / get Mode RabbitMQ Client
       class Client
         include Splash::Config
         include Splash::Transports
         include Splash::Loggers
 
+        # Constructor initialize a Bunny Client
         def initialize
           @config = get_config.transports
           host = @config[:rabbitmq][:host]
@@ -48,7 +59,7 @@ module Splash
           conf  = { :host => host, :vhost => vhost, :user => user, :password => passwd, :port => port.to_i}
 
           begin
-            @connection = Bunny.new conf 
+            @connection = Bunny.new conf
             @connection.start
             @channel = @connection.create_channel
           rescue Bunny::Exception
@@ -56,19 +67,32 @@ module Splash
           end
         end
 
-
+        # purge a queue
+        # @param [Hash] options
+        # @option options [String] :queue the name of the queue to purge
         def purge(options)
           @channel.queue(options[:queue]).purge
         end
 
+        # publish to a queue
+        # @param [Hash] options
+        # @option options [String] :queue the name of the queue to purge
+        # @option options [String] :message the message to send
         def publish(options ={})
           return @channel.default_exchange.publish(options[:message], :routing_key => options[:queue])
         end
 
+        # ack a specific message for manual ack with a delivery tag to a queue
+        # @param [String] ack
+        # @return [Boolean]
         def ack(ack)
           return @channel.acknowledge(ack, false)
         end
 
+
+        # send an execution order message (verb+payload) via RabbitMQ to an slash input queue
+        # @param [Hash] order
+        # @return [Void] unserialized Void object from YAML
         def execute(order)
           queue = order[:return_to]
           lock = Mutex.new
@@ -76,7 +100,6 @@ module Splash
           condition = ConditionVariable.new
           get_default_subscriber(queue: queue).subscribe do |delivery_info, properties, payload|
             res = YAML::load(payload)
-
             lock.synchronize { condition.signal }
           end
           get_logger.send "Verb : #{order[:verb].to_s} to queue : #{order[:queue]}."
@@ -85,6 +108,11 @@ module Splash
           return res
         end
 
+        # Get a message from a RabbitMQ queue
+        # @param [Hash] options
+        # @option options [String] :queue the name of the queue to query
+        # @option options [String] :manual_ack flag to inhibit ack
+        # @return [Hash] Payload + ack tag if :manual_ack
         def get(options ={})
           queue = @channel.queue(options[:queue])
           opt = {}; opt[:manual_ack] = (options[:manual_ack])? true : false
@@ -94,6 +122,7 @@ module Splash
           return res
         end
 
+        # close the RabbitMQ connection
         def close
           @connection.close
         end