cloudscale-monitor 0.0.1

data/lib/cloudscale/plugins/net/sigar_byte_information.rb

@@ -0,0 +1,64 @@
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/chart"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/measurement"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/value"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/statistic"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/unit"
+require "#{File.dirname(__FILE__)}/../preops/plugin_preop"
+require "#{File.dirname(__FILE__)}/../plugin"
+
+##
+#
+# @author Johannes Hiemer, cloudscale.
+#
+##
+module Cloudscale
+  module Plugins
+    class SigarNetByteInformation < Plugins::Plugin
+      attr_reader :menu, :receivedBytes, :sendBytes
+  
+      def log
+        @log = Logger.new(STDOUT)
+      end
+  
+      def is_enabled
+        true
+      end
+  
+      def initialize
+        super
+        if is_enabled
+          menus = Plugins::PluginMenus.load_hash()
+          @menu = load_menu(menus["network"], "sendAndReceivedBytes")
+                   
+          hash = Plugins::PluginComponents.load_hash()
+          @receivedBytes = load_metric(hash["network"], "receivedBytes")
+          @sendBytes = load_metric(hash["network"], "sendBytes")
+          @sigar = Plugins::PluginPreop.instance.sigar
+        end
+        
+      end
+               
+      def collect(agentInstanceId)
+        log.info("Calling Collect on SigarNetByteInformation")
+        iflist = @sigar.net_interface_list
+  
+        iflist.each do |ifname|
+          ifconfig = @sigar.net_interface_config(ifname)
+          if (!ifname.include?(":"))
+            ifstat = @sigar.net_interface_stat(ifname)
+            begin
+              report(receivedBytes, menu, ifstat.rx_bytes, true, agentInstanceId, "receivedBytesChart-" + ifname,
+                "Received bytes " + ifname, "Total number of received bytes" , "area", "Time", "GB", "B", "s")
+    
+              report(sendBytes, menu, ifstat.tx_bytes, true, agentInstanceId, "sendBytesChart-" + ifname,
+                "Send bytes " + ifname, "Total number of send bytes" , "area", "Time", "GB", "B", "s")
+            rescue Exception => e
+              puts e
+            end
+          end
+        end
+      end
+      
+    end
+  end
+end

data/lib/cloudscale/plugins/net/sigar_packet_information.rb

@@ -0,0 +1,84 @@
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/chart"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/measurement"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/value"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/statistic"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/unit"
+require "#{File.dirname(__FILE__)}/../preops/plugin_preop"
+require "#{File.dirname(__FILE__)}/../plugin"
+
+##
+#
+# @author Johannes Hiemer, cloudscale.
+#
+##
+module Cloudscale
+  module Plugins
+    class SigarNetPacketInformation < Plugins::Plugin
+      attr_reader :menu, :receivedPackets, :receivedPacketsError, :receivedPacketsDropped, :sendPackets,
+         :sendPacketsError, :sendPacketsDropped
+  
+      def log
+        @log = Logger.new(STDOUT)
+      end
+  
+      def is_enabled
+        true
+      end
+  
+      def initialize
+        super
+        if is_enabled
+          menus = Plugins::PluginMenus.load_hash()
+          @menu = load_menu(menus["network"], "packetStatistics")
+                    
+          hash = Plugins::PluginComponents.load_hash()
+          @receivedPackets = load_metric(hash["network"], "receivedPackets")
+          @receivedPacketsError = load_metric(hash["network"], "receivedPacketsError")
+          @receivedPacketsDropped = load_metric(hash["network"], "receivedPacketsDropped")
+          @sendPackets = load_metric(hash["network"], "sendPackets")
+          @sendPacketsError = load_metric(hash["network"], "sendPacketsError")
+          @sendPacketsDropped = load_metric(hash["network"], "sendPacketsDropped")
+          @sigar = Plugins::PluginPreop.instance.sigar
+        end 
+      end
+  
+      def collect(agentInstanceId)
+        log.info("Calling Collect on SigarNetPacketInformation")      
+        iflist = @sigar.net_interface_list
+  
+        iflist.each do |ifname|
+          ifconfig = @sigar.net_interface_config(ifname)
+          if (!ifname.include?(":"))
+            ifstat = @sigar.net_interface_stat(ifname)
+            begin
+              report(receivedPackets, menu, ifstat.rx_packets, true, agentInstanceId, "receivedPacketsChart-" + ifname, 
+                "Received packets " + ifname,
+                "Total number of received packets" , "area", "Time", "Number of Packets", "c", "s")
+              report(receivedPacketsError, menu, ifstat.rx_errors, true, agentInstanceId, "receivedPacketsErrorChart-" + ifname, 
+                "Received packet errors " + ifname,
+                "Total number of received packet errors" , "area", "Time", "Number of Packets", "c", "s")
+              report(receivedPacketsDropped, menu, ifstat.rx_dropped, true, agentInstanceId, "receivedPacketsDroppedChart-" + ifname,
+                "Received packet dropped " + ifname,
+                "Total number of received packet dropped" , "area", "Time", "Number of Packets", "c", "s")
+                
+              report(sendPackets, menu, ifstat.tx_packets, true, agentInstanceId, "sendPacketsChart-" + ifname, 
+                "Send packets " + ifname,
+                "Total number of send packets" , "area", "Time", "Number of Packets", "c", "s")
+              report(sendPacketsError, menu, ifstat.tx_errors, true, agentInstanceId, "sendPacketsErrorChart-" + ifname, 
+                "Send packet errors " + ifname,
+                "Total number of send packet errors" , "area", "Time", "Number of Packets", "c", "s")
+              
+              report(sendPacketsDropped, menu, ifstat.tx_dropped, true, agentInstanceId, "sendPacketsDroppedChart-" + ifname, 
+                "Send packet dropped " + ifname,
+                "Total number of send packet dropped" , "area", "Time", "Number of Packets", "c", "s")
+            rescue Exception => e
+              puts e
+            end           
+          end
+        end
+      end
+  
+      
+    end
+  end
+end

data/lib/cloudscale/plugins/plugin.rb

@@ -0,0 +1,252 @@
+require "fsdb"
+require "yaml/store"
+require "logger"
+require "#{File.dirname(__FILE__)}/../monitor/agent/init_components"
+require "#{File.dirname(__FILE__)}/../monitor/agent/init_menus"
+require "#{File.dirname(__FILE__)}/../monitor/model/constants/agent_instance_store"
+require "#{File.dirname(__FILE__)}/../rest/rest_client"
+
+##
+#
+# @author Johannes Hiemer, cloudscale.
+#
+##
+module Cloudscale
+  module Plugins
+    class Plugin
+  
+      attr_accessor :log, :plugins, :statistics, :units, :chartSettings, :client, :local_chart_db, :value_store
+      @plugins = Set.new
+      
+      def self.log
+        @log = Logger.new(STDOUT)
+      end
+      
+      def initialize()
+        @local_chart_db = FSDB::Database.new("#{File.dirname(__FILE__)}/../store/charts")
+        @value_store = Hash.new
+        @client = RestClientWrapper.instance
+        @statistics = client.list("statistics")
+        @units = client.list("units")
+      end
+      
+      def self.reset(agentInstanceId)      
+        resetRestEndpoint('coredatas', agentInstanceId, 'Coredata')
+        resetRestEndpoint('tables', agentInstanceId, 'Table')
+        resetRestEndpoint('charts', agentInstanceId, 'Chart')
+        
+        Monitor::InitComponents.remove_components()
+        Monitor::InitMenus.remove_menus()
+        
+        remove_measurements(agentInstanceId)
+        remove_agenInstance(agentInstanceId)
+        
+                
+        local_chart_db = FSDB::Database.new("#{File.dirname(__FILE__)}/../store/charts/")
+        local_chart_db.fetch.each do | element |
+          local_chart_db.delete(element)
+        end
+      end
+      
+      def self.remove(agentInstanceId)
+        reset(agentInstanceId)
+        
+        resetRestEndpoint('components', agentInstanceId, 'Components')
+        resetRestEndpoint('agentInstanceMenus', agentInstanceId, 'Agent Insntance Menus')
+        
+        Constants::AgentInstance.remove()
+      end
+      
+      def self.resetRestEndpoint(entityRel, agentInstanceId, entityName)
+        client = RestClientWrapper.instance
+        
+        entities = client.searchAny(entityRel,"findByAgentInstanceId",
+                        { :agentInstanceId => agentInstanceId })
+  
+        puts "Found #{entityName} elements for Agent (#{agentInstanceId}): #{entities["content"].length.to_s}"
+        puts "    Starting to delete all elements..."      
+        
+        entities["content"].each do  | entity |
+          client.delete(entityRel, RestClientWrapper.load_entity_id(entity))
+        end
+        
+        puts "    Deleting all elements was successful \n\n"
+      end
+      
+      def self.remove_agenInstance(agentInstanceId)
+        puts "Removing AgentInstance completely:\n\n"
+              
+        client = RestClientWrapper.instance
+        agent_instance = client.searchOne('agentInstances',
+                'findByAgentInstanceId', { :id => agentInstanceId })
+  
+        client.delete('agentInstances', RestClientWrapper.load_entity_id(agent_instance))
+          
+        puts "    Deleting AgentInstance was successful \n\n"
+      end
+      
+      def self.remove_measurements(agentInstanceId)
+        puts "Removing Measurement elements for Agent (#{agentInstanceId}):\n\n"
+        puts "    Starting to delete all elements..."      
+              
+        client = RestClientWrapper.instance
+        client.customDelete("measurements", "removeByAgentInstanceId", { :agentInstanceId => agentInstanceId })
+          
+        puts "    Deleting all elements was successful \n\n"
+      end
+      
+      def self.plugins
+        @plugins 
+      end
+      
+      def self.inherited(klass)
+        @plugins << klass
+        #log.debug("Initialize for " << klass.to_s)
+      end
+      
+      def load_metric(component, name)
+        component["metrics"].each { | metric |
+          if (metric["name"] == name)
+            return metric
+          end
+        }
+      end
+      
+      def load_menu(menu, id)
+        menu["subItems"].each { | item | 
+          if (item["menuId"] == id)
+            return item
+          end
+        }
+      end
+      
+      def load_unit_id(id)
+        unit = load_unit(@units["content"], id)
+        if (unit != nil)
+          unit["unitId"]
+        end
+      end
+      
+      def load_unit(units, id)
+        if (units.is_a?(Array))
+          units.each { | unit |
+            if (unit["unitId"] == id)
+              return unit
+            end
+          }
+        end
+        return nil
+      end
+      
+      def load_statistic_id(id)
+        statistic = load_statistic(@statistics["content"], id)
+        if (statistic != nil)
+          statistic["statisticId"]
+        end
+      end
+      
+      def load_statistic(statistics, id)
+        if (statistics.is_a?(Array))
+          statistics.each { | statistic |
+            if (statistic["statisticId"] == id)
+              return statistic
+            end
+          }
+        end
+        return nil
+      end
+      
+      def store(key, chart)
+        local_chart_db[key] = chart.to_h
+      end
+      
+      def load(key)
+        return local_chart_db[key]
+      end
+      
+      def persist_table(table, agentInstanceId, menu)
+         persist_entity(table, 'tables', agentInstanceId, menu["menuId"]) 
+      end
+      
+      def persist_coredata(coredata, agentInstanceId, menu)
+        persist_entity(coredata, 'coredatas', agentInstanceId, menu["menuId"])   
+      end
+      
+      def persist_measurement(measurement, entityId, agentInstanceId, metricId)
+        if (measurement != nil && agentInstanceId != nil && metricId != nil)
+          measurement.metricId = metricId
+          measurement.chartId = entityId
+          measurement.agentInstanceId = agentInstanceId
+          return client.post('measurements', measurement)
+        end
+      end
+      
+      def persist_chart(chart, agentInstanceId, menu)
+        hash = load(chart.name)
+        if (hash == nil)
+          persist_entity(chart, 'charts', agentInstanceId, menu["menuId"])
+          store(chart.name, chart)
+          return chart.to_h
+        else
+          return hash
+        end      
+      end
+      
+      def persist_entity(entity, entityRel, agentInstanceId, menuId)
+        timestamp = Time.now.to_i * 1000
+        
+        if (entity != nil && agentInstanceId != nil && menuId != nil)
+          entity.menuId = menuId
+          entity.agentInstanceId = agentInstanceId
+          
+          entity_exists = client.searchOne(entityRel,"findByAgentInstanceIdAndMenuIdAndEntityId",
+            { :agentInstanceId => agentInstanceId, :menuId => menuId, :entityId => entity.entityId  })
+          
+          if (entity_exists)
+            return client.put(entityRel, RestClientWrapper.load_entity_id(entity_exists), entity)
+          else
+            return client.post(entityRel, entity)
+          end
+        else
+          raise "plugin.persist_entity(): Insufficient number of paramters, or parameter values"
+        end      
+      end
+      
+      def report(metric, menu, value, useStore, agentInstanceId, chartName, chartLabel, chartDescription, chartType, chartxaxis, chartyaxis, unit, divisor)
+        if (value_store.key?(chartName) && useStore)
+          result = (value - value_store[chartName]) / 30
+        elsif (!value_store.key?(chartName) && useStore)
+          result = nil
+        else
+          result = value
+        end
+        
+        if (result != nil)
+          timestamp = Time.now.to_i * 1000
+          statistic = load_statistic_id("avg");
+          if (unit != nil)
+            unit = load_unit_id(unit);
+          end
+    
+          if (divisor != nil)
+            divisor = load_unit_id(divisor);
+          end
+    
+          values = [Monitor::Value.new(statistic, unit, divisor, result)]
+          measurement = Monitor::Measurement.new(timestamp, values.to_dh)
+    
+          chart = Monitor::Chart.new(chartName, chartLabel, chartDescription, chartType, chartxaxis, chartyaxis)
+          chartHash = persist_chart(chart, agentInstanceId, menu)
+          if (chartHash != nil)
+            persist_measurement(measurement, chartHash["entityId"], agentInstanceId,  metric["metricId"])
+          end
+        end
+  
+        if (useStore)
+          value_store[chartName] = value
+        end
+      end
+      
+    end
+  end
+end

data/lib/cloudscale/plugins/postgres/postgres_server_status.rb

@@ -0,0 +1,148 @@
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/chart"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/measurement"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/value"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/statistic"
+require "#{File.dirname(__FILE__)}/../../monitor/model/measurement/unit"
+#require "#{File.dirname(__FILE__)}/../preops/plugin_postgres_preop"
+require "#{File.dirname(__FILE__)}/../plugin"
+
+##
+#
+# @author Johannes Hiemer, cloudscale.
+#
+##
+module Cloudscale
+  module Plugins
+    class PostgresServerStatus < Plugins::Plugin
+      attr_reader :menu_generalInformationm, :menu_numBackends, :menu_transactionInformation, :menu_bulkOps, :menu_rowStats,
+        :numBackends, :xactCommit, :xactRollback, :xactTotal, :blksRead, :blksHit,
+        :generalInformation, :rowsSelectIdx, :rowsSelectScan, :rowInserts, :rowUpdated, :rowDeleted, :rowTotal
+      def log
+        @log = Logger.new(STDOUT)
+      end
+  
+      def is_enabled
+        false
+      end
+  
+      def initialize
+        super
+        if is_enabled
+          menus = Plugins::PluginMenus.load_hash()
+          @menu_generalInformation = load_menu(menus["pgDatabaseStats"], "generalInformation")
+          @menu_numBackends = load_menu(menus["pgDatabaseStats"], "numBackends")
+          @menu_transactionInformation = load_menu(menus["pgDatabaseStats"], "transactionInformation")
+          @menu_bulkOps = load_menu(menus["pgDatabaseStats"], "bulkOps")
+          @menu_rowStats = load_menu(menus["pgTableStats"], "rowStats")
+                    
+          @connection = Plugins::PostgresDbWrapper.instance.connection
+          components = Plugins::PluginComponents.load_hash()
+          @numBackends = load_metric(components["pgDatabaseStats"], "numBackends")        
+          @xactCommit = load_metric(components["pgDatabaseStats"], "xactCommit")
+          @xactRollback = load_metric(components["pgDatabaseStats"], "xactRollback")
+          @xactTotal = load_metric(components["pgDatabaseStats"], "xactTotal")
+          @blksRead = load_metric(components["pgDatabaseStats"], "blksRead")
+          @blksHit = load_metric(components["pgDatabaseStats"], "blksHit")
+            
+          @generalInformation = load_metric(components["pgDatabaseStats"], "generalInformation")
+            
+          @rowsSelectIdx = load_metric(components["pgTableStats"], "rowsSelectIdx")
+          @rowsSelectScan = load_metric(components["pgTableStats"], "rowsSelectScan")
+          @rowInserts = load_metric(components["pgTableStats"], "rowInserts")
+          @rowUpdated = load_metric(components["pgTableStats"], "rowUpdated")
+          @rowDeleted = load_metric(components["rowDeleted"], "rowDeleted")
+          @rowTotal = load_metric(components["rowTotal"], "rowTotal")
+            
+        end
+      end
+  
+      def collect(agentInstanceId)
+        log.info("Calling Collect on MongoServerStatus")
+        report_general_information(agentInstanceId)
+        report_database_stats(agentInstanceId)
+        report_table_stats(agentInstanceId)
+      end
+  
+      def report_database_stats(agentInstanceId)
+        result = @connection.exec('SELECT sum(idx_tup_fetch) AS "rows_select_idx",
+                  sum(seq_tup_read) AS "rows_select_scan",
+                  sum(n_tup_ins) AS "rows_insert",
+                  sum(n_tup_upd) AS "rows_update",
+                  sum(n_tup_del) AS "rows_delete",
+                  (sum(idx_tup_fetch) + sum(seq_tup_read) + sum(n_tup_ins) + sum(n_tup_upd) + sum(n_tup_del)) AS "rows_total"
+                  FROM pg_stat_all_tables;')  
+        row = result[0]
+        
+        report(rowsSelectIdx, menu_rowStats, row['rows_select_idx'], false, agentInstanceId, "rowsSelectIdxChart",
+            "Select indexes", "Select indexes across all tables in the database", "area", "Time", "Count", "c",  nil) 
+        
+        report(rowsSelectScan, menu_rowStats, row['rows_select_scan'], false, agentInstanceId, "rowsSelectScanChart",
+            "Row select scan", "Row select scans across all tables in the database", "area", "Time", "Count", "c",  nil)
+        
+        report(rowInserts, menu_rowStats, row['rows_insert'], false, agentInstanceId, "rowInsertsChart",
+            "Row inserts", "Row inserts across all tables in the database", "area", "Time", "Count", "c",  nil)
+            
+        report(rowUpdated, menu_rowStats, row['rows_update'], false, agentInstanceId, "rowUpdatedChart",
+            "Row updates", "Row updates across all tables in the database", "area", "Time", "Count", "c",  nil)
+            
+        report(rowDeleted, menu_rowStats, row['rows_delete'], false, agentInstanceId, "rowDeletedChart",
+            "Row deletes", "Row deletes across all tables in the database", "area", "Time", "Count", "c",  nil)
+        
+        report(rowTotal, menu_rowStats, row['rows_total'], false, agentInstanceId, "rowTotalChart",
+            "Rows affected", "Rows affected across all tables in the database", "area", "Time", "Count", "c",  nil)
+      end
+  
+      def report_table_stats(agentInstanceId)
+        result = @connection.exec('SELECT sum(numbackends) AS "numbackends",
+                    sum(xact_commit) AS "xact_commit",
+                    sum(xact_rollback) AS "xact_rollback",
+                    sum(xact_commit+xact_rollback) AS "xact_total",
+                    sum(blks_read) AS "blks_read",
+                    sum(blks_hit) AS "blks_hit"
+             FROM pg_stat_database;')
+        row = result[0]
+        
+        report(xactCommit, menu_transactionInformation, row['xact_commit'], false, agentInstanceId, "xactCommitChart",
+            "Transaction commits", "Transaction commits across all tables in the database", "area", "Time", "Count", "c",  nil)
+        
+        report(xactRollback, menu_transactionInformation, row['xact_rollback'], false, agentInstanceId, "xactRollbackChart",
+            "Transaction rollback", "Transaction rollback across all tables in the database", "area", "Time", "Count", "c",  nil)
+            
+        report(xactTotal, menu_transactionInformation, row['xact_total'], false, agentInstanceId, "xactTotalChart",
+            "Transactions total", "Transactions total across all tables in the database", "area", "Time", "Count", "c",  nil)
+            
+        report(blksRead, menu_bulkOps, row['blks_read'], false, agentInstanceId, "blksReadChart",
+            "Bulk reads", "Bulk reads across all tables in the database", "area", "Time", "Count", "c",  nil)
+        
+        report(blksHit, menu_bulkOps, row['blks_hit'], false, agentInstanceId, "blksHitChart",
+            "Bulk hits", "Bulk hits across all tables in the database", "area", "Time", "Count", "c",  nil)
+      end
+  
+      def report_general_information(agentInstanceId)
+        result = @connection.exec('SELECT datid,
+                 datname, pid, usesysid, usename, 
+                 client_addr, client_hostname, 
+                 client_port, backend_start, state, waiting
+                 FROM pg_stat_activity;')
+        row = result[0]
+        
+        body = Array.new
+        body.push(["DatId", row['datid']])
+        body.push(["Datname", row['datname']])
+        body.push(["Pid", row['pid']])
+        body.push(["Use Sysid", row['usesysid']])
+        body.push(["usename", row['usename']])
+        body.push(["client_addr", row['client_addr']])
+        body.push(["client_hostname", row['client_hostname']])
+        body.push(["client_port", row['client_port']])
+        body.push(["backend_start", row['backend_start']])
+        body.push(["state", row['state']])
+        body.push(["waiting", row['waiting']])
+        mongo_general_inf_table = Monitor::Table.new("generalInformationPostgres", "Postgres General Information", nil, body, "compact")
+        mongo_general_inf_table.caption = "General Information regarding your Postgres Instance"
+  
+        persist_table(mongo_general_inf_table, agentInstanceId, menu_generalInformation)
+      end
+    end
+  end
+end