cloudscale 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a07957c059e3e1c75fefb1a79e9f56e2e49f47e2
-  data.tar.gz: 7dc1299feeebd1098edd361f92caf9dfc3057af2
+  metadata.gz: 36ae38eb4e8a1200b5e879ca565843d78846eb32
+  data.tar.gz: 4912656914e1c5d406c2b84bec72b22cb71d99fc
 SHA512:
-  metadata.gz: f653403bd281f41c3eb9aed46b40c37b732999f968719fad63c690546885e946f3d3efc5dca20bdf16da41e24afe41f34b4883c6983517cdb73aa74934f2e7a6
-  data.tar.gz: 497ca67326712445f8a7bb87bd79519971c12d68dce3db3580b17b18faf41ec3b221fa7862ed7e1f576232c90fe20ba544fc6b392342432f61d81ec965361ed9
+  metadata.gz: f2702905e295c443f6216f31832d629d21753d2d842bb92169163c5089864faf4fc6c66f66a275def7347ced503cae2edec22a64ae550f88d29988849bbd5775
+  data.tar.gz: 2bb12b0c1586ff22105bcffbfda0f8ef5002a41d6e7e6a5d860778b81a936e19bec780db64e46c7864835d9b3e269b2411e975ae3296f81b65dd9b3e368ab52a

data/lib/cloudscale.rb

@@ -3,11 +3,16 @@ require "singleton"
 require "yaml"
 require "json"
 require "#{File.dirname(__FILE__)}/cloudscale/version"
-require "#{File.dirname(__FILE__)}/cloudscale/monitor/agent/init"
 require "#{File.dirname(__FILE__)}/cloudscale/schedule"
-require "#{File.dirname(__FILE__)}/cloudscale/monitor/agent/preops/general_preop"
-require "#{File.dirname(__FILE__)}/cloudscale/plugins/plugin"
+require "#{File.dirname(__FILE__)}/cloudscale/monitor/agent/agent"
 require "#{File.dirname(__FILE__)}/cloudscale/monitor/model/constants/agent_store"
+require "#{File.dirname(__FILE__)}/cloudscale/monitor/preops/general_preop"
+
+require "#{File.dirname(__FILE__)}/cloudscale/monitor/plugin/settings_db"
+require "#{File.dirname(__FILE__)}/cloudscale/plugins/preops/plugin_preop"
+Dir["#{File.dirname(__FILE__)}/cloudscale/plugins/*/preops/*.rb"].each { | f | require f }
+
+require "#{File.dirname(__FILE__)}/cloudscale/plugins/plugin"
 Dir["#{File.dirname(__FILE__)}/cloudscale/plugins/*/*.rb"].each { | f | require f }
 
 ##
@@ -23,7 +28,7 @@ module Cloudscale
 
     scheduler = nil
 
-    Monitor::GeneralPreop.instance.init
+    preopDb = Preops::PluginSettingsDb.instance
 
     parser = OptionParser.new do | parser |
 
@@ -47,35 +52,44 @@ module Cloudscale
       parser.separator "\n"
       parser.separator "Monitoring specific Settings [Mandatory]"
 
+      preopDb.options.merge!(Monitor::GeneralPreop.instance.options)
       Monitor::GeneralPreop.instance.options.each do | key, value |
         if (value[:required])
           mandatoryGeneral.push(key)
         end
           parser.on(value[:argument][1,2] + " n", value[:argument] + "=n", value[:description]) do | value |
-            Monitor::GeneralPreop.instance.set_option_value(key, value)
+            preopDb.set_option_value(key, value)
         end
       end
 
       parser.separator "\n"
       parser.separator "Plugin specific Settings [Mandatory]"
 
-      Plugins::PluginPreop.instance.options.each do | key, value |
-        if (value[:required])
-          mandatory.push(key)
-        end
-        parser.on(value[:argument][1,2] + " n", value[:argument] + "=n", value[:description]) do | value |
-          Plugins::PluginPreop.instance.set_option_value(key, value)
+      Preops::PluginPreop.pluginPreops.each do | pluginPreop_class |
+        pluginPreop = pluginPreop_class.instance
+
+        preopDb.options.merge!(pluginPreop.options)
+        if (pluginPreop.is_enabled)
+          pluginPreop.options.each do | key, value |
+            if (value[:required])
+              mandatory.push(key)
+            end
+            parser.on(value[:argument][1,2] + " n", value[:argument] + "=n", value[:description]) do | value |
+              preopDb.set_option_value(key, value)
+            end
+          end
         end
+
       end
     end
 
     begin
       parser.parse!
       missingGeneral = mandatoryGeneral.select { | param |
-        Monitor::GeneralPreop.instance.get_option_value(param).nil?
+        preopDb.get_option_value(param).nil?
       }
       missing = mandatory.select{ | param |
-        Plugins::PluginPreop.instance.get_option_value(param).nil?
+        preopDb.get_option_value(param).nil?
       }
       if (!missing.empty? || !missingGeneral.empty?)
         puts "\n"
@@ -86,18 +100,18 @@ module Cloudscale
         puts parser
         exit
       else
-        Plugins::PluginPreop.instance.save_options
-        Monitor::GeneralPreop.instance.save_options
+        preopDb.save_options
+        preopDb.save_options
       end
     rescue OptionParser::InvalidOption, OptionParser::MissingArgument
       puts $!.to_s
       puts parser
       exit
     end
-    
+
     begin
-        host = GeneralPreop.instance.get_option_value("host")
-        port = GeneralPreop.instance.get_option_value("port")
+        host = preopDb.get_option_value("host")
+        port = preopDb.get_option_value("port")
 
         config = Constants::Agent.load()
         if (port != nil) then config["port"] = port end
@@ -105,15 +119,13 @@ module Cloudscale
         Constants::Agent.save(config)
     end
 
-    agent_instance = Monitor::Init.new(GeneralPreop.instance.get_option_value("token"))
+    agent_instance = Monitor::Agent.new(preopDb.get_option_value("token"))
     agent_instance_id = registry = Monitor::Registry.instance.agent_instance_id
 
     case ARGV[0]
       when "start"
         plugins = Set.new
 
-        Plugins::PluginPreop.instance.register
-
         Plugins::Plugin.plugins.each do | plugin_class |
           plugin = plugin_class.new
           plugins << plugin
@@ -124,12 +136,12 @@ module Cloudscale
         puts "call stop on options #{options.inspect}"
       when "reset"
         puts "Starting cleanup process"
-        Plugins::Plugin.reset(agent_instance_id)
-        Plugins::PluginPreop.instance.clear_options
+        Monitor::Agent.reset(agent_instance_id)
+        preopDb.clear_options
       when "delete"
         puts "Starting removal process"
-        Plugins::Plugin.remove(agent_instance_id)
-        Plugins::PluginPreop.instance.clear_options
+        Monitor::Agent.remove(agent_instance_id)
+        preopDb.clear_options
       else
         puts parser
     end

data/lib/cloudscale/monitor/agent/{init.rb → agent.rb}

@@ -13,7 +13,7 @@ require "#{File.dirname(__FILE__)}/../../rest/rest_client"
 ##
 module Cloudscale
   module Monitor
-    class Init
+    class Agent
       attr_accessor :log, :server, :client
 
       def initialize(token)
@@ -58,6 +58,47 @@ module Cloudscale
         end
       end
 
+      def self.reset(agentInstanceId)
+        resetRestEndpoint('coredatas', agentInstanceId, 'Coredata')
+        resetRestEndpoint('tables', agentInstanceId, 'Table')
+        resetRestEndpoint('charts', agentInstanceId, 'Chart')
+
+        remove_agenInstance(agentInstanceId)
+      end
+
+      def self.remove(agentInstanceId)
+        reset(agentInstanceId)
+
+        Constants::AgentInstance.remove()
+      end
+
+      def self.resetRestEndpoint(entityRel, agentInstanceId, entityName)
+        rest_client = RestClientWrapper.instance
+        entities = rest_client.searchAny(entityRel,"findByAgentInstanceId",
+                        { :agentInstanceId => agentInstanceId })
+
+        puts "Found #{entityName} elements for Agent (#{agentInstanceId}): #{entities["content"].length.to_s}"
+        puts "    Starting to delete all elements..."
+
+        entities["content"].each do  | entity |
+          rest_client.delete(entityRel, RestClientWrapper.load_entity_id(entity))
+        end
+
+        puts "    Deleting all elements was successful \n\n"
+      end
+
+      def self.remove_agenInstance(agentInstanceId)
+        rest_client = RestClientWrapper.instance
+        puts "Removing AgentInstance completely:\n"
+
+        agent_instance = rest_client.searchOne('agentInstances',
+                'findByAgentInstanceId', { :id => agentInstanceId })
+
+        rest_client.delete('agentInstances', RestClientWrapper.load_entity_id(agent_instance))
+
+        puts "    Deleting AgentInstance was successful \n\n"
+      end
+
     end
   end
 end

data/lib/cloudscale/monitor/plugin/settings_db.rb

@@ -0,0 +1,60 @@
+require "fsdb"
+require "#{File.dirname(__FILE__)}/../../registry"
+
+##
+#
+# @author Johannes Hiemer.
+#
+##
+module Cloudscale
+  module Preops
+    class PluginSettingsDb
+      include Singleton
+      attr_accessor :log, :options
+
+      def initialize
+        @log = Logger.new(STDOUT)
+        @options = Hash.new
+        @plugin_settings_db = FSDB::Database.new("#{File.dirname(__FILE__)}/../../../store/plugin/")
+        init_options()
+      end
+
+      def set_option_value(key, val)
+        @options[key.to_sym][:value] = val
+      end
+
+      def get_option(key)
+        @options[key.to_sym]
+      end
+
+      def get_option_value(key)
+        if (@options[key.to_sym] != nil)
+          @options[key.to_sym][:value] #|| @plugin_settings_db[key.to_s]
+        else
+          nil
+        end
+      end
+
+      def init_options
+        @plugin_settings_db.fetch.each do | key |
+          if (@options[key.to_sym])
+            @options[key.to_sym][:value] = @plugin_settings_db[key]
+          end
+        end
+      end
+
+      def save_options
+        @options.each do | key, value |
+          @plugin_settings_db[key.to_s] = value[:value].to_s
+        end
+      end
+
+      def clear_options
+        @plugin_settings_db.fetch.each do | element |
+          @plugin_settings_db.delete(element)
+        end
+      end
+
+    end
+  end
+end

data/lib/cloudscale/monitor/preops/preop.rb

@@ -0,0 +1,62 @@
+require "fsdb"
+require "#{File.dirname(__FILE__)}/../../registry"
+
+##
+#
+# @author Johannes Hiemer.
+#
+##
+module Cloudscale
+  module Preops
+    class Preop
+      attr_accessor :log, :registry
+
+      def init
+        @log = Logger.new(STDOUT)
+        @registry = Monitor::Registry.instance      
+      end
+
+      def init_charts(path)
+        log.info("Starting monitoring chart synchronization...")
+
+        file = File.read(path);
+        plugin = YAML.load(JSON.parse(file).to_yaml)
+        plugin.each do | chart, value |
+          chart["agentInstanceId"] = agentInstanceId
+
+          chart["queries"].each { | query |
+            query["command"].sub! '[[agentInstanceId]]', agentInstanceId
+          }
+
+          test_chart = @client.searchOne("charts", "findByAgentInstanceIdAndChartId", { :agentInstanceId => agentInstanceId, :chartId => chart["chartId"]})
+          if (test_chart == nil)
+            client.post("charts", chart)
+          else
+            client.patch("charts", RestClientWrapper.load_entity_id(test_chart), chart)
+          end
+        end
+
+        Monitor::InitCharts.new(@registry.agent_instance_id, path)
+
+        log.info("Successfully synchronized charts...")
+      end
+
+      def init_menus(path)
+       log.info("Starting monitoring menu synchronization...")
+
+       file = File.read(path);
+       plugin = YAML.load(JSON.parse(file).to_yaml)
+       menuItem["agentInstanceId"] = agentInstanceId
+       test_menu = @client.searchOne("menus", "findByAgentInstanceIdAndMenuId", { :agentInstanceId => agentInstanceId, :menuId => menuItem["menuId"]})
+       if (test_menu == nil)
+         client.post("menus", menuItem)
+       else
+         client.patch("menus", RestClientWrapper.load_entity_id(test_menu), menuItem)
+       end
+
+       log.info("Successfully synchronized menus...")
+     end
+
+    end
+  end
+end

data/lib/cloudscale/plugins/mongo/mongo_db_status.rb

@@ -1,4 +1,4 @@
-#require "#{File.dirname(__FILE__)}/../preops/plugin_mongodb_preop"
+require "#{File.dirname(__FILE__)}/preops/mongodb_preop"
 require "#{File.dirname(__FILE__)}/../plugin"
 
 ##
@@ -9,23 +9,22 @@ require "#{File.dirname(__FILE__)}/../plugin"
 module Cloudscale
   module Plugins
     class MongoDbStatus < Plugins::Plugin
-      attr_reader
-
-      def log
-        @log = Logger.new(STDOUT)
-      end
+      attr_reader :db
 
       def is_enabled
-        false
+        Preops::MongodbPreop.instance.is_enabled
       end
 
       def initialize
         super
         if is_enabled
+          @db = Preops::MongodbPreop.instance.db
         end
       end
 
       def collect(agentInstanceId)
+        registry = Monitor::Registry.instance
+        metrics = Metrics::Agent.new
         log.info("Calling Collect on MongoDbStatus")
         stats = @db.stats
 
@@ -33,23 +32,29 @@ module Cloudscale
           { :value => stats['objects'] }
         end
 
-        report(indexMetric, menu, stats['indexes'], false, agentInstanceId, "dbIndexChart",
-               "Indexes in database", "Total number of indexes in the database" , "area", "Time", "Count", "c", nil)
+        registry.metrics["mongo.status.indexes"] = metrics.gauge :status_indexes do
+          { :value => stats['indexes'] }
+        end
 
-        report(dataSizeMetric, menu, stats['dataSize'], false, agentInstanceId, "dbDataSizeChart",
-               "Data size of database", "Total data size of the database" , "area", "Time", "MB", "MB", nil)
+        registry.metrics["mongo.status.indexes"] = metrics.gauge :status_indexes do
+          { :value => stats['indexes'] }
+        end
 
-        report(indexSizeMetric, menu, stats['indexSize'], false, agentInstanceId, "dbStorageSizeChart",
-               "Storage size of database", "Total storage size of the database", "area", "Time", "MB", "MB", nil)
+        registry.metrics["mongo.status.dataSize"] = metrics.gauge :status_dataSize do
+          { :value => stats['dataSize'] }
+        end
 
-        report(storageSizeMetric, menu, stats['storageSize'], false, agentInstanceId, "dbIndexSizeChart",
-              "Index size of database", "Total index size of the database" , "area", "Time", "MB", "MB", nil)
+        registry.metrics["mongo.status.storageSize"] = metrics.gauge :status_storageSize do
+          { :value => stats['storageSize'] }
+        end
 
-        report(collectionsMetric, menu, stats['collections'], false, agentInstanceId, "dbCollectionsChart",
-             "Collections in database", "Total number of collections in the database" , "area", "Time", "Count", "c", nil)
+        registry.metrics["mongo.status.collections"] = metrics.gauge :status_collections do
+          { :value => stats['collections'] }
+        end
 
-        report(avgObjSizeMetric, menu, stats['avgObjSize'], false, agentInstanceId, "dbAvgObjSizeChart",
-             "Average object size in database", "The average object size across all collections in the database" , "area", "Time", "MB", "MB", nil)
+        registry.metrics["mongo.status.avgObjSize"] = metrics.gauge :status_avgObjSize do
+          { :value => stats['avgObjSize'] }
+        end
       end
 
     end

data/lib/cloudscale/plugins/mongo/mongo_replica_status.rb

@@ -1,4 +1,4 @@
-#require "#{File.dirname(__FILE__)}/../preops/plugin_mongodb_preop"
+require "#{File.dirname(__FILE__)}/preops/mongodb_preop"
 require "#{File.dirname(__FILE__)}/../plugin"
 
 ##
@@ -9,95 +9,68 @@ require "#{File.dirname(__FILE__)}/../plugin"
 module Cloudscale
   module Plugins
     class MongoReplicaStatus < Plugins::Plugin
-      
-      attr_reader :menu_replicaClusterStatus, :db
-      def log
-        @log = Logger.new(STDOUT)
-      end
-      
+      attr_reader :db
+
       def is_enabled
-        false
+        Preops::MongodbPreop.instance.is_enabled
       end
-      
+
       def initialize
         super
         if is_enabled
-          menus = Plugins::PluginMenus.load_hash()
-          @menu_replicaClusterStatus = load_menu(menus["mongodb-server-stats"], "replicaClusterStatus")
-                   
-          @db = Plugins::PluginPreop.instance.db
-          components = Plugins::PluginComponents.load_hash()
-          @metric = load_metric(components["mongodb-replica-stats"], "replicaClusterStatus")  
+          @db = Preops::MongodbPreop.instance.db
         end
       end
-      
-      def collectHourly(agentInstanceId)
+
+      def collectHourly()
+        registry = Monitor::Registry.instance
+        metrics = Metrics::Agent.new
         log.info("Calling Collect on MongoReplicaStatus")
         replset_status = db.command({'replSetGetStatus' => 1}, :check_response => false)
-        
+
         unless replset_status['ok'] == 1
           return "Node isn't a member of a Replica Set","Unable to fetch Replica Set status information."
         end
-        
-        body = Array.new        
-        body.push(["Replica Set", replset_status['set']])
-        body.push(["Replica Set State", get_replica_state(replset_status['myState'])])      
-           
-        primary = replset_status['members'].detect { | member | member['state'] == 1 }
-        if primary
-          member = replset_status['members'].detect do | member |
-            member['self']
+
+        replset_status['members'].detect do | member |
+          registry.metrics["mongo.replicastate.member." + name + ".errmsg"] = metrics.gauge :errmsg do
+            { :value => member["errmsg"] }
           end
-    
-          if member
-            body.push(["Optime Difference for member " + member["name"], current_member['optimeDate'] - primary['optimeDate']])
+
+          registry.metrics["mongo.replicastate.member." + name + ".health"] = metrics.gauge :health do
+            { :value => member["health"] }
           end
-        end
-        
-        primary_server_table = Monitor::Table.new("replicaClusterStatus", "Replica Status Information", nil, body, "compact")
-        primary_server_table.caption = "Replica Status Information for a MongoDB Cluster"
-        persist_table(primary_server_table, agentInstanceId, metric["metricId"])
-        
-        head = ["Name", "Error Message (most recent)", "Health", "State", "Uptime", "Optime Date", "Last Heartbeat", "Last Hearbeat Received",
-          "Last Heartbeat Message", "Ping Ms"]
-        memberBody = Array.new
-        replset_status['members'].detect do | member | 
-          memberBody.push(member["name"], member["errmsg"], member["health"], member["stateStr"], member["uptime"], 
-            member["optimeDate"], member["lastHeartbeat"], member["lastHeartbeatRecv"], member["lastHeartbeatMessage"],
-            member["pingMs"])
-        end
-        member_server_table = Monitor::Table.new("replicaClusterStatus", "Replica Status Information", nil, memberBody, "compact")
-        member_server_table.caption = "Replica Status Information for a MongoDB Cluster"
-        persist_table(member_server_table, agentInstanceId, menu_replicaClusterStatus)
-        
-      end
-      
-      def get_replica_state(state)
-        member_state = case state
-          when 0
-            'Starting Up'
-          when 1
-            'Primary'
-          when 2
-            'Secondary'
-          when 3
-            'Recovering'
-          when 4
-            'Fatal'
-          when 5
-            'Starting up (forking threads)'
-          when 6
-            'Unknown'
-          when 7
-            'Arbiter'
-          when 8
-            'Down'
-          when 9
-            'Rollback'
+
+          registry.metrics["mongo.replicastate.member." + name + ".stateStr"] = metrics.gauge :stateStr do
+            { :value => member["stateStr"] }
+          end
+
+          registry.metrics["mongo.replicastate.member." + name + ".uptime"] = metrics.gauge :uptime do
+            { :value => member["uptime"] }
           end
-        return member_state
+
+          registry.metrics["mongo.replicastate.member." + name + ".optimeDate"] = metrics.gauge :optimeDate do
+            { :value => member["optimeDate"] }
+          end
+
+          registry.metrics["mongo.replicastate.member." + name + ".lastHeartbeat"] = metrics.gauge :lastHeartbeat do
+            { :value => member["lastHeartbeat"] }
+          end
+
+          registry.metrics["mongo.replicastate.member." + name + ".lastHeartbeatRecv"] = metrics.gauge :lastHeartbeatRecv do
+            { :value => member["lastHeartbeatRecv"] }
+          end
+
+          registry.metrics["mongo.replicastate.member." + name + ".lastHeartbeatMessage"] = metrics.gauge :lastHeartbeatMessage do
+            { :value => member["lastHeartbeatMessage"] }
+          end
+
+          registry.metrics["mongo.replicastate.member." + name + ".pingMs"] = metrics.gauge :pingMs do
+            { :value => member["pingMs"] }
+          end
+        end
       end
-      
+
     end
   end
-end
+end