cloudscale 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36ae38eb4e8a1200b5e879ca565843d78846eb32
-  data.tar.gz: 4912656914e1c5d406c2b84bec72b22cb71d99fc
+  metadata.gz: 2ae417e9a0ad0393aa9d12c3ad5ebe2011b43469
+  data.tar.gz: ac2fc6528ec4b6de4182394d63dec48ed6dc982b
 SHA512:
-  metadata.gz: f2702905e295c443f6216f31832d629d21753d2d842bb92169163c5089864faf4fc6c66f66a275def7347ced503cae2edec22a64ae550f88d29988849bbd5775
-  data.tar.gz: 2bb12b0c1586ff22105bcffbfda0f8ef5002a41d6e7e6a5d860778b81a936e19bec780db64e46c7864835d9b3e269b2411e975ae3296f81b65dd9b3e368ab52a
+  metadata.gz: 02d8676dadadc22ade4d50f3386d397e9e186c2b5ea642a0d1fcdffcb8fe4ca0fe632a28ff5394bb3bb032f8971cb8cd8dcda9f7b12d12a5115b6c3d6a2c1ba8
+  data.tar.gz: 6a3310a96126b81bca26a7a31e54af3e44743a5056b34e77bebc1580d0c8f9fed55c900a5decb3def3e57a708786fb098ff4e55c036726aa9451124e297020bf

data/lib/cloudscale.rb

@@ -66,16 +66,15 @@ module Cloudscale
       parser.separator "Plugin specific Settings [Mandatory]"
 
       Preops::PluginPreop.pluginPreops.each do | pluginPreop_class |
-        pluginPreop = pluginPreop_class.instance
-
-        preopDb.options.merge!(pluginPreop.options)
-        if (pluginPreop.is_enabled)
-          pluginPreop.options.each do | key, value |
+        preopDb.options.merge!(pluginPreop_class.instance.options)
+        if (pluginPreop_class.instance.is_enabled)
+          pluginPreop_class.instance.options.each do | key, value |
             if (value[:required])
               mandatory.push(key)
             end
-            parser.on(value[:argument][1,2] + " n", value[:argument] + "=n", value[:description]) do | value |
-              preopDb.set_option_value(key, value)
+            parser.on(value[:argument][1,2] + " n", value[:argument] + "=n", value[:description]) do | val |
+              pluginPreop_class.instance.options[key.to_sym][:value] = val
+              preopDb.set_option_value(key, val)
             end
           end
         end
@@ -101,7 +100,6 @@ module Cloudscale
         exit
       else
         preopDb.save_options
-        preopDb.save_options
       end
     rescue OptionParser::InvalidOption, OptionParser::MissingArgument
       puts $!.to_s
@@ -126,6 +124,10 @@ module Cloudscale
       when "start"
         plugins = Set.new
 
+        Preops::PluginPreop.pluginPreops.each do | pluginPreop_class |
+          pluginPreop_class.instance.register
+        end
+
         Plugins::Plugin.plugins.each do | plugin_class |
           plugin = plugin_class.new
           plugins << plugin

data/lib/cloudscale/monitor/agent/agent.rb

@@ -14,30 +14,30 @@ require "#{File.dirname(__FILE__)}/../../rest/rest_client"
 module Cloudscale
   module Monitor
     class Agent
-      attr_accessor :log, :server, :client
+      attr_accessor :log, :registry, :server, :client
 
       def initialize(token)
         @log = Logger.new(STDOUT)
         @server = Socket.gethostname
-        @client = RestClientWrapper.instance
-        @registry = Monitor::Registry.instance
+        client = RestClientWrapper.instance
+        registry = Monitor::Registry.instance
 
         puts " "
         puts "Running preflight verification for cloudscale monitoring agent"
 
-        if (@registry.stored_agent == nil)
+        if (registry.stored_agent == nil)
           raise "You did not provide any valid Agent file under /store/agent/agent.store."
         end
 
-        agent = client.searchOne('agents', 'findByAgentId', { :agentId => @registry.stored_agent["agentId"] })
+        agent = client.searchOne('agents', 'findByAgentId', { :agentId => registry.stored_agent["agentId"] })
 
         if (agent == nil)
           raise "No agent registered yet. It is not allowed to connect this agent instance to cloudscale monitoring."
         end
 
-        if (@registry.stored_agent_instance != nil)
+        if (registry.stored_agent_instance != nil)
           agent_instance = client.searchOne('agentInstances',
-          'findByAgentInstanceId', { :id => @registry.stored_agent_instance["agentInstanceId"] })
+            'findByAgentInstanceId', { :id => registry.agent_instance_id })
         end
 
         if (agent_instance == nil)
@@ -45,7 +45,7 @@ module Cloudscale
 
           agent_instance_settings = Monitor::Settings.new(15, 0, 1500, 5)
           agent_instance = Monitor::AgentInstance.new(RestClientWrapper.load_entity_ref(agent)["href"],
-            @registry.agent_instance_id, token, server, agent_instance_settings, Time.now.to_i * 1000)
+            registry.agent_instance_id, token, server, agent_instance_settings, Time.now.to_i * 1000)
 
           client.post('agentInstances', agent_instance)
           Constants::AgentInstance.create(agent_instance.to_h)
@@ -62,6 +62,7 @@ module Cloudscale
         resetRestEndpoint('coredatas', agentInstanceId, 'Coredata')
         resetRestEndpoint('tables', agentInstanceId, 'Table')
         resetRestEndpoint('charts', agentInstanceId, 'Chart')
+        resetRestEndpoint('menus', agentInstanceId, 'Menu')
 
         remove_agenInstance(agentInstanceId)
       end

data/lib/cloudscale/monitor/preops/preop.rb

@@ -1,5 +1,6 @@
 require "fsdb"
 require "#{File.dirname(__FILE__)}/../../registry"
+require "#{File.dirname(__FILE__)}/../../rest/rest_client"
 
 ##
 #
@@ -13,45 +14,48 @@ module Cloudscale
 
       def init
         @log = Logger.new(STDOUT)
-        @registry = Monitor::Registry.instance      
+        @registry = Monitor::Registry.instance
+        @rest_client = RestClientWrapper.instance
       end
 
       def init_charts(path)
         log.info("Starting monitoring chart synchronization...")
 
-        file = File.read(path);
-        plugin = YAML.load(JSON.parse(file).to_yaml)
-        plugin.each do | chart, value |
+        agentInstanceId = @registry.agent_instance_id
+        
+        plugin = YAML.load(JSON.parse(File.read(path)).to_yaml)
+        plugin.each do | key, chart |
           chart["agentInstanceId"] = agentInstanceId
 
           chart["queries"].each { | query |
             query["command"].sub! '[[agentInstanceId]]', agentInstanceId
           }
 
-          test_chart = @client.searchOne("charts", "findByAgentInstanceIdAndChartId", { :agentInstanceId => agentInstanceId, :chartId => chart["chartId"]})
+          test_chart = @rest_client.searchOne("charts", "findByAgentInstanceIdAndChartId", { :agentInstanceId => agentInstanceId, :chartId => chart["chartId"]})
           if (test_chart == nil)
-            client.post("charts", chart)
+            @rest_client.post("charts", chart)
           else
-            client.patch("charts", RestClientWrapper.load_entity_id(test_chart), chart)
+            @rest_client.patch("charts", RestClientWrapper.load_entity_id(test_chart), chart)
           end
         end
 
-        Monitor::InitCharts.new(@registry.agent_instance_id, path)
-
         log.info("Successfully synchronized charts...")
       end
 
       def init_menus(path)
        log.info("Starting monitoring menu synchronization...")
 
-       file = File.read(path);
-       plugin = YAML.load(JSON.parse(file).to_yaml)
-       menuItem["agentInstanceId"] = agentInstanceId
-       test_menu = @client.searchOne("menus", "findByAgentInstanceIdAndMenuId", { :agentInstanceId => agentInstanceId, :menuId => menuItem["menuId"]})
-       if (test_menu == nil)
-         client.post("menus", menuItem)
-       else
-         client.patch("menus", RestClientWrapper.load_entity_id(test_menu), menuItem)
+       agentInstanceId = @registry.agent_instance_id
+
+       plugin = YAML.load(JSON.parse(File.read(path)).to_yaml)
+       plugin.each do | key, menuItem |
+         menuItem["agentInstanceId"] = agentInstanceId
+         test_menu = @rest_client.searchOne("menus", "findByAgentInstanceIdAndMenuId", { :agentInstanceId => agentInstanceId, :menuId => menuItem["menuId"]})
+         if (test_menu == nil)
+           @rest_client.post("menus", menuItem)
+         else
+           @rest_client.patch("menus", RestClientWrapper.load_entity_id(test_menu), menuItem)
+         end
        end
 
        log.info("Successfully synchronized menus...")

data/lib/cloudscale/monitor/reporter/influxdb_reporter.rb

@@ -13,7 +13,7 @@ module Cloudscale
     class InfluxDbReporter
       include Singleton
 
-      @@prefix = Monitor::Registry.instance.agent_instance_id + "."
+      attr_reader :prefix
 
       @@iDB = Constants::InfluxDb.load
 
@@ -85,7 +85,8 @@ module Cloudscale
         0.0
       ]]
 
-      def initialize()
+      def initialize
+        @prefix = Monitor::Registry.instance.agent_instance_id + "."
       end
 
       def report(metrics)
@@ -126,7 +127,7 @@ module Cloudscale
     		p[14] = timer.mean_rate
     		p[15] = timer.count
         raise "timer dataset not equals column length" unless p.length == @@COLUMNS_TIMER.length
-        appendSeries(@@prefix, name, ".timer", @@COLUMNS_TIMER, p)
+        appendSeries(@prefix, name, ".timer", @@COLUMNS_TIMER, p)
 	    end
 
       def reportHistogram(name, histogram, timestamp)
@@ -143,7 +144,7 @@ module Cloudscale
     		p[9] = histogram.quantiles(0.99)
     		p[11] = histogram.one_minute_rate
         raise "histogram dataset not equals column length" unless p.length == @@COLUMNS_HISTOGRAM.length
-        appendSeries(@@prefix, name, ".histogram", @@COLUMNS_HISTOGRAM, p)
+        appendSeries(@prefix, name, ".histogram", @@COLUMNS_HISTOGRAM, p)
 	    end
 
       def reportCounter(name, counter, timestamp)
@@ -151,15 +152,15 @@ module Cloudscale
         p[0] = timestamp
         p[1] = counter.to_i
         raise "counter dataset not equals column length" unless p.length == @@COLUMNS_COUNT.length
-        appendSeries(@@prefix, name, ".count", @@COLUMNS_COUNT, p)
+        appendSeries(@prefix, name, ".count", @@COLUMNS_COUNT, p)
       end
 
       def reportGauge(name, gauge, timestamp)
         p = @@pointsGauge[0]
         p[0] = timestamp
         p[1] = gauge.get[:value]
-        raise "gauge dataset not equals column length" unless p.length == @@COLUMNS_GAUGE.length
-        appendSeries(@@prefix, name, ".value", @@COLUMNS_GAUGE, p)
+        raise "gauge dataset not equals column length" unless p.length == @@COLUMNS_GAUGE.length        
+        appendSeries(@prefix, name, ".value", @@COLUMNS_GAUGE, p)
       end
 
       def reportMeter(name, meter, timestamp)
@@ -170,7 +171,7 @@ module Cloudscale
     		p[4] = meter.fifteen_minute_rate
     		p[5] = meter.mean_rate
         raise "meter dataset not equals column length" unless p.length == @@COLUMNS_METER.length
-        appendSeries(@@prefix, name, ".value", @@COLUMNS_METER, p)
+        appendSeries(@prefix, name, ".value", @@COLUMNS_METER, p)
       end
 
       def appendSeries(namePrefix, name, nameSuffix, columns, points)
@@ -179,7 +180,7 @@ module Cloudscale
           data[columns[index]] = points[index]
         }
         client = InfluxDB::Client.new @@iDB["database"], { :host => @@iDB["host"], :port => @@iDB["port"], :username => @@iDB["username"],
-          :password => @@iDB["password"] }        
+          :password => @@iDB["password"] }
         client.write_point(namePrefix + name + nameSuffix, data, false)
       end
 

data/lib/cloudscale/plugins/mongo/data/mongo_chart.json

@@ -0,0 +1,434 @@
+{
+	"mongo.status" : {
+		"chartId" : "mongo.status",
+		"options" : {
+			"title" : { "text" : "Collection Stats" },
+			"subtitle" : { "text" : "The Collection usage/performance details" },
+			"chart" : {
+				"type": "area"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Date"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "short"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Collection Usage",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" :[{
+			"command" : "select mean(value) from /^[[agentInstanceId]].mongo.status\\./i"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.status",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	},
+	"mongo.mem" : {
+		"chartId" : "mongo.mem",
+				"options" : {
+			"title" : { "text" : "Memory Usage" },
+			"subtitle" : { "text" : "The Memory usage/performance details" },
+			"chart" : {
+				"type": "area"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Date"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "short"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Memory Usage",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select mean(value) from /^[[agentInstanceId]].mongo.mem\\./i"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.server",
+		"order" : 0,
+		"width" : 6,
+		"height" : 0
+	},
+	"mongo.network" : {
+		"chartId" : "mongo.network",
+				"options" : {
+			"title" : { "text" : "Network Usage" },
+			"subtitle" : { "text" : "The Network usage/performance details" },
+			"chart" : {
+				"type": "area"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Date"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "short"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Network Usage",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select mean(value) from /^[[agentInstanceId]].mongo.network\\./i"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.server",
+		"order" : 0,
+		"width" : 6,
+		"height" : 0
+	},
+	"mongo.general" : {
+		"chartId" : "mongo.general",
+		"options" : {
+			"title" : { "text" : "Mongo general" },
+			"subtitle" : { "text" : "Mongo general information" },
+			"chart" : {
+				"type": "area"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Date"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "bytes"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Mongo general information",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select derivative(value) from /^[[agentInstanceId]].mongo.general.connections\\./i WHERE time > now() - 300s GROUP BY time(10s)"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.server",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	},
+	"mongo.opcounts" : {
+		"chartId" : "mongo.opcounts",
+		"options" : {
+			"title" : { "text" : "Mongo Operation counts" },
+			"subtitle" : { "text" : "The mongo operation counts" },
+			"chart" : {
+				"type": "area"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Date"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "bytes"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Mongo Operation Counts",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select derivative(value) from /^[[agentInstanceId]].mongo.opcounts\\./i WHERE time > now() - 300s GROUP BY time(10s)"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.server",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	},
+	"mongo.index.counter" : {
+		"chartId" : "mongo.index.counter",
+		"options" : {
+			"title" : { "text" : "Mongo Index Counter" },
+			"subtitle" : { "text" : "The index counter of the database" },
+			"chart" : {
+				"type": "area"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Date"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "bytes"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Mongo Index Counter",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select derivative(value) from /^[[agentInstanceId]].mongo.index.counters\\./i WHERE time > now() - 300s GROUP BY time(10s)"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.server",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	},
+	"mongo.lock" : {
+		"chartId" : "mongo.lock",
+		"options" : {
+			"title" : { "text" : "Mongo lock times" },
+			"subtitle" : { "text" : "Mongo lock times" },
+			"chart" : {
+				"type": "area"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Date"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "bytes"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Mongo lock times",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select derivative(value) from /^[[agentInstanceId]].mongo.global.lock\\./i WHERE time > now() - 300s GROUP BY time(10s)"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.server",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	},
+	"mongo.replicastate" : {
+		"chartId" : "mongo.replicastate",
+		"options" : {
+			"title" : { "text" : "Mongo Replica State" },
+			"subtitle" : { "text" : "The replica state of the MongoDB Cluster" },
+			"chart" : {
+				"type": "line"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Last 15 Minutes"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "bytes"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Replica state",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select value from /^[[agentInstanceId]].mongo.replicatestate.member\\./i where time > now() - 15m group by time(15m)"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.replica",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	},
+	"mongo.collection.count" : {
+		"chartId" : "mongo.collection.count",
+		"options" : {
+			"title" : { "text" : "Mongo Collection Count Stats" },
+			"subtitle" : { "text" : "The collection count stats of the MongoDB Cluster" },
+			"chart" : {
+				"type": "line"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Last 15 Minutes"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "bytes"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Collection stats",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select mean(value) from /^[[agentInstanceId]].mongo.coll.count\\./i where time > now() - 15m group by time(15m)"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.collection",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	},
+	"mongo.collection.index" : {
+		"chartId" : "mongo.collection.index",
+		"options" : {
+			"title" : { "text" : "Mongo Collection index Stats" },
+			"subtitle" : { "text" : "The collection index stats of the MongoDB Cluster" },
+			"chart" : {
+				"type": "line"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Last 15 Minutes"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "bytes"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Collection index stats",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select mean(value) from /^[[agentInstanceId]].mongo.coll.index\\./i where time > now() - 15m group by time(15m)"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.collection",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	},
+	"mongo.collection.size" : {
+		"chartId" : "mongo.collection.size",
+		"options" : {
+			"title" : { "text" : "Mongo Collection size Stats" },
+			"subtitle" : { "text" : "The collection size stats of the MongoDB Cluster" },
+			"chart" : {
+				"type": "line"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Last 15 Minutes"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "bytes"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Collection size stats",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select mean(value) from /^[[agentInstanceId]].mongo.coll.size\\./i where time > now() - 15m group by time(15m)"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.collection",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	},
+	"mongo.collection.item" : {
+		"chartId" : "mongo.collection.item",
+		"options" : {
+			"title" : { "text" : "Mongo Collection item Stats" },
+			"subtitle" : { "text" : "The collection item of the MongoDB Cluster" },
+			"chart" : {
+				"type": "line"
+			},
+			"xAxis" : {
+				"title" : {
+					"text" : "Last 15 Minutes"
+				},
+				"type" : "datetime"
+			},
+			"yAxis" : {
+				"title" : {
+					"text" : "bytes"
+				},
+				"type" : "linear"
+			},
+			"legend" : {
+				"title" : "Collection item stats",
+				"enabled" : true,
+				"align" : "center",
+				"verticalAlign" : "bottom"
+			}
+		},
+		"queries" : [{
+			"command" : "select mean(value) from /^[[agentInstanceId]].mongo.coll.item\\./i where time > now() - 15m group by time(15m)"
+		}],
+		"agentInstanceId" : null,
+		"menuId" : "mongo.collection",
+		"order" : 0,
+		"width" : 12,
+		"height" : 0
+	}
+}