hq-mongodb-check-collection-size 0.0.0

data/bin/hq-mongodb-check-collection-size

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require "hq/mongodb/check-collection-size/script"
+
+script = HQ::MongoDB::CheckCollectionSize::Script.new
+script.args = ARGV
+script.main
+exit script.status

data/features/runs.feature

@@ -0,0 +1,21 @@
+Feature: The script can be run
+
+  Scenario: Run with basic options
+
+    When I run the following:
+      """
+      hq-mongodb-check-collection-size
+        --total-warning 2g
+        --total-critical 10g
+        --unsharded-warning 500m
+        --unsharded-critical 1g
+        --efficiency-warning 0.8
+        --efficiency-critical 0.5
+        --efficiency-size 100m
+      """
+
+    Then the exit status should be 0
+    And the output should match:
+      """
+      MongoDB collection size OK: biggest is (.+)
+      """

data/features/support/steps.rb

@@ -0,0 +1,26 @@
+When /^I run the following:$/ do
+	|command_string|
+
+	command_string.gsub! "\n", " "
+
+	@output = `bundle exec #{command_string}`
+
+	@status = $?.exitstatus
+
+end
+
+Then /^the exit status should be (\d+)$/ do
+	|status_str|
+
+	@status.should == status_str.to_i
+
+end
+
+Then /^the output should match:$/ do
+	|output_re_str|
+
+	output_re = /^#{output_re_str}$/
+
+	@output.strip.should match output_re
+
+end

data/lib/hq/mongodb/check-collection-size/script.rb

@@ -0,0 +1,481 @@
+require "mongo"
+
+require "hq/tools/check-script"
+require "hq/tools/future"
+require "hq/tools/getopt"
+require "hq/tools/thread-pool"
+
+module HQ
+module MongoDB
+module CheckCollectionSize
+
+class Script < Tools::CheckScript
+
+	def initialize
+		super
+		@name = "MongoDB collection size"
+	end
+
+	def process_args
+
+		@opts, @args =
+			Tools::Getopt.process ARGV, [
+
+				{ :name => :timeout,
+					:default => 10,
+					:regex => /[0-9]+(\.[0-9]+)?/,
+					:convert => :to_f },
+
+				{ :name => :hostname,
+					:default => "localhost" },
+
+				{ :name => :port,
+					:default => 27017,
+					:convert => :to_i },
+
+				{ :name => :total_warning,
+					:convert => method(:decode_bytes),
+					:required => true },
+
+				{ :name => :total_critical,
+					:convert => method(:decode_bytes),
+					:required => true },
+
+				{ :name => :unsharded_warning,
+					:convert => method(:decode_bytes),
+					:required => true },
+
+				{ :name => :unsharded_critical,
+					:convert => method(:decode_bytes),
+					:required => true },
+
+				{ :name => :efficiency_warning,
+					:convert => :to_i,
+					:required => true },
+
+				{ :name => :efficiency_critical,
+					:convert => :to_i,
+					:required => true },
+
+				{ :name => :efficiency_size,
+					:convert => method(:decode_bytes),
+					:required => true },
+
+				{ :name => :verbose,
+					:boolean => true },
+
+				{ :name => :threads,
+					:convert => :to_i,
+					:default => 20 },
+
+				{ :name => :breakdown,
+					:boolean => true }
+
+			]
+
+		@args.empty? \
+			or raise "Extra args on command line"
+
+	end
+
+	def get_namespace_stats db_name, namespace_name
+
+		mongo =
+			Thread.current[:mongo]
+
+		db =
+			mongo.db(db_name)
+
+		stats =
+			db.command({
+				"collStats" => namespace_name
+			})
+
+		return {
+			:data_size => stats["size"],
+			:storage_size => stats["storageSize"],
+		}
+
+	end
+
+	def get_collection_stats db_name, coll_name
+
+		mongo =
+			Thread.current[:mongo]
+
+		db =
+			mongo.db(db_name)
+
+		# get collection stats
+
+		coll_stats =
+			get_namespace_stats \
+				db_name,
+				coll_name
+
+		# get index stats
+
+		prefix_size =
+			db_name.length + 1 + coll_name.length + 1
+
+		index_names =
+			db["system.indexes"] \
+				.find({
+					"ns" => "#{db_name}.#{coll_name}",
+				})
+				.map { |row| row["name"] }
+
+		index_stats = Hash[
+			index_names.map do
+				|index_name|
+				[
+					index_name,
+					get_namespace_stats(
+						db_name,
+						"#{coll_name}.$#{index_name}")
+				]
+			end
+		]
+
+		# work out overall figures
+
+		total_stats =
+			index_stats.values.reduce coll_stats do
+				|memo, object|
+				{
+					:data_size =>
+						memo[:data_size] + object[:data_size],
+					:storage_size =>
+						memo[:storage_size] + object[:storage_size],
+				}
+			end
+
+		# get sharding info
+
+		config_db =
+			mongo.db("config")
+
+		collection_row =
+			config_db["collections"]
+				.find_one({
+					"_id" => "#{db_name}.#{coll_name}"
+				})
+
+		sharding_enabled =
+			collection_row && collection_row["key"] ? true : false
+
+		# and return
+
+		return {
+			:total => total_stats,
+			:collection => coll_stats,
+			:indexes => index_stats,
+			:sharding_enabled => sharding_enabled,
+		}
+
+	end
+
+	def perform_checks
+
+		# connect
+
+		mongo =
+			Mongo::Connection.new \
+				@opts[:hostname],
+				@opts[:port]
+
+		# stats to collect
+
+		@biggest = 0
+
+		@warning_count = 0
+		@critical_count = 0
+		@ok_count = 0
+		@error_count = 0
+
+		# thread pool
+
+		@thread_pool =
+			Tools::ThreadPool.new
+
+		@thread_pool.init_hook do
+
+			Thread.current[:mongo] =
+				Mongo::Connection.new \
+					@opts[:hostname],
+					@opts[:port]
+
+		end
+
+		@thread_pool.start \
+			@opts[:threads]
+
+		# get collection names (in parallel)
+
+		collection_name_futures =
+			mongo.database_names.map do
+				|database_name|
+
+				Tools::Future.new @thread_pool, database_name do
+					|database_name|
+
+					mongo =
+						Thread.current[:mongo]
+
+					mongo.db(database_name)
+						.collection_names
+						.map {
+							|collection_name|
+							[ database_name, collection_name ]
+						}
+
+				end
+
+			end
+
+		collection_names =
+			collection_name_futures
+				.map {
+					|future|
+					future.get
+				}
+				.flatten(1)
+
+		# analyse collections (in parallel)
+
+		collection_stat_futures = Hash[
+			collection_names.map do
+				|database_name, collection_name|
+
+				future =
+					Tools::Future.new \
+						@thread_pool,
+						database_name,
+						collection_name do
+							|database_name, collection_name|
+							get_collection_stats(
+								database_name,
+								collection_name)
+						end
+
+				[
+					"#{database_name}.#{collection_name}",
+					future,
+				]
+
+			end
+		]
+
+		collection_stat_futures.each do
+			|full_collection_name, future|
+
+			stats = future.get
+
+			analyse_collection \
+				full_collection_name,
+				stats
+
+		end
+
+		# results
+
+		critical "#{@critical_count} critical" \
+			if @critical_count > 0
+
+		warning "#{@warning_count} warning" \
+			if @warning_count > 0
+
+		unknown "#{@error_count} errors" \
+			if @error_count > 0
+
+		message "biggest is #{byte_size @biggest}"
+
+	end
+
+	def analyse_collection full_collection_name, stats
+
+		# work out total efficiency
+
+		total_data_size =
+			stats[:total][:data_size]
+
+		total_storage_size =
+			stats[:total][:storage_size]
+
+		total_efficiency =
+			total_data_size * 100 / total_storage_size
+
+		sharding_enabled =
+			stats[:sharding_enabled]
+
+		# work out critical and warning status
+
+		critical = false
+		warning = false
+
+		if total_data_size >= @opts[:total_critical]
+			critical = true
+		elsif total_data_size >= @opts[:total_warning]
+			warning = true
+		end
+
+		if sharding_enabled == false
+			if total_data_size >= @opts[:unsharded_critical]
+				critical = true
+			elsif total_data_size >= @opts[:unsharded_warning]
+				warning = true
+			end
+		end
+
+		if total_data_size >= @opts[:efficiency_size]
+			if total_efficiency < @opts[:efficiency_critical]
+				total_critical = true
+			elsif total_efficiency < @opts[:efficiency_warning]
+				warning = true
+			end
+		end
+
+		# update counters
+
+		if critical
+			@critical_count += 1
+		elsif warning
+			@warning_count += 1
+		else
+			@ok_count += 1
+		end
+
+		@biggest = total_data_size \
+			if total_data_size > @biggest
+
+		# output message(s)
+
+		if @opts[:verbose] && (critical || warning)
+
+			sharding_string =
+				if sharding_enabled
+					"sharded"
+				else
+					"unsharded"
+				end
+
+			status_string =
+				if critical
+					"*** CRITICAL ***"
+				elsif warning
+					"warning"
+				else
+					"ok"
+				end
+
+			print_n \
+				"%s %s %d%% %s %s" % [
+					"#{full_collection_name}",
+					byte_size(total_storage_size),
+					total_efficiency,
+					sharding_string,
+					status_string,
+				]
+
+			if @opts[:breakdown]
+
+				collection_data_size =
+					stats[:collection][:data_size]
+
+				collection_storage_size =
+					stats[:collection][:storage_size]
+
+				collection_efficiency =
+					collection_data_size * 100 / collection_storage_size
+
+				print_n \
+					"  data %s %d%%" % [
+						byte_size(collection_storage_size),
+						collection_efficiency,
+					]
+
+				stats[:indexes].each do
+					|index_name,
+					index_stats|
+
+					index_data_size =
+						index_stats[:data_size]
+
+					index_storage_size =
+						index_stats[:storage_size]
+
+					index_efficiency =
+						index_data_size * 100 / index_storage_size
+
+					print_n \
+						"  %s %s %d%%" % [
+							index_name,
+							byte_size(index_storage_size),
+							index_efficiency,
+						]
+
+				end
+
+			end
+
+		end
+
+	end
+
+	def print_n str
+		puts "#{str}\n"
+	end
+
+	def byte_size bytes
+
+		size = bytes.to_f
+
+		series = [
+			"bytes",
+			"kilobytes",
+			"megabytes",
+			"gigabytes",
+			"terabytes",
+		]
+
+		while size >= 1024
+			size /= 1024
+			series.shift
+		end
+
+		unit = series.shift
+
+		return "%.3g %s" % [ size, unit ]
+
+	end
+
+	def decode_bytes string
+
+		case string
+
+			when /^([0-9]+)b?$/
+
+			when /^([0-9]+)k$/
+				$1.to_i * 1024
+
+			when /^([0-9]+)m$/
+				$1.to_i * 1024 * 1024
+
+			when /^([0-9]+)g$/
+				$1.to_i * 1024 * 1024 * 1024
+
+			when /^([0-9]+)t$/
+				$1.to_i * 1024 * 1024 * 1024 * 1024
+
+			else
+				raise "Invalid size: #{string}"
+
+		end
+
+	end
+
+end
+
+end
+end
+end

metadata

@@ -0,0 +1,183 @@
+--- !ruby/object:Gem::Specification
+name: hq-mongodb-check-collection-size
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+  prerelease: 
+platform: ruby
+authors:
+- James Pharaoh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.8.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.8.5
+- !ruby/object:Gem::Dependency
+  name: hq-tools
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.8.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.8.5
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.7.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.7.7
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.13.0
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+description: HQ MongoDB script to check collection sizes, as a nagios/icinga plugin
+  or from the command line to generate reports
+email:
+- james@phsys.co.uk
+executables:
+- hq-mongodb-check-collection-size
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/hq/mongodb/check-collection-size/script.rb
+- features/runs.feature
+- features/support/steps.rb
+- features/support/env.rb
+- bin/hq-mongodb-check-collection-size
+homepage: https://github.com/jamespharaoh/hq-mongodb-check-collection-size
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: hq-mongodb-check-collection-size
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: HQ MongoDB check collection size
+test_files:
+- features/runs.feature
+- features/support/steps.rb
+- features/support/env.rb