cloud66 0.0.26

data/lib/plugins/redis_backup.rb

@@ -0,0 +1,95 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+
+class RedisBackup < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "6342c1ef0d8bb2a47ab1362a6b02c058", :name => "Redis Backup", :version => get_version }
+	end
+
+	def run(message)
+		pl = payload(message)
+
+		@job_name 			= pl['job_name'].gsub(/[^\w\s_-]+/, '').gsub(/(^|\b\s)\s+($|\s?\b)/, '\\1\\2').gsub(/\s/, '_')
+		@redisdump_utility 	= pl['redis_client'] || '/usr/bin/redis-cli'
+		@name 				= pl['db_name'] || 'dump'
+		@password 			= pl['password']
+		@socket 			= pl['socket']
+		@host 				= pl['host'] || '127.0.0.1'
+		@port 				= pl['port'] || 6379
+		@additional_options = pl['additional_options'] || []
+		@path 				= pl['db_path']
+		@dump_path			= pl['backup_folder']
+
+		@name = 'dump' if @name.empty?
+		@host = '127.0.0.1' if @host.empty?
+		@port = 6379 if @port.empty?
+		@additional_options = [] if @additional_options.empty?
+		@redisdump_utility = '/usr/bin/redis-cli' if @redisdump_utility.empty?
+
+		save_result = invoke_save
+		return save_result unless save_result.nil?
+
+		result = copy
+		if result[:ok]
+			run_result(true, "Redis Backup finished successfully")
+		else
+			run_result(false, result[:message])
+		end
+	end
+
+	private 
+
+	## based on Backup Gem with minor modifications
+
+	def database
+		"#{@name}.rdb"
+	end
+
+	def credential_options
+		@password.to_s.empty? ? '' : "-a '#{@password}'"
+	end
+
+	def connectivity_options
+		%w[host port socket].map do |option|
+			value = instance_variable_get("@#{option}")
+			next if value.to_s.empty?
+			"-#{option[0,1]} '#{value}'"
+		end.compact.join(' ')
+	end
+
+	def user_options
+		@additional_options.join(' ')
+	end
+
+	def invoke_save
+		command = "#{@redisdump_utility} #{credential_options} #{connectivity_options} #{user_options} SAVE"
+		@log.debug "Running #{command}"
+		response = run_shell(command)
+		@log.debug "redis-cli run result: #{response}"
+		unless response[:ok]
+			run_result(false, "Failed to save from server '#{response[:message]}'")
+		else
+			nil
+		end
+    end
+
+	def copy
+		src_path = File.join(@path, database)
+		unless File.exist?(src_path)
+			raise "Redis database dump not found at #{src_path}"
+		end
+
+		FileUtils.mkdir_p(@dump_path)
+
+		dst_path = File.join(@dump_path, database)
+		dump_cmd = "gzip -c #{src_path} > #{dst_path}.gz"
+		@log.debug "Running #{dump_cmd}"
+		run_shell dump_cmd
+	end
+
+end	

data/lib/plugins/s3_backup.rb

@@ -0,0 +1,200 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+require 'fog'
+
+class S3Backup < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "d3533989f9d542f393566511e8eb2090", :name => "S3 Backup", :version => get_version }
+	end
+
+	def run(message)
+		pl = payload(message)
+
+		@log.debug "Pruned payload #{pl}"
+
+		@access_key 			= pl['access_key']
+		@secret_key 			= pl['secret_key']
+		@bucket 				= pl['bucket']
+		@region 				= pl['region']
+		@remote_path 			= pl['remote_path']
+		@local_pattern 			= pl['local_pattern']
+		@keep 					= pl['keep'].empty? ? 0 : pl['keep'].to_i
+
+		@testing				= pl['testing']
+
+		return transfer
+	end
+
+	private
+
+	def get_connection
+		#Fog.mock! unless @testing.nil? || @testing == false
+		connection = Fog::Storage.new(
+			:provider               => 'AWS',
+			:aws_access_key_id     	=> @access_key,
+			:aws_secret_access_key	=> @secret_key,
+			:region                 => @region
+		)
+		connection
+	end
+
+	def transfer
+		begin
+			#set up the s3 connection
+			@connection = get_connection
+			#synchronie the file times
+			@connection.sync_clock
+			if @keep <= 0
+				sync_files_without_version_history
+			else	
+				sync_files_with_version_history
+			end
+
+		rescue Excon::Errors::SocketError => exc
+			@log.error exc.message
+			return run_result(false, exc.message)
+		rescue Excon::Errors::Error => exc
+			@log.error exc.message
+			result = exc.response.body
+			message = result.match(/\<Message\>(.*)\<\/Message\>/)
+			if !message.nil?
+				message = message[1] 
+				return run_result(false, message)
+			elsif exc.response.status == 404
+				return run_result(false, "Remote s3 service or bucket not found (404)")
+			elsif exc.response.status != 0
+				return run_result(false, "Remote s3 service returned error #{exc.response.status} without any more details")
+			else
+				return run_result(false, exc.message)
+			end
+		end
+	end
+
+	def remove_initial_slash(filename)
+		filename.sub(/^\//, '')
+	end
+
+	def ensure_trailing_slash(filename)
+		return "" if filename.empty?
+		filename.sub(/\/$/, '') + '/'
+	end
+
+	def sync_files_without_version_history
+
+		#prepare the remote directory variable
+		@remote_path = remove_initial_slash(ensure_trailing_slash(@remote_path))
+		count = 0
+
+		#for each local file match
+		Dir.glob(File.expand_path(@local_pattern)).each do |f|
+				
+			#skip to next match if current is a directory
+			next if File.directory?(f)
+
+			#assign the remote filename
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, f))
+			@log.debug "Copying #{f} to #{new_remote_filename}"
+			count += 1
+
+			#push file to s3
+			File.open(f, 'r') do |file| 
+				@connection.put_object(@bucket, new_remote_filename, file) 
+			end
+		end
+
+		return run_result(true, "Successully pushed #{count} file(s) to Amazon s3 bucket (without version history)")
+
+	end
+
+	def sync_files_with_version_history
+
+		#prepare the remote directory variable
+		@remote_path = remove_initial_slash(ensure_trailing_slash(@remote_path))
+		count = 0
+
+		#get remote directory
+		directory = @connection.directories.get(@bucket)
+
+		#cache the s3 remote directory identifing all appropriate files
+		remote_path_match = Regexp.new("^#{@remote_path}", "i")
+		s3_directory = directory.files.map {|f| f.key }
+
+		all_remote_files = s3_directory.select {|m| m =~ remote_path_match}.map {|m| remove_initial_slash(m.gsub(remote_path_match, ''))}
+		archive_regex = /(?<folder>^Archive_Cloud66 \(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\)\/)/
+		non_archive_files = all_remote_files.reject { |m| m =~ archive_regex }
+		archived_files = all_remote_files.select { |m| m =~ archive_regex }
+
+		new_archive_folder = "Archive_Cloud66 (#{Time.now.strftime("%Y-%m-%d %H:%M:%S")})/"
+		
+		#copy all non-archive files to new backup folder
+		non_archive_files.each do |relative_file|
+
+			puts "name is #{relative_file}"
+			
+			#move file to archive
+			existing_remote_filename = remove_initial_slash(File.join(@remote_path, relative_file))
+			new_remote_relative_filename = File.join(new_archive_folder, "#{relative_file}")
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, new_remote_relative_filename))
+				
+			@log.debug "Copying #{existing_remote_filename} to #{new_remote_filename}"
+			@connection.copy_object @bucket, existing_remote_filename, @bucket, new_remote_filename
+			
+			@log.debug "Removing #{existing_remote_filename}"
+			@connection.delete_object @bucket, existing_remote_filename
+
+			#add newly archived file to list of archived files
+			archived_files << new_remote_relative_filename
+		end
+
+		#copy up all new files from source
+		all_local_files = Dir.glob(File.expand_path(@local_pattern))
+		return run_result(true, "No file(s) identified to push to Amazon s3 bucket (with version history)") if all_local_files.size == 0
+ 
+		#determine a local root to create relative files (TODO?)
+		#local_root = ""
+		#local_root_regex = Regexp.new local_root
+		
+		#copy all local matches up to s3
+		all_local_files.each do |f|
+			
+			#skip to next match if current is a directory
+			next if File.directory?(f)
+				
+			#assign the remote filename
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, f))
+			@log.debug "Copying #{f} to #{new_remote_filename}"
+			count += 1
+			
+			#push file to s3
+			File.open(f, 'r') do |file|				
+				@connection.put_object @bucket, new_remote_filename, file
+			end
+
+		end
+
+		#get list of archive folders
+		archive_folders = archived_files.map {|m| archive_regex.match(m)['folder']}.uniq.sort.reverse
+
+		#if we have too many archive folders
+		while archive_folders.size > @keep do
+			archive_folder = archive_folders.delete_at(archive_folders.size-1)
+			archive_regex = Regexp.new "^#{Regexp.escape(archive_folder)}", "i"
+			
+			#remove old archived files
+			archived_files.select { |m| m =~ archive_regex }.each do |file|
+				remote_file_to_remove = remove_initial_slash(File.join(@remote_path, file))
+				@log.debug "Removing old archive file #{remote_file_to_remove}"
+				@connection.delete_object @bucket, remote_file_to_remove
+			end
+		end
+
+		return run_result(true, "Successully pushed #{count} file(s) to Amazon s3 bucket (with version history)")
+
+	end
+
+end	

data/lib/plugins/shell.rb

@@ -0,0 +1,26 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+
+class Shell < QuartzPlugin
+	
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "20e07c656e2f477d969e9561e13229fb", :name => "Shell", :version => get_version }
+	end
+
+	def run(message)
+		@log.debug "Running with #{message}"
+		payload = payload(message)
+		command = payload['command']
+		@log.info "Shell command '#{command}'"
+
+		begin
+			result = run_shell("#{command}")
+			run_result(result[:ok], result[:message])
+		rescue => ex
+			run_result(false, "Failed to run shell command due to #{ex}")
+		end
+	end
+end	

data/lib/plugins/tester.rb

@@ -0,0 +1,22 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+
+class Tester < QuartzPlugin
+	
+	@@version_major = 1
+	@@version_minor = 0
+	@@version_revision = 0
+
+	def info
+		{ :uid => "c0bb6ed7950b489f9abba8071ff0e0ab", :name => "Tester", :version => get_version }
+	end
+
+	def run(message)
+		@log.info "Running with #{message}"
+		i = Random.rand(10)
+		@log.info "Waiting for #{i} seconds"
+		sleep i
+		@log.info "Done"
+
+		run_result(true, "Super! Done in #{i} seconds")
+	end
+end	

data/lib/plugins/webget.rb

@@ -0,0 +1,39 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'httparty'
+
+class Webget < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "6b5f722d214f4d71a5be237d44094721", :name => "WebGet", :version => get_version }
+	end
+
+	def run(message)
+		@log.debug "Running with #{message}"
+		payload = payload(message)
+		url = payload['url']
+		local = payload['local file']
+		@log.info "Webget from #{url} into #{local}"
+
+		begin
+			response = HTTParty.get(url)
+			if response.code == 200
+				body = response.body
+				file = File.new(local, "w")
+				begin
+					file.write(body)
+					run_result(true, "Saved WebGet to local file")
+				ensure
+					file.close
+				end
+			else
+				run_result(false, response.message)
+			end
+		rescue => ex
+			run_result(false, "Failed to webget due to #{ex}")
+		end
+	end
+end	

data/lib/version.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+module Agent
+	class Version
+
+		##
+		# Change the MAJOR, MINOR and PATCH constants below
+		# to adjust the version of the Cloud66 Agent gem
+		#
+		# MAJOR:
+		#  Defines the major version
+		# MINOR:
+		#  Defines the minor version
+		# PATCH:
+		#  Defines the patch version
+		MAJOR, MINOR, PATCH  = 0, 0, 26
+
+		#ie. PRERELEASE_MODIFIER = 'beta1'
+		PRERELEASE_MODIFIER = nil
+
+		##
+		# Returns the major version ( big release based off of multiple minor releases )
+		def self.major
+			MAJOR
+		end
+
+		##
+		# Returns the minor version ( small release based off of multiple patches )
+		def self.minor
+			MINOR
+		end
+
+		##
+		# Returns the patch version ( updates, features and (crucial) bug fixes )
+		def self.patch
+			PATCH
+		end
+
+		##
+		# Returns the prerelease modifier ( not quite ready for public consumption )
+		def self.prerelease_modifier
+			PRERELEASE_MODIFIER
+		end
+
+		##
+		# Returns the current version of the Backup gem ( qualified for the gemspec )
+		def self.current
+			prerelease_modifier.nil? ? "#{major}.#{minor}.#{patch}" : "#{major}.#{minor}.#{patch}.#{prerelease_modifier}"
+		end
+
+	end
+end

data/lib/vital_signs_utils.rb

@@ -0,0 +1,161 @@
+require 'sys/filesystem'
+require 'socket'
+
+class VitalSignsUtils
+
+	def self.get_disk_usage_info
+		space_info = {}
+		Sys::Filesystem.mounts do |mount|
+			stat = Sys::Filesystem.stat(mount.mount_point)
+
+			#skip if this mount is not active
+			#next if stat.blocks_available == 0 && stat.blocks == 0
+
+			mb_free = Float(stat.block_size) * Float(stat.blocks_available) / 1000 / 1000
+			mb_total = Float(stat.block_size) * Float(stat.blocks) / 1000 / 1000
+			mb_used = mb_total - mb_free
+			percent_used = mb_total > 0.0 ? mb_used / mb_total * 100 : 0.0
+
+			space_info[mount.mount_point] = { mb_free: mb_free, mb_used: mb_used, mb_total: mb_total, percent_used: percent_used }
+			mount.mount_point
+		end
+		return space_info
+	rescue => exc
+		@log.error "Failure during disk usage gathering due to #{exc}"
+		return { invalid: exc.message }
+	end
+
+	def self.get_cpu_usage_info
+
+		#NOTE: we can get core-level info with mpstat -P ALL 1 1
+		#parse mpstat result
+		mpstat_result = `mpstat 1 5`
+
+#		mpstat_result = <<-SAMPLE
+#Linux 3.2.0-23-generic (precise64) 	12/07/2012 	_x86_64_	(2 CPU)
+#
+#10:42:50 AM  CPU    %usr   %nice    %sys   %ddle    %irq   %soft  %steal  %guest   %idle
+#10:42:51 AM  all    0.00    0.00    0.50    5.00    0.00    0.00    0.00    0.00   99.50
+#Average:     all    0.00    0.00    0.50    50.00    0.00    0.00    0.00    0.00   99.50
+#SAMPLE
+
+#split output into lines
+		lines = mpstat_result.split(/\r?\n/)
+
+		#get rid of time (first 13 chars)
+		lines = lines.map { |line| line[13..-1] }
+
+		#get the header line and split into columns
+		header_line = lines.detect { |line| line =~ /%idle/ }
+		columns = header_line.split(/\s+/)
+
+		#detect position of %idle column
+		idle_index = columns.index('%idle')
+
+		#get average line
+		average_line = lines[-1]
+		columns = average_line.split(/\s+/)
+
+		#get idle value
+		idle_string = columns[idle_index]
+		idle_value = idle_string.to_f
+
+		percent_used = 100.0 - idle_value
+
+		#get average utilization value
+		return { percent_used: percent_used, percent_free: idle_value }
+	rescue => exc
+		@log.error "Failure during CPU usage gathering due to #{exc}"
+		return { invalid: exc.message }
+	end
+
+	def self.get_memory_usage_info
+		free_m_result = `free -m`
+#		free_m_result = <<-SAMPLE
+#             total       used       free     shared    buffers     cached
+#Mem:           590        480        109          0         37        227
+#-/+ buffers/cache:        216        373
+#Swap:            0          0          0
+#SAMPLE
+
+		free_m_result.each_line do |line|
+			if line =~ /^Mem:/
+				parts = line.scan(/.*?(\d+)/)
+				parts.flatten!
+
+				mb_total = parts[0].to_f
+				# mb_used is not a true representation due to OS gobbling up mem
+				# mb_used = parts[1].to_i
+				mb_free = parts[2].to_f
+				mb_shared = parts[3].to_f
+				mb_buffers = parts[4].to_f
+				mb_cached = parts[5].to_f
+
+				#The total free memory available to proceses is calculated by adding up Mem:cached + Mem:buffers + Mem:free (99 + 63 + 296)
+				#This then needs to be divided by Mem:total to get the total available free memory (1692)
+				mb_available = mb_cached + mb_buffers + mb_free
+				mb_used = mb_total - mb_available
+				mb_free = mb_total - mb_used
+				percent_used = mb_used / mb_total * 100
+				return { mb_free: mb_free, mb_used: mb_used, mb_total: mb_total, percent_used: percent_used }
+			end
+		end
+	rescue => exc
+		@log.error "Failure during memory usage gathering due to #{exc}"
+		return { invalid: exc.message }
+	end
+
+	def self.get_facter_info
+		# facter command
+		facter_text = `sudo facter`
+		return parse_facter_text(facter_text)
+	end
+
+	def self.get_ip_address_info
+		# facter command for ip information (collect up to 5 local ip addresses)
+
+		facter_text = `facter ipaddress ec2_public_ipv4 ipaddress_eth0 ipaddress6 ipaddress6_eth0`
+		ip_hash = parse_facter_text(facter_text)
+
+		result = {}
+
+		if ip_hash.has_key?('ec2_public_ipv4')
+			# return ec2 info first (most specific)
+			result[:ext_ipv4] = ip_hash['ec2_public_ipv4']
+		elsif ip_hash.has_key?('ipaddress')
+			# return ipaddress next (general)
+			result[:ext_ipv4] = ip_hash['ipaddress']
+		end
+		result[:int_ipv4] = ip_hash['ipaddress_eth0'] if ip_hash.has_key?('ipaddress_eth0')
+		result[:ext_ipv6] = ip_hash['ipaddress6'] if ip_hash.has_key?('ipaddress6')
+		result[:int_ipv6] = ip_hash['ipaddress6_eth0'] if ip_hash.has_key?('ipaddress6_eth0')
+
+		# don't have any ip address info
+		return {} if result.empty?
+		# return ip address info
+		return { :ip_addresses => result }
+	end
+
+	# parse the factor text, not using YAML due to YAML parsing issues, and facter JSON output not working
+	def self.parse_facter_text(facter_text)
+		facter_hash = {}
+		facter_text.lines.each do |line|
+			split = line.split('=>')
+			key = split[0]
+			if split.size == 2
+				value = split[1]
+				if !value.nil?
+					value = value.strip
+					# exclude empty or long results (like ssh keys)
+					if !value.empty? && value.size < 100
+						key = key.strip
+						facter_hash[key] = value
+					end
+				end
+			end
+		end
+		return facter_hash
+	end
+end
+
+