cloud66 0.0.26

data/bin/c66-agent

@@ -0,0 +1,632 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require File.expand_path("../../lib/cloud-quartz", __FILE__)
+require File.expand_path('../../lib/version', __FILE__)
+require File.expand_path("../../lib/client_auth", __FILE__)
+require File.expand_path("../../lib/vital_signs_utils", __FILE__)
+require 'optparse'
+require 'socket'
+require 'logger'
+require 'highline/import'
+
+begin
+	gem 'eventmachine', '~>1.0.0.beta.4'
+	gem 'faye', '~>0.8.3'
+	gem 'highline', '~>1.6.11'
+
+	require 'eventmachine'
+	require 'faye'
+rescue LoadError => exc
+	warn "Cannot find required ruby gems needed to run this agent. Please install the cloud66 gem by running 'gem install cloud66'"
+	warn exc
+	exit -1
+end
+
+#statuses
+ST_UNREGISTERED = 0
+ST_STOPPED = 1
+ST_STARTED = 2
+
+#number of pulses between pulses that check for network settings
+NETWORK_CHECK_PULSE_FREQUENCY = 5
+
+private
+
+def safefork
+	tryagain = true
+
+	while tryagain
+		tryagain = false
+		begin
+			if pid = fork
+				return pid
+			end
+		rescue Errno::EWOULDBLOCK
+			sleep 5
+			tryagain = true
+		end
+	end
+end
+
+def daemonize(oldmode=0, closefd=false)
+	srand # Split rand streams between spawning and daemonized process
+	safefork and exit # Fork and exit from the parent
+
+	# Detach from the controlling terminal
+	unless sess_id = Process.setsid
+		raise 'Cannot detach from controlled terminal'
+	end
+
+	# Prevent the possibility of acquiring a controlling terminal
+	if oldmode.zero?
+		trap 'SIGHUP', 'IGNORE'
+		exit if pid = safefork
+	end
+
+	Dir.chdir "/" # Release old working directory
+	File.umask 0000 # Insure sensible umask
+
+	if closefd
+		# Make sure all file descriptors are closed
+		ObjectSpace.each_object(IO) do |io|
+			unless [STDIN, STDOUT, STDERR].include?(io)
+				io.close rescue nil
+			end
+		end
+	end
+
+	STDIN.reopen "/dev/null" # Free file descriptors and
+	STDOUT.reopen "/dev/null", "a" # point them somewhere sensible
+	STDERR.reopen STDOUT # STDOUT/STDERR should go to a logfile
+	return oldmode ? sess_id : 0 # Return value is mostly irrelevant
+end
+
+def save_config
+	Dir.mkdir(@config_dir) if !FileTest::directory?(@config_dir)
+	File.open(@config_full, 'w+') do |out|
+		data = {
+			'api_key' => @api_key,
+			'agent_id' => @agent_id,
+			'secret_key' => @secret_key
+		}
+		# store the url if it is different
+		data['url'] = @url if @url != 'https://api.cloud66.com'
+		# store the faye url if it is different
+		data['faye_url'] = @faye_url if @faye_url != 'https://sockets.cloud66.com/push'
+		YAML::dump(data, out)
+	end
+end
+
+def load_config
+	if File.exists?(@config_full)
+		# config file present
+		config = YAML::load(File.open(@config_full))
+		@api_key = config['api_key']
+		@agent_id = config['agent_id']
+		@secret_key = config['secret_key']
+
+		# set if it exists in the config
+		config_url = config['url']
+		@url = config_url if !config_url.nil? && !config_url.strip.empty?
+
+		# set if it exists in the config
+		config_faye_url = config['faye_url']
+		@faye_url = config_faye_url if !config_faye_url.nil? && !config_faye_url.strip.empty?
+	end
+end
+
+def delete_config
+	File.delete(@config_full) if File.exists?(@config_full)
+end
+
+def get_pid
+	if File.exists?(@pid_full)
+		file = File.new(@pid_full, "r")
+		pid = file.read
+		file.close
+		pid
+	else
+		0
+	end
+end
+
+def delete_pid
+	File.delete(@pid_full) if get_pid != 0
+end
+
+def pid_process_running?(pid)
+	begin
+		pid_number = pid.to_i
+		Process.getpgid(pid_number)
+		true
+	rescue Errno::ESRCH
+		false
+	end
+end
+
+def get_status
+	pid = get_pid
+	#check if the process is actually running
+	return ST_STARTED if pid != 0 && pid_process_running?(pid)
+	return ST_STOPPED if File.exists?(@config_full)
+	return ST_UNREGISTERED
+end
+
+public
+
+def register
+
+	if get_status != ST_UNREGISTERED
+		begin
+			unregister
+		rescue
+		end
+	end
+
+	puts 'Cloud 66 Agent Registration:' if @api_key.empty? || @secret_key.empty?
+	# still no api key, we need to get it
+	if @api_key.empty?
+		@api_key = ask('Please enter your API key. (you can find it at https://cloud66.com/me): ')
+		if @api_key.length != 32
+			puts 'Invalid API key'
+			exit -1
+		end
+	end
+	if @secret_key.empty?
+		@secret_key = ask('Please enter your Secret Key (you can find it at https://cloud66.com/me): ')
+		if @secret_key.length != 32
+			puts 'Invalid Secret key'
+			exit -1
+		end
+	end
+
+	@quartz = CloudQuartz.new(:api_key => @api_key, :url => @url, :secret_key => @secret_key)
+	puts "Registering the Cloud 66 Agent..."
+
+	os_name = RUBY_PLATFORM
+	os_id = os_name.include?('darwin') ? 5 : 1
+
+	timezone = Time.new.zone
+	agent = { :agent_type_id => os_id, :agent_name => @name, :agent_timezone => timezone, :extra => os_name, :server_uid => @server_uid }
+	result = @quartz.register(agent)
+
+	if result['ok']
+		@agent_id = result['uid']
+		puts "Registered successfully (and now ready to be started)!"
+		save_config
+	else
+		puts "Failed to register due to #{result['error']}"
+		exit -1
+	end
+end
+
+def unregister
+	begin
+		stop if get_status == ST_STARTED
+
+		load_config
+		@quartz = CloudQuartz.new(:api_key => @api_key, :url => @url, :agent_id => @agent_id, :secret_key => @secret_key)
+
+		puts "Unregistering the Cloud 66 Agent..."
+		@agent_id = ""
+		@quartz.unregister(@agent_id)
+
+	rescue
+	ensure
+		delete_config
+		puts "Unregistered successfully!"
+	end
+end
+
+def stop(signalCatch = false)
+
+	#signalCatch indicates a TERM or INT trap (where the app is already stopped, but server not told)
+	if !signalCatch && get_status == ST_STOPPED
+		puts "This agent was already stopped."
+
+		#delete the pid file just in case its hanging around
+		delete_pid
+
+		exit -1
+	end
+
+	#unregister the agent on the server
+	begin
+		load_config
+		@quartz = CloudQuartz.new(:api_key => @api_key, :url => @url, :agent_id => @agent_id, :secret_key => @secret_key)
+		puts "Stopping the Cloud 66 Agent..."
+		@log.debug "Stopping the Cloud 66 Agent..."
+		@quartz.status(2)
+	rescue
+	end
+
+	begin
+		EM.stop
+	rescue
+	end
+
+	pid = get_pid
+	if pid != 0
+		begin
+			Process.kill('TERM', pid.to_i)
+		rescue
+		end
+	end
+	delete_pid
+
+	puts "Stopped successfully!"
+	@log.debug "Stopped successfully!"
+end
+
+def start
+
+	if get_status == ST_STARTED
+		puts "This agent is already started. To stop it, please use the 'stop' command."
+		exit -1
+	end
+
+	#we know it isn't running, so delete leftover pid file if it exists
+	delete_pid
+
+	load_config
+	@quartz = CloudQuartz.new(:api_key => @api_key, :url => @url, :agent_id => @agent_id, :secret_key => @secret_key)
+	load_plugins
+
+	begin
+		begin
+			facter_data = VitalSignsUtils.get_facter_info
+		rescue => exc
+			facter_data = {}
+		end
+		@log.info @quartz.init({ :version => Agent::Version.current, plugins: plugin_meta_data, facter: facter_data })
+	rescue => exc
+		message = exc.message
+		if message =~ /Couldn't find Agent with uid =/
+			@log.warn "This agent is no longer registered at the server. The old registration details have been removed from this agent. Please re-run the agent to re-register it."
+			puts "This agent is no longer registered at the server. The old registration details have been removed from this agent. Please re-run the agent to re-register it."
+			puts @config_full
+			File.delete(@config_full)
+		else
+			@log.error exc.message
+		end
+		exit -1
+	end
+
+	puts "Starting the Cloud 66 Agent..."
+	if @daemon_mode
+		daemonize
+		pid = Process.pid
+		begin
+			file = File.new(@pid_full, "w")
+			file.write(pid)
+			file.close
+		rescue => exc
+			Process.kill('TERM', pid)
+			warn "Cannot start the Cloud 66 Agent: #{exc.message}"
+		end
+
+		run
+	else
+		run
+	end
+	exit 0
+end
+
+private
+
+def handle(result)
+	# if the server sends a shutdown signal
+	if !result.nil? && result.is_a?(Hash) && result['shut_down']
+		puts "Agent shutting down (server sent shut_down command)"
+		@log.debug "Agent shutting down (server sent shut_down command)"
+		# ensure bluepill doesn't bring this up again
+		`sudo bluepill cloud66_agent unmonitor` rescue nil
+		# stop the agent
+		stop(true)
+	end
+end
+
+def load_plugins
+	@load_path = File.expand_path(File.join(File.dirname(__FILE__), '../lib/plugins'))
+	@log.info "Loading plugins from #{@load_path}"
+
+	files = Dir.glob("#{@load_path}/*.rb")
+	files.each do |file|
+		unless file =~ /quartz_plugin/
+
+			# is it a valid plugin?
+			require "#{file}"
+			classname = File.basename(file, '.rb').split('_').collect { |part| part.capitalize }.join
+			begin
+				clazz = Kernel.const_get(classname)
+				if clazz.ancestors[1].name == 'QuartzPlugin'
+					instance = clazz.new(@log, { :api_key => @api_key, :agent_id => @agent_id })
+					guid = instance.info[:uid]
+					@plugins = @plugins.merge({ guid => instance })
+					@log.info "Found plugin #{instance.info[:name]}/#{instance.info[:version]} with uid #{guid}"
+				else
+					@log.error "Invalid plugin found #{clazz}"
+				end
+			rescue
+			end
+		end
+	end
+
+	@log.debug "All plugins #{plugin_meta_data}"
+end
+
+def plugin_meta_data
+	result = []
+	@plugins.each do |k, v|
+		result << v.info
+	end
+	result
+end
+
+def get_job
+	begin
+		result = @quartz.get_job
+		if result['ok']
+			if !result['empty']
+				message = JSON.parse(result['message'])
+				guid = message['plugin_uid']
+				name = message['template_name']
+				drt = message['desired_run_time']
+
+				@log.info "Going to run #{name} (uid:#{guid})"
+
+				# get the plugin
+				if @plugins.include?(guid)
+					plugin = @plugins[guid]
+
+					#run the job (new thread)
+					operation = proc { run_plugin(plugin, message) }
+					EM.defer(operation)
+
+					#drain the queue until it is empty
+					get_job
+				else
+					@log.error "No plugin found with uid #{guid}"
+					job_id = message['job_id']
+					data = { :run_start => Time.now.utc.to_i, :run_end => Time.now.utc.to_i, :agent_uid => @agent_id, :ok => false, :fail_reason => "Requested plugin not found. Does this agent support this job type?" }
+					@quartz.post_results(job_id, data)
+				end
+			end
+		else
+			@log.error "Failed to retrieve job due to #{result['error']}"
+		end
+	rescue => exc
+		@log.error "Failed to retrieve job due to #{exc}"
+	end
+end
+
+def run_plugin(plugin, message)
+	run_start = Time.now.utc.to_i
+	begin
+		job_id = message['job_id']
+		result = plugin.run(message)
+		@log.debug "Run returned for job #{job_id} with #{result}"
+		@log.debug result
+		ok = result[:ok]
+		to_return = result[:message]
+	rescue => exc
+		@log.error "Failure during execution of plugin #{plugin} due to #{exc}"
+		ok = false
+		if result.nil?
+			to_return = exc.message
+		else
+			to_return = result[:message]
+		end
+	ensure
+		data = { :run_start => run_start, :run_end => Time.now.utc.to_i, :agent_uid => @agent_id, :ok => ok }
+		data = ok ? data.merge({ :run_result => to_return }) : data.merge({ :fail_reason => to_return })
+		begin
+			@log.debug "Posting results for job #{job_id} back to the server #{data}"
+			@quartz.post_results(job_id, data)
+		rescue => e
+			@log.error "Failed to post results back to server due to #{e}"
+		end
+	end
+end
+
+def pulsate
+	begin
+		if @pulse_count == NETWORK_CHECK_PULSE_FREQUENCY
+
+			#reset pulse count
+			@pulse_count = 0
+
+			begin
+				handle(@quartz.pulse_with_ip_address(VitalSignsUtils.get_ip_address_info))
+			rescue => exc
+				#do a normal pulse if we have any detection issues
+				handle(@quartz.pulse_without_ip_address)
+			end
+		else
+			handle(@quartz.pulse_without_ip_address)
+
+		end
+		@pulse_count += 1
+	rescue => exc
+		@log.error "Failed to pulsate due to #{exc.message}"
+	end
+end
+
+def update_vital_signs
+	begin
+		data = {
+			:disk => VitalSignsUtils.get_disk_usage_info,
+			:cpu => VitalSignsUtils.get_cpu_usage_info,
+			:memory => VitalSignsUtils.get_memory_usage_info
+		}
+		handle(@quartz.send_vital_signs(data))
+	rescue => exc
+		@log.error "Failed to update vital signs due to #{exc.message}"
+	end
+end
+
+def run
+	EM.run {
+		Signal.trap('INT') { @log.debug("trapped INT signal"); stop(true) }
+		Signal.trap('TERM') { @log.debug("trapped TERM signal"); stop(true) }
+
+		# pulse
+		@pulse_count = NETWORK_CHECK_PULSE_FREQUENCY
+		pulsate
+		update_vital_signs
+		begin
+
+			EM.add_periodic_timer 60 do
+				pulsate
+			end
+			EM.add_periodic_timer 1800 do
+				update_vital_signs
+			end
+		rescue => exc
+			@log.error "Unable to add EM timer due to: #{exc.message}"
+			@log.error "#{exc.backtrace}"
+			exit -1
+		end
+
+		if @realtime
+			channel = "/agent_user/#{@api_key}/agent/#{@agent_id}/newjob"
+			@log.info "Listening to realtime notifications from '#{channel}' on '#{@faye_url}'"
+			client = Faye::Client.new(@faye_url)
+			client.subscribe(channel) do |message|
+				@log.info "Got realtime notice for a new job #{message}"
+				get_job
+			end
+		else
+			@log.info "Checking for new jobs every 60 seconds"
+			#reduced job check frequency (for stale checker)
+			EM.add_periodic_timer 45 do
+				get_job
+			end
+		end
+	}
+end
+
+public
+
+config_file = 'agent.yml'
+@pid_file = 'c66-agent.pid'
+@log_file = 'c66-agent.log'
+@config_dir = '/etc/cloud66/'
+@config_full = File.join(@config_dir, config_file)
+@cb_tmp_dir = '/tmp/cloud66'
+Dir.mkdir(@cb_tmp_dir) if !File.exists?(@cb_tmp_dir)
+
+@pid_full = File.join(@cb_tmp_dir, @pid_file)
+@log_full = File.join(@cb_tmp_dir, @log_file)
+commands = %w[register unregister start stop]
+@plugins = {}
+command = nil
+
+
+OptionParser.new do |opts|
+	opts.banner = <<-EOF
+	Cloud 66 Agent. v#{Agent::Version.current} (c) 2012 Cloud 66
+	For more information please visit http://cloud66.com
+
+	Usage: c66-agent [register|unregister|start|stop] [options]
+
+	Options:
+	EOF
+
+	opts.on('--url URL', 'Server URL') do |server_url|
+		@url = server_url
+	end
+	@url ||= 'https://api.cloud66.com'
+
+	opts.on('-d', '--no-daemon', 'Not in daemon mode') do |v|
+		@daemon_mode = false
+	end
+	@daemon_mode ||= true
+
+	opts.on('-p', '--pid PID', 'PID file path') do |v|
+		@pid_full = v
+	end
+
+	opts.on('-l', '--log LOG', 'Full log file path') do |v|
+		@log_full = v
+	end
+
+	opts.on('-n', '--name NAME', 'Name of this agent') do |v|
+		@name = v
+	end
+	@name = Socket.gethostname if @name.nil? || @name.empty?
+
+	opts.on('-c', '--config CONFIG', 'Config file path') do |v|
+		@config_full = v
+	end
+
+	opts.on('--sockets SOCKETS', 'Sockets URL') do |v|
+		@faye_url = v
+	end
+	@faye_url ||= 'https://sockets.cloud66.com/push'
+
+	opts.on('--api-key APIKEY', 'API key') do |v|
+		@api_key = v
+	end
+	@api_key ||= ''
+
+	opts.on('--secret-key SECRETKET', 'Secret Key') do |v|
+		@secret_key = v
+	end
+	@secret_key ||= ''
+
+	opts.on('--server SERVERUID', 'Server id') do |v|
+		@server_uid = v
+	end
+
+	opts.on('-r', '--realtime', 'Enable realtime notifications (default)') do |v|
+		@realtime = true
+	end
+	opts.on('-R', '--no-realtime', 'Disable realtime notifications') do |v|
+		@realtime = false
+	end
+	@realtime ||= true
+
+	opts.on_tail("-h", "--help", "Show this message") do
+		puts opts
+		puts <<-EOF
+
+		Commands:
+		register 			Register the Cloud 66 Agent
+		start 				Starts the Cloud 66 Agent as a deamon
+		stop 				Stops the Cloud 66 Agent daemon
+		unregister			Unregisters the Cloud 66 Agent
+
+		EOF
+		exit 0
+	end
+end.parse!
+
+#set logging output
+@log = @daemon_mode ? Logger.new(@log_full) : Logger.new(STDOUT)
+@log.level = Logger::DEBUG
+
+#return status information
+command = ARGV[0].downcase unless ARGV[0].nil?
+if command.nil? || command.empty?
+	status = get_status
+	puts "v#{Agent::Version.current} Started (use --help for commands)" if status == ST_STARTED
+	puts "v#{Agent::Version.current} Stopped (use --help for commands)" if status == ST_STOPPED
+	puts "v#{Agent::Version.current} Unregistered (use --help for commands)" if status == ST_UNREGISTERED
+	exit -1
+end
+
+unless commands.include?(command)
+	puts 'Invalid command. Use --help for more information'
+	exit -1
+end
+
+begin
+	send(command)
+rescue => exc
+	@log.error exc.message
+	puts "An error has occurred: #{exc.message}"
+	exit -1
+end
+

data/lib/client_auth.rb

@@ -0,0 +1,32 @@
+class ClientAuth
+
+	def initialize api_key, secret_key
+		@headers = self.class.build_headers api_key, secret_key
+	end
+
+	def outgoing(message, callback)
+ 
+    	# Again, leave non-subscribe messages alone
+    	if message['channel'] != '/meta/subscribe'
+    		return callback.call(message)
+    	end
+
+    	# Add ext field if it's not present	
+    	message['ext'] ||= {}
+
+    	# Set the tokens
+    	message['ext']['api_key'] = @headers['api_key']
+    	message['ext']['hash'] = @headers['hash']
+    	message['ext']['time'] = @headers['time']
+
+    	# Carry on and send the message to the server
+    	callback.call(message)
+    end
+
+    def self.build_headers api_key, secret_key
+		time = Time.now.utc.to_i
+		hash = Digest::SHA1.hexdigest("#{api_key}#{secret_key}#{time}").downcase
+		{ 'api_key' => api_key, 'hash' => hash, 'time' => time.to_s }
+	end
+
+end