cloud66 0.0.26

data/lib/cloud-quartz.rb

@@ -0,0 +1,64 @@
+require 'httparty'
+require 'json'
+
+class CloudQuartz
+	include HTTParty
+	@api_key = ""
+	@agent_id = ""
+	
+	def initialize(options = {})
+		@api_key = options[:api_key]
+		@agent_id = options[:agent_id]
+		@secret_key = options[:secret_key]
+		self.class.base_uri options[:url] || 'https://api.cloud66.com'
+	end
+
+	def get_job
+		process(self.class.get("/queue/#{@agent_id}.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key) } ))
+	end
+
+	def register(agent)
+		process(self.class.post('/agent.json', { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => agent.to_json }))
+	end
+
+	def unregister(agent)
+		process(self.class.delete("/agent/#{@agent_id}.json", :headers => ClientAuth.build_headers(@api_key, @secret_key)))
+	end
+
+	def post_results(job_id, data)
+		process(self.class.post("/job/#{job_id}/complete.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => data.to_json } ))
+	end
+
+	def pulse_without_ip_address
+		process(self.class.get("/agent/#{@agent_id}/pulse.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key) } ))
+	end
+
+	def pulse_with_ip_address(data)
+		process(self.class.post("/agent/#{@agent_id}/pulse.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => data.to_json } ))
+	end
+
+	def status(stat)
+		data = { :status => stat }
+		process(self.class.post("/agent/#{@agent_id}/status.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => data.to_json }))
+	end
+
+	def init(data)
+		process(self.class.post("/agent/#{@agent_id}/initialize.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => data.to_json }))
+	end
+
+	def send_vital_signs(data)
+		process(self.class.post("/agent/#{@agent_id}/vitalsigns.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => data.to_json } ))
+	end
+
+	private
+
+	def process(response)
+		if response.code != 200
+			raise response.body
+		else
+			response.parsed_response
+		end
+	end
+end
+
+

data/lib/plugins/backup.rb

@@ -0,0 +1,74 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'tempfile'
+require 'fileutils'
+
+class Backup < QuartzPlugin
+	
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "64cec894b98b43ce9e6047a9284a4b7d", :name => "Backup", :version => get_version }
+	end
+
+	def run(message)
+		
+		pl = payload(message)
+
+		job_name = pl['job_name']
+		backup_script = "#Script generated by Cloud66 job '#{job_name}'\n\n" + pl['script']
+		script_name = pl['script_name']
+
+		random_file = Tempfile.new('cloud66').path + '.rb'
+    	File.open(random_file, 'w') do |f|  
+      		f.puts backup_script
+    	end
+		
+		backup_root_directory = File.expand_path('~/.cloud66/backup')
+		FileUtils.mkdir_p(backup_root_directory)
+		
+		command = "backup perform --trigger #{script_name} --root-path #{backup_root_directory} --config_file #{random_file} 2>&1"
+		@log.info "Shell command '#{command}'"
+
+		begin
+			result = run_shell("#{command}")
+			data = 	result[:message]
+		
+			#regex to identify all the "error" lines
+			errorRegex = Regexp.new('(?<line>\[\d{4}\/\d{2}\/\d{2}\s\d{2}\:\d{2}\:\d{2}\]\[\\e\[31merror\\e\[0m\].*?)(\\n|$)')
+			errors = data.scan(errorRegex)
+			
+			if errors.size > 0
+				
+				#log all the errors
+				@log.info errors
+
+				#get rid of the backtrace
+				removeRegex = Regexp.new('(Backtrace\:|\d{2}\:\d{2}\]\[\\e\[31merror\\e\[0m\]\s*\/)')
+
+				#subregex to get rid of nasty timestamp and [\e[31merror\e[0m]
+				replaceRegex = Regexp.new('^.*\[\\e\[31merror\\e\[0m\]')
+				
+				errorResult = []
+				errors.each do |errorArr|
+					error = errorArr[0]
+					errorResult << error.gsub(replaceRegex,'').chomp unless error =~ removeRegex
+				end
+
+				run_result(false, errorResult.join("\n"))
+			else
+				if (result[:ok])
+					run_result(true, "Backup completed successfully!")
+				else
+					run_result(true, "An error occurred!")
+				end
+			end
+
+		rescue => ex
+			run_result(false, "Failed to execute backup due to #{ex}")
+		ensure 
+			File.delete random_file if File.exists? random_file
+		end
+	end
+end	

data/lib/plugins/broken.rb

@@ -0,0 +1,19 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+
+class Broken < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+	
+	def info
+		{ :uid => "04165a45fde840a9a17b41f019b3dca3", :name => "Broken", :version => get_version }
+	end
+
+	def run(message)
+		@log.info "Running with #{message}"
+		@log.info "This is a failure"
+
+		run_result(false, "Boo! It's broken")
+	end
+end	

data/lib/plugins/file_rotate.rb

@@ -0,0 +1,66 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+
+class FileRotate < QuartzPlugin
+
+        @@version_major = 0
+        @@version_minor = 0
+        @@version_revision = 1
+
+	def info
+		{ :uid => "02f7d8237bcc438e8f0659babfef2911", :name => "File Rotater", :version => get_version }
+	end
+
+	def run(message)
+		pl = payload(message)
+
+		source_pattern = pl['source_pattern']
+		dest_folder = pl['location']
+		keep = pl['keep'].nil? ? 5 : pl['keep'].to_i
+		post_rotate = pl['post_rotate']
+
+		archive = File.join(dest_folder, '/archive')
+
+        ext = Time.now.utc.strftime('%Y%m%d%H%M%S')
+
+        FileUtils.mkdir_p(archive)
+        rotated = false
+        Dir.glob(source_pattern).each do |f|
+        	rotated = true
+        	fname = File.basename(f)
+        	dest_filename = "#{fname}.#{ext}"
+        	dest_path = File.join(archive, dest_filename)
+        	@log.debug "Moving #{f} to #{dest_path}"
+
+        	move_shell = run_shell("mv #{f} #{dest_path}")
+        	return run_result(false, move_shell[:message]) unless move_shell[:ok]
+
+                return run_result(true, "Files moved successfully with no rotation") if keep == 0
+
+        	# find all the files from this one's rotations
+        	rotated_pattern = "#{File.join(archive, fname)}.*"
+        	@log.debug "Looking for rotated files #{rotated_pattern}"
+        	all_rotated = Dir.glob(rotated_pattern)
+        	if all_rotated.count > keep
+        		remove_count = all_rotated.count - keep
+	        	to_remote = all_rotated.sort! { |a,b| File.mtime(a) <=> File.mtime(b) }[0...remove_count]
+			@log.debug "Removing extra files"
+			to_remote.each do |tr|
+				@log.debug "Removing #{tr}"
+				remove_shell = run_shell("rm #{tr}")
+				return run_result(false, remove_shell[:message]) unless remove_shell[:ok]
+			end
+	        end
+        end
+
+        if !post_rotate.empty? && rotated
+        	@log.debug "Running post rotate step #{post_rotate}"
+        	post_shell = run_shell(post_rotate)
+        	return run_result(false, post_shell[:message]) unless post_shell[:ok]
+        end
+
+        result = rotated ? "Files rotated successfully" : "No files to rotate"
+
+        run_result(true, result)
+	end
+end

data/lib/plugins/log_rotate.rb

@@ -0,0 +1,65 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+
+class LogRotate < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "8f4286bfd946c8b08b234833673b8860", :name => "Log Rotate", :version => get_version }
+	end
+
+	def run(message)
+		pl = payload(message)
+
+		@source_pattern 	= pl['source_pattern']
+		@dest_folder 		= pl['destination']
+		@keep 				= pl['keep'].empty? ? 0 : pl['keep'].to_i
+		@post_run_step 		= pl['post_rotate']
+
+        ext = Time.now.utc.strftime('%Y%m%d%H%M%S')
+
+        FileUtils.mkdir_p(@dest_folder)
+
+		Dir.glob(@source_pattern).each do |f|
+			dest_file = File.join(@dest_folder, File.basename(f))
+			next if File.directory?(f)
+			fname = "#{dest_file}.gz"
+			dump_cmd = "cat #{f} | gzip > '#{fname}.#{ext}'"
+	        @log.debug "Running #{dump_cmd}"
+	        copy_result = run_shell(dump_cmd)
+
+	        return run_result(false, copy_result[:message]) unless copy_result[:ok]
+
+	        @log.debug "Removing source log files"
+	        remove_result = run_shell("rm #{f}")
+
+	        return run_result(false, remove_result[:message]) unless remove_result[:ok]
+
+	    	# find all the files from this one's rotations
+	    	rotated_pattern = "#{fname}.*"
+	    	@log.debug "Looking for rotated files #{rotated_pattern}"
+	    	all_rotated = Dir.glob(rotated_pattern)
+	    	if all_rotated.count > @keep
+	    		remove_count = all_rotated.count - @keep
+	        	to_remote = all_rotated.sort! { |a,b| File.mtime(a) <=> File.mtime(b) }[0...remove_count]
+				@log.debug "Removing extra #{remove_count} files"
+				to_remote.each do |tr|
+					@log.debug "Removing #{tr}"
+					remove_shell = run_shell("rm #{tr}")
+					return run_result(false, remove_shell[:message]) unless remove_shell[:ok]
+				end
+	        end
+        end
+
+        if !@post_run_step.nil? && !@post_run_step.empty?
+        	@log.debug "Running post rotate step #{@post_run_step}"
+        	post_shell = run_shell(@post_run_step)
+        	return run_result(false, post_shell[:message]) unless post_shell[:ok]
+        end
+
+        return run_result(true, "Log files rotated succesfully")
+   	end
+end	

data/lib/plugins/mysql_backup.rb

@@ -0,0 +1,92 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+
+class MysqlBackup < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "67deb35a555344c8a7651c656e6c8e2e", :name => "MySQL Backup", :version => get_version }
+	end
+
+	def run(message)
+		pl = payload(message)
+		pl = pl.select { |k,v| !v.nil? && !v.empty? }
+
+		@log.debug "Pruned payload #{pl}"
+
+		@job_name 			= pl['job_name'].gsub(/[^\w\s_-]+/, '').gsub(/(^|\b\s)\s+($|\s?\b)/, '\\1\\2').gsub(/\s/, '_')
+		@mysqldump_utility 	= pl['dump_utility'] || '/usr/bin/mysqldump'
+		@name 				= pl['db_name'] || :all
+		@username 			= pl['username']
+		@password 			= pl['password']
+		@socket 			= pl['socket']
+		@host 				= pl['host']
+		@port 				= pl['port']
+		@skip_tables 		= pl['skip_tables']
+		@only_tables 		= pl['only_tables']
+		@additional_options = pl['additional_options'] || ['--single-transaction', '--quick']
+		@path 				= pl['backup_folder']
+
+		@only_tables = @only_tables.split(',') unless @only_tables.nil?
+		@skip_tables = @skip_tables.split(',') unless @skip_tables.nil?
+
+        dump_cmd = "#{mysqldump} | gzip > '#{ File.join(@path, @job_name.downcase) }.sql.gz'"
+        @log.debug "Running #{dump_cmd}"
+        
+        FileUtils.mkdir_p(@path)
+
+		result = run_shell dump_cmd
+		if result[:ok]
+			run_result(true, "MySQL Backup finished successfully")
+		else
+			run_result(false, result[:message])
+		end
+	end
+
+	private 
+
+	# copied from backup gem with little mods
+	def mysqldump
+		"#{ @mysqldump_utility } #{ credential_options } #{ connectivity_options } " +
+		"#{ user_options } #{ @name } #{ tables_to_dump } #{ tables_to_skip }"
+	end
+
+	def credential_options
+		%w[username password].map do |option|
+			value = instance_variable_get("@#{option}")
+			next if value.to_s.empty?
+			"--#{option}='#{value}'".gsub('--username', '--user')
+		end.compact.join(' ')
+	end
+
+	def connectivity_options
+		%w[host port socket].map do |option|
+			value = instance_variable_get("@#{option}")
+			next if value.to_s.empty?
+			"--#{option}='#{value}'"
+		end.compact.join(' ')
+	end
+
+	def user_options
+		@additional_options.join(' ') unless @additional_options.nil?
+	end
+
+	def tables_to_dump
+		@only_tables.join(' ') unless @only_tables.nil? || dump_all?
+	end
+
+	def tables_to_skip
+		return '' if @skip_tables.nil?
+		@skip_tables.map do |table|
+			"--ignore-table='#{@name}.#{table}'"
+		end.join(' ') unless dump_all?
+	end
+
+	def dump_all?
+		@name == :all
+	end
+
+end	

data/lib/plugins/quartz_plugin.rb

@@ -0,0 +1,44 @@
+require 'open4'
+require 'json'
+
+class QuartzPlugin
+	
+	def initialize(log, options)
+		@log = log
+		@options = options
+	end	
+
+	def run_result(success, message)
+		result = { :ok => success, :message => message }
+		@log.debug "Job finished with result #{result}"
+
+		result
+	end
+
+	def payload(message)
+		v = message['payload']
+		@log.debug "Payload received: #{v}"
+		v = v.merge({'job_name' => message['job_name']})
+		@log.debug "Payload used: #{v}"
+		v
+	end
+
+	def run_shell(command)
+		pid, stdin, stdout, stderr = Open4::popen4("#{command}")
+		ignored, status = Process::waitpid2 pid
+
+		if status.exitstatus == 0
+			{ :ok => true, :message => stdout.read.strip }
+		else
+			{ :ok => false, :message => stderr.read.strip}
+		end
+	end
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def get_version
+		"#{@@version_major}.#{@@version_minor}.#{@@version_revision}"
+	end
+end

data/lib/plugins/rackspace_backup.rb

@@ -0,0 +1,207 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+require 'fog'
+require 'set'
+
+class RackspaceBackup < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "86a34908c51311e1a0a923db6188709b", :name => "Rackspace Backup", :version => get_version } 
+	end
+
+	def run(message)
+
+		pl = payload(message)
+		@log.debug "Pruned payload #{pl}"
+
+		@username 				= pl['username']
+		@api_key 				= pl['api_key']
+		@container 				= pl['container']
+		@remote_path 		 	= pl['remote_path']
+		@region					= pl['region']
+		@keep 					= pl['keep'].empty? ? 0 : pl['keep'].to_i
+		@local_pattern 			= pl['local_pattern']
+		@testing				= pl['testing']
+
+		return transfer
+	end
+
+	private
+
+	def get_connection
+		#Fog.mock! unless @testing.nil? || @testing == false	
+		if @region == 'europe'
+			connection = Fog::Storage.new(
+  				:provider           	=> 'Rackspace',
+  				:rackspace_username 	=> @username,
+  				:rackspace_api_key  	=> @api_key,
+  				:rackspace_auth_url 	=> "lon.auth.api.rackspacecloud.com"
+				)
+		else
+			connection = Fog::Storage.new(
+  				:provider           	=> 'Rackspace',
+  				:rackspace_username 	=> @username,
+  				:rackspace_api_key  	=> @api_key,
+  				)
+		end
+		connection
+
+	end
+
+	def transfer
+		begin
+			#set up the rackspace connection
+			@connection = get_connection
+			if @keep <= 0
+				sync_files_without_version_history
+			else	
+				sync_files_with_version_history
+			end
+
+		rescue Excon::Errors::SocketError => exc
+			@log.error exc.message
+			return run_result(false, exc.message)
+		rescue Excon::Errors::Error => exc
+			@log.error exc.message
+			result = exc.response.body
+			message = result.match(/\<Message\>(.*)\<\/Message\>/)
+			if !message.nil?
+				message = message[1] 
+				return run_result(false, message)
+			elsif exc.response.status == 404
+				return run_result(false, "Remote rackspace serivce or container not found (404)")
+			elsif exc.response.status != 0
+				return run_result(false, "Remote rackspace serivce returned error #{exc.response.status} without any more details")
+			else
+				return run_result(false, exc.message)
+			end
+		end
+	end
+
+	def remove_initial_slash(filename)
+		filename.sub(/^\//, '')
+	end
+
+	def ensure_trailing_slash(filename)
+		return "" if filename.empty?
+		filename.sub(/\/$/, '') + '/'
+	end
+
+	def sync_files_without_version_history
+
+		#prepare the remote directory variable
+		@remote_path = remove_initial_slash(ensure_trailing_slash(@remote_path))
+		count = 0
+
+		#for each local file match
+		Dir.glob(File.expand_path(@local_pattern)).each do |f|
+				
+			#skip to next match if current is a directory
+			next if File.directory?(f)
+
+			#assign the remote filename
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, f))
+			@log.debug "Copying #{f} to #{new_remote_filename}"
+			count += 1
+
+			#push file to rackspace
+			File.open(f, 'r') do |file| 
+				@connection.put_object(@container, new_remote_filename, file) 
+			end
+		end
+
+		return run_result(true, "Successully pushed #{count} file(s) to Rackspace Cloud Files container (without version history)")
+
+	end
+
+	def sync_files_with_version_history
+
+		#prepare the remote directory variable
+		@remote_path = remove_initial_slash(ensure_trailing_slash(@remote_path))
+		count = 0
+
+		#get remote directory
+		directory = @connection.directories.get(@container)
+
+		#cache the rackspace remote directory identifing all appropriate files
+		remote_path_match = Regexp.new("^#{@remote_path}", "i")
+		rackspace_directory = directory.files.map {|f| f.key }
+
+		all_remote_files = rackspace_directory.select {|m| m =~ remote_path_match}.map {|m| remove_initial_slash(m.gsub(remote_path_match, ''))}
+		archive_regex = /(?<folder>^Archive_Cloud66 \(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\)\/)/
+		non_archive_files = all_remote_files.reject { |m| m =~ archive_regex }
+		archived_files = all_remote_files.select { |m| m =~ archive_regex }
+
+		new_archive_folder = "Archive_Cloud66 (#{Time.now.strftime("%Y-%m-%d %H:%M:%S")})/"
+		
+		#copy all non-archive files to new backup folder
+		non_archive_files.each do |relative_file|
+
+			puts "name is #{relative_file}"
+			
+			#move file to archive
+			existing_remote_filename = remove_initial_slash(File.join(@remote_path, relative_file))
+			new_remote_relative_filename = File.join(new_archive_folder, "#{relative_file}")
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, new_remote_relative_filename))
+				
+			@log.debug "Copying #{existing_remote_filename} to #{new_remote_filename}"
+			@connection.copy_object @container, existing_remote_filename, @container, new_remote_filename
+			
+			@log.debug "Removing #{existing_remote_filename}"
+			@connection.delete_object @container, existing_remote_filename
+
+			#add newly archived file to list of archived files
+			archived_files << new_remote_relative_filename
+		end
+
+		#copy up all new files from source
+		all_local_files = Dir.glob(File.expand_path(@local_pattern))
+		return run_result(true, "No file(s) identified to push to Rackspace Cloud Files container (with version history)") if all_local_files.size == 0
+ 
+		#determine a local root to create relative files (TODO?)
+		#local_root = ""
+		#local_root_regex = Regexp.new local_root
+		
+		#copy all local matches up to rackspace
+		all_local_files.each do |f|
+			
+			#skip to next match if current is a directory
+			next if File.directory?(f)
+				
+			#assign the remote filename
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, f))
+			@log.debug "Copying #{f} to #{new_remote_filename}"
+			count += 1
+			
+			#push file to rackspace
+			File.open(f, 'r') do |file|				
+				@connection.put_object @container, new_remote_filename, file
+			end
+
+		end
+
+		#get list of archive folders
+		archive_folders = archived_files.map {|m| archive_regex.match(m)['folder']}.uniq.sort.reverse
+
+		#if we have too many archive folders
+		while archive_folders.size > @keep do
+			archive_folder = archive_folders.delete_at(archive_folders.size-1)
+			archive_regex = Regexp.new "^#{Regexp.escape(archive_folder)}", "i"
+			
+			#remove old archived files
+			archived_files.select { |m| m =~ archive_regex }.each do |file|
+				remote_file_to_remove = remove_initial_slash(File.join(@remote_path, file))
+				@log.debug "Removing old archive file #{remote_file_to_remove}"
+				@connection.delete_object @container, remote_file_to_remove
+			end
+		end
+
+		return run_result(true, "Successully pushed #{count} file(s) to Rackspace Cloud Files container (with version history)")
+
+	end
+
+end	

data/lib/plugins/rake.rb

@@ -0,0 +1,29 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+
+class Rake < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "62e3583abfc24f209916c4ff97661fa0", :name => "Rake", :version => get_version }
+	end
+
+	def run(message)
+		@log.debug "Running with #{message}"
+		payload = payload(message)
+		task = payload['task']
+		location = payload['location']
+		params = payload['parameters']
+		@log.info "Rake #{task} in #{location} with params:#{params}"
+
+		begin
+			command = "cd #{location} && bundle exec rake #{task} #{params}"
+			result = run_shell(command)
+			run_result(result[:ok], result[:message])
+		rescue => ex
+			run_result(false, "Failed to run rake due to #{ex}")
+		end
+	end
+end