cloudblocks 0.0.14 → 0.0.15

data/bin/cb-agent

@@ -50,13 +50,26 @@ def start
 
 		check_version
 		load_plugins
-
-		@log.info @quartz.status(1, @version, plugin_meta_data)
 	rescue => exc
 		@log.error exc.message
 		exit -1
 	end
 
+	begin
+		@log.info @quartz.status(1, @version, plugin_meta_data)
+	rescue Exception => e
+		message = e.message
+		if message =~ /Couldn't find Agent with uid =/
+			@log.warn "This agent is no longer registered at the server. The old registration details have been removed from this agent. Please re-run the agent to re-register it."
+			puts "This agent is no longer registered at the server. The old registration details have been removed from this agent. Please re-run the agent to re-register it."
+			puts @config_full
+		File.delete(@config_full)
+		else
+			@log.error e.message
+		end
+		exit -1
+	end
+
 	if @daemon_mode
 		pid = fork {
 			run
@@ -137,10 +150,11 @@ def load_plugins
 
 	files = Dir.glob("#{@load_path}/*.rb")
 	files.each do |file|
-		if file != 'quartz_plugin'
+		unless file =~ /quartz_plugin/
+
 			# is it a valid plugin?
 			require "#{file}"
-			classname = File.basename(file, '.rb').capitalize
+			classname = File.basename(file, '.rb').split('_').collect{ |part| part.capitalize }.join
 			begin
 				clazz = Kernel.const_get(classname)
 				if clazz.ancestors[1].name == 'QuartzPlugin'

data/lib/plugins/{rotater.rb → file_rotate.rb}

@@ -1,7 +1,7 @@
 require File.join(File.dirname(__FILE__), 'quartz_plugin')
 require 'fileutils'
 
-class Rotater < QuartzPlugin
+class FileRotate < QuartzPlugin
 
         @@version_major = 0
         @@version_minor = 0

data/lib/plugins/{logrotate.rb → log_rotate.rb}

@@ -1,7 +1,7 @@
 require File.join(File.dirname(__FILE__), 'quartz_plugin')
 require 'fileutils'
 
-class Logrotate < QuartzPlugin
+class LogRotate < QuartzPlugin
 
 	@@version_major = 0
 	@@version_minor = 0

data/lib/plugins/{mysql.rb → mysql_backup.rb}

@@ -1,7 +1,7 @@
 require File.join(File.dirname(__FILE__), 'quartz_plugin')
 require 'fileutils'
 
-class Mysql < QuartzPlugin
+class MysqlBackup < QuartzPlugin
 
 	@@version_major = 0
 	@@version_minor = 0

data/lib/plugins/quartz_plugin.rb

@@ -16,7 +16,6 @@ class QuartzPlugin
 	end
 
 	def payload(message)
-		@log.debug "Message #{message}"
 		raw_payload = message['payload']
 		@log.debug "Payload #{raw_payload}"
 		parsed_payload = JSON.parse(raw_payload) unless raw_payload.nil?

data/lib/plugins/rackspace_backup.rb

@@ -0,0 +1,207 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+require 'fog'
+require 'set'
+
+class RackspaceBackup < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "86a34908c51311e1a0a923db6188709b", :name => "Rackspace Backup", :version => get_version } 
+	end
+
+	def run(message)
+
+		pl = payload(message)
+		@log.debug "Pruned payload #{pl}"
+
+		@username 				= pl['username']
+		@api_key 				= pl['api_key']
+		@container 				= pl['container']
+		@remote_path 		 	= pl['remote_path']
+		@region					= pl['region']
+		@keep 					= pl['keep'].empty? ? 0 : pl['keep'].to_i
+		@local_pattern 			= pl['local_pattern']
+		@testing				= pl['testing']
+
+		return transfer
+	end
+
+	private
+
+	def get_connection
+		#Fog.mock! unless @testing.nil? || @testing == false	
+		if @region == 'europe'
+			connection = Fog::Storage.new(
+  				:provider           	=> 'Rackspace',
+  				:rackspace_username 	=> @username,
+  				:rackspace_api_key  	=> @api_key,
+  				:rackspace_auth_url 	=> "lon.auth.api.rackspacecloud.com"
+				)
+		else
+			connection = Fog::Storage.new(
+  				:provider           	=> 'Rackspace',
+  				:rackspace_username 	=> @username,
+  				:rackspace_api_key  	=> @api_key,
+  				)
+		end
+		connection
+
+	end
+
+	def transfer
+		begin
+			#set up the rackspace connection
+			@connection = get_connection
+			if @keep <= 0
+				sync_files_without_version_history
+			else	
+				sync_files_with_version_history
+			end
+
+		rescue Excon::Errors::SocketError => exc
+			@log.error exc.message
+			return run_result(false, exc.message)
+		rescue Excon::Errors::Error => exc
+			@log.error exc.message
+			result = exc.response.body
+			message = result.match(/\<Message\>(.*)\<\/Message\>/)
+			if !message.nil?
+				message = message[1] 
+				return run_result(false, message)
+			elsif exc.response.status == 404
+				return run_result(false, "Remote rackspace serivce or container not found (404)")
+			elsif exc.response.status != 0
+				return run_result(false, "Remote rackspace serivce returned error #{exc.response.status} without any more details")
+			else
+				return run_result(false, exc.message)
+			end
+		end
+	end
+
+	def remove_initial_slash(filename)
+		filename.sub(/^\//, '')
+	end
+
+	def ensure_trailing_slash(filename)
+		return "" if filename.empty?
+		filename.sub(/\/$/, '') + '/'
+	end
+
+	def sync_files_without_version_history
+
+		#prepare the remote directory variable
+		@remote_path = remove_initial_slash(ensure_trailing_slash(@remote_path))
+		count = 0
+
+		#for each local file match
+		Dir.glob(File.expand_path(@local_pattern)).each do |f|
+				
+			#skip to next match if current is a directory
+			next if File.directory?(f)
+
+			#assign the remote filename
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, f))
+			@log.debug "Copying #{f} to #{new_remote_filename}"
+			count += 1
+
+			#push file to rackspace
+			File.open(f, 'r') do |file| 
+				@connection.put_object(@container, new_remote_filename, file) 
+			end
+		end
+
+		return run_result(true, "Successully pushed #{count} file(s) to Rackspace Cloud Files container (without version history)")
+
+	end
+
+	def sync_files_with_version_history
+
+		#prepare the remote directory variable
+		@remote_path = remove_initial_slash(ensure_trailing_slash(@remote_path))
+		count = 0
+
+		#get remote directory
+		directory = @connection.directories.get(@container)
+
+		#cache the rackspace remote directory identifing all appropriate files
+		remote_path_match = Regexp.new("^#{@remote_path}", "i")
+		rackspace_directory = directory.files.map {|f| f.key }
+
+		all_remote_files = rackspace_directory.select {|m| m =~ remote_path_match}.map {|m| remove_initial_slash(m.gsub(remote_path_match, ''))}
+		archive_regex = /(?<folder>^Archive_CloudBlocks \(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\)\/)/
+		non_archive_files = all_remote_files.reject { |m| m =~ archive_regex }
+		archived_files = all_remote_files.select { |m| m =~ archive_regex }
+
+		new_archive_folder = "Archive_CloudBlocks (#{Time.now.strftime("%Y-%m-%d %H:%M:%S")})/"
+		
+		#copy all non-archive files to new backup folder
+		non_archive_files.each do |relative_file|
+
+			puts "name is #{relative_file}"
+			
+			#move file to archive
+			existing_remote_filename = remove_initial_slash(File.join(@remote_path, relative_file))
+			new_remote_relative_filename = File.join(new_archive_folder, "#{relative_file}")
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, new_remote_relative_filename))
+				
+			@log.debug "Copying #{existing_remote_filename} to #{new_remote_filename}"
+			@connection.copy_object @container, existing_remote_filename, @container, new_remote_filename
+			
+			@log.debug "Removing #{existing_remote_filename}"
+			@connection.delete_object @container, existing_remote_filename
+
+			#add newly archived file to list of archived files
+			archived_files << new_remote_relative_filename
+		end
+
+		#copy up all new files from source
+		all_local_files = Dir.glob(File.expand_path(@local_pattern))
+		return run_result(true, "No file(s) identified to push to Rackspace Cloud Files container (with version history)") if all_local_files.size == 0
+ 
+		#determine a local root to create relative files (TODO?)
+		#local_root = ""
+		#local_root_regex = Regexp.new local_root
+		
+		#copy all local matches up to rackspace
+		all_local_files.each do |f|
+			
+			#skip to next match if current is a directory
+			next if File.directory?(f)
+				
+			#assign the remote filename
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, f))
+			@log.debug "Copying #{f} to #{new_remote_filename}"
+			count += 1
+			
+			#push file to rackspace
+			File.open(f, 'r') do |file|				
+				@connection.put_object @container, new_remote_filename, file
+			end
+
+		end
+
+		#get list of archive folders
+		archive_folders = archived_files.map {|m| archive_regex.match(m)['folder']}.uniq.sort.reverse
+
+		#if we have too many archive folders
+		while archive_folders.size > @keep do
+			archive_folder = archive_folders.delete_at(archive_folders.size-1)
+			archive_regex = Regexp.new "^#{Regexp.escape(archive_folder)}", "i"
+			
+			#remove old archived files
+			archived_files.select { |m| m =~ archive_regex }.each do |file|
+				remote_file_to_remove = remove_initial_slash(File.join(@remote_path, file))
+				@log.debug "Removing old archive file #{remote_file_to_remove}"
+				@connection.delete_object @container, remote_file_to_remove
+			end
+		end
+
+		return run_result(true, "Successully pushed #{count} file(s) to Rackspace Cloud Files container (with version history)")
+
+	end
+
+end	

data/lib/plugins/{redis.rb → redis_backup.rb}

@@ -1,7 +1,7 @@
 require File.join(File.dirname(__FILE__), 'quartz_plugin')
 require 'fileutils'
 
-class Redis < QuartzPlugin
+class RedisBackup < QuartzPlugin
 
 	@@version_major = 0
 	@@version_minor = 0

data/lib/plugins/s3_backup.rb

@@ -0,0 +1,200 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+require 'fog'
+
+class S3Backup < QuartzPlugin
+
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "d3533989f9d542f393566511e8eb2090", :name => "S3 Backup", :version => get_version }
+	end
+
+	def run(message)
+		pl = payload(message)
+
+		@log.debug "Pruned payload #{pl}"
+
+		@access_key 			= pl['access_key']
+		@secret_key 			= pl['secret_key']
+		@bucket 				= pl['bucket']
+		@region 				= pl['region']
+		@remote_path 		= pl['remote_path']
+		@local_pattern 			= pl['local_pattern']
+		@keep 					= pl['keep'].empty? ? 0 : pl['keep'].to_i
+
+		@testing				= pl['testing']
+
+		return transfer
+	end
+
+	private
+
+	def get_connection
+		#Fog.mock! unless @testing.nil? || @testing == false
+		connection = Fog::Storage.new(
+			:provider               => 'AWS',
+			:aws_access_key_id     	=> @access_key,
+			:aws_secret_access_key	=> @secret_key,
+			:region                 => @region
+		)
+		connection
+	end
+
+	def transfer
+		begin
+			#set up the s3 connection
+			@connection = get_connection
+			#synchronie the file times
+			@connection.sync_clock
+			if @keep <= 0
+				sync_files_without_version_history
+			else	
+				sync_files_with_version_history
+			end
+
+		rescue Excon::Errors::SocketError => exc
+			@log.error exc.message
+			return run_result(false, exc.message)
+		rescue Excon::Errors::Error => exc
+			@log.error exc.message
+			result = exc.response.body
+			message = result.match(/\<Message\>(.*)\<\/Message\>/)
+			if !message.nil?
+				message = message[1] 
+				return run_result(false, message)
+			elsif exc.response.status == 404
+				return run_result(false, "Remote s3 service or bucket not found (404)")
+			elsif exc.response.status != 0
+				return run_result(false, "Remote s3 service returned error #{exc.response.status} without any more details")
+			else
+				return run_result(false, exc.message)
+			end
+		end
+	end
+
+	def remove_initial_slash(filename)
+		filename.sub(/^\//, '')
+	end
+
+	def ensure_trailing_slash(filename)
+		return "" if filename.empty?
+		filename.sub(/\/$/, '') + '/'
+	end
+
+	def sync_files_without_version_history
+
+		#prepare the remote directory variable
+		@remote_path = remove_initial_slash(ensure_trailing_slash(@remote_path))
+		count = 0
+
+		#for each local file match
+		Dir.glob(File.expand_path(@local_pattern)).each do |f|
+				
+			#skip to next match if current is a directory
+			next if File.directory?(f)
+
+			#assign the remote filename
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, f))
+			@log.debug "Copying #{f} to #{new_remote_filename}"
+			count += 1
+
+			#push file to s3
+			File.open(f, 'r') do |file| 
+				@connection.put_object(@bucket, new_remote_filename, file) 
+			end
+		end
+
+		return run_result(true, "Successully pushed #{count} file(s) to Amazon s3 bucket (without version history)")
+
+	end
+
+	def sync_files_with_version_history
+
+		#prepare the remote directory variable
+		@remote_path = remove_initial_slash(ensure_trailing_slash(@remote_path))
+		count = 0
+
+		#get remote directory
+		directory = @connection.directories.get(@bucket)
+
+		#cache the s3 remote directory identifing all appropriate files
+		remote_path_match = Regexp.new("^#{@remote_path}", "i")
+		s3_directory = directory.files.map {|f| f.key }
+
+		all_remote_files = s3_directory.select {|m| m =~ remote_path_match}.map {|m| remove_initial_slash(m.gsub(remote_path_match, ''))}
+		archive_regex = /(?<folder>^Archive_CloudBlocks \(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\)\/)/
+		non_archive_files = all_remote_files.reject { |m| m =~ archive_regex }
+		archived_files = all_remote_files.select { |m| m =~ archive_regex }
+
+		new_archive_folder = "Archive_CloudBlocks (#{Time.now.strftime("%Y-%m-%d %H:%M:%S")})/"
+		
+		#copy all non-archive files to new backup folder
+		non_archive_files.each do |relative_file|
+
+			puts "name is #{relative_file}"
+			
+			#move file to archive
+			existing_remote_filename = remove_initial_slash(File.join(@remote_path, relative_file))
+			new_remote_relative_filename = File.join(new_archive_folder, "#{relative_file}")
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, new_remote_relative_filename))
+				
+			@log.debug "Copying #{existing_remote_filename} to #{new_remote_filename}"
+			@connection.copy_object @bucket, existing_remote_filename, @bucket, new_remote_filename
+			
+			@log.debug "Removing #{existing_remote_filename}"
+			@connection.delete_object @bucket, existing_remote_filename
+
+			#add newly archived file to list of archived files
+			archived_files << new_remote_relative_filename
+		end
+
+		#copy up all new files from source
+		all_local_files = Dir.glob(File.expand_path(@local_pattern))
+		return run_result(true, "No file(s) identified to push to Amazon s3 bucket (with version history)") if all_local_files.size == 0
+ 
+		#determine a local root to create relative files (TODO?)
+		#local_root = ""
+		#local_root_regex = Regexp.new local_root
+		
+		#copy all local matches up to s3
+		all_local_files.each do |f|
+			
+			#skip to next match if current is a directory
+			next if File.directory?(f)
+				
+			#assign the remote filename
+			new_remote_filename = remove_initial_slash(File.join(@remote_path, f))
+			@log.debug "Copying #{f} to #{new_remote_filename}"
+			count += 1
+			
+			#push file to s3
+			File.open(f, 'r') do |file|				
+				@connection.put_object @bucket, new_remote_filename, file
+			end
+
+		end
+
+		#get list of archive folders
+		archive_folders = archived_files.map {|m| archive_regex.match(m)['folder']}.uniq.sort.reverse
+
+		#if we have too many archive folders
+		while archive_folders.size > @keep do
+			archive_folder = archive_folders.delete_at(archive_folders.size-1)
+			archive_regex = Regexp.new "^#{Regexp.escape(archive_folder)}", "i"
+			
+			#remove old archived files
+			archived_files.select { |m| m =~ archive_regex }.each do |file|
+				remote_file_to_remove = remove_initial_slash(File.join(@remote_path, file))
+				@log.debug "Removing old archive file #{remote_file_to_remove}"
+				@connection.delete_object @bucket, remote_file_to_remove
+			end
+		end
+
+		return run_result(true, "Successully pushed #{count} file(s) to Amazon s3 bucket (with version history)")
+
+	end
+
+end	

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloudblocks
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.15
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-29 00:00:00.000000000 Z
+date: 2012-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -149,13 +149,14 @@ files:
 - lib/config-chief.rb
 - lib/cloud-quartz.rb
 - lib/plugins/broken.rb
-- lib/plugins/logrotate.rb
-- lib/plugins/mysql.rb
+- lib/plugins/file_rotate.rb
+- lib/plugins/log_rotate.rb
+- lib/plugins/mysql_backup.rb
 - lib/plugins/quartz_plugin.rb
+- lib/plugins/rackspace_backup.rb
 - lib/plugins/rake.rb
-- lib/plugins/redis.rb
-- lib/plugins/rotater.rb
-- lib/plugins/s3backup.rb
+- lib/plugins/redis_backup.rb
+- lib/plugins/s3_backup.rb
 - lib/plugins/shell.rb
 - lib/plugins/tester.rb
 - lib/plugins/webget.rb

data/lib/plugins/s3backup.rb

@@ -1,110 +0,0 @@
-require File.join(File.dirname(__FILE__), 'quartz_plugin')
-require 'fileutils'
-require 'fog'
-
-class S3backup < QuartzPlugin
-
-	@@version_major = 0
-	@@version_minor = 0
-	@@version_revision = 1
-
-	def info
-		{ :uid => "d3533989f9d542f393566511e8eb2090", :name => "S3 Backup", :version => get_version }
-	end
-
-	def run(message)
-		pl = payload(message)
-
-		@log.debug "Pruned payload #{pl}"
-
-		@access_key_id 			= pl['access_key']
-		@secret_access_key 		= pl['secret_key']
-		@bucket 				= pl['bucket']
-		@remote_path 			= pl['remote_path']
-		@region 				= pl['region']
-		@local_pattern 			= pl['local_pattern']
-		@keep 					= pl['keep'].empty? ? 0 : pl['keep'].to_i
-
-		@testing				= pl['testing']
-
-		return transfer
-	end
-
-	private
-
-	def connection
-		Fog.mock! unless @testing.nil? || @testing == false
-		@connection ||= Fog::Storage.new(
-			:provider               => 'AWS',
-			:aws_access_key_id      => @access_key_id,
-			:aws_secret_access_key  => @secret_access_key,
-			:region                 => @region
-		)
-	end
-
-	def remote_path_for(filename)
-		filename.sub(/^\//, '')
-	end
-
-	def transfer
-		begin
-			remote_path = remote_path_for(@remote_path)
-			@log.debug "Remote path is #{remote_path}"
-
-			@log.debug "Syncronizing local and remote clocks"
-			connection.sync_clock
-
-			count = 0
-			# get local files
-			directory = connection.directories.get(@bucket)
-			all_rotated = directory.files.reject { |m| File.dirname(m.key) != @remote_path }
-
-			Dir.glob(@local_pattern).each do |f|
-				base_file = File.basename(f)
-				remote_files = all_rotated.map {|m| File.basename(m.key)}
-				unless remote_files.include? base_file
-					remote_file = File.join(@remote_path, base_file)
-					next if File.directory?(f)
-					@log.debug "Copying #{f} to #{remote_file}"
-					count += 1
-					File.open(f, 'r') do |file|
-						connection.put_object(@bucket, File.join(remote_path, base_file), file)
-					end
-				end
-			end
-
-			return run_result(true, "Files copied to S3 bucket successfully with no rotation") if @keep == 0
-
-			@log.debug "Found #{all_rotated.count} in the remote bucket"
-			if all_rotated.count > @keep
-				remove_count = all_rotated.count - @keep
-				@log.debug "Removing #{remove_count} and keeping the most recent #{@keep}"
-				to_remove = all_rotated.sort { |a,b| a.last_modified <=> b.last_modified }.map{|m| m.key }[0...remove_count]
-				@log.debug "Removing extra files"
-				to_remove.each do |tr|
-					@log.debug "Removing #{tr}"
-					connection.delete_object(@bucket, tr)
-				end
-			end
-		rescue Excon::Errors::SocketError => exc
-			@log.error exc.message
-			return run_result(false, exc.message)
-		rescue Excon::Errors::Error => exc
-			@log.error exc.message
-			result = exc.response.body
-			message = result.match(/\<Message\>(.*)\<\/Message\>/)
-			if !message.nil?
-				message = message[1] 
-				return run_result(false, message)
-			elsif exc.response.status == 404
-				return run_result(false, "Remote S3 serivce or bucket not found (404)")
-			elsif exc.response.status != 0
-				return run_result(false, "Remote S3 serivce returned error #{exc.response.status} without any more details")
-			else
-				return run_result(false, exc.message)
-			end
-		end
-
-		run_result(true, "Successfully copied #{count} files to S3")
-	end
-end