cloudblocks 0.0.7 → 0.0.8

data/lib/plugins/logrotate.rb

@@ -0,0 +1,61 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+
+class Logrotate < QuartzPlugin
+
+	def info
+		{ :uid => "8f4286bfd946c8b08b234833673b8860", :name => "Log Rotate", :version => "0.0.0" }
+	end
+
+	def run(message)
+		pl = payload(message)
+
+		@source_pattern 	= pl['source pattern']
+		@dest_folder 		= pl['destination']
+		@keep 				= pl['keep'].empty? ? 0 : pl['keep'].to_i
+		@post_run_step 		= pl['post rotate']
+
+        ext = Time.now.utc.strftime('%Y%m%d%H%M%S')
+
+        FileUtils.mkdir_p(@dest_folder)
+
+		Dir.glob(@source_pattern).each do |f|
+			dest_file = File.join(@dest_folder, File.basename(f))
+			next if File.directory?(f)
+			fname = "#{dest_file}.gz"
+			dump_cmd = "cat #{f} | gzip > '#{fname}.#{ext}'"
+	        @log.debug "Running #{dump_cmd}"
+	        copy_result = run_shell(dump_cmd)
+
+	        return run_result(false, copy_result[:message]) unless copy_result[:ok]
+
+	        @log.debug "Removing source log files"
+	        remove_result = run_shell("rm #{f}")
+
+	        return run_result(false, remove_result[:message]) unless remove_result[:ok]
+
+	    	# find all the files from this one's rotations
+	    	rotated_pattern = "#{fname}.*"
+	    	@log.debug "Looking for rotated files #{rotated_pattern}"
+	    	all_rotated = Dir.glob(rotated_pattern)
+	    	if all_rotated.count > @keep
+	    		remove_count = all_rotated.count - @keep
+	        	to_remote = all_rotated.sort! { |a,b| File.mtime(a) <=> File.mtime(b) }[0...remove_count]
+				@log.debug "Removing extra #{remove_count} files"
+				to_remote.each do |tr|
+					@log.debug "Removing #{tr}"
+					remove_shell = run_shell("rm #{tr}")
+					return run_result(false, remove_shell[:message]) unless remove_shell[:ok]
+				end
+	        end
+        end
+
+        if !@post_run_step.nil? && !@post_run_step.empty?
+        	@log.debug "Running post rotate step #{post_run_step}"
+        	post_shell = run_shell(@post_run_step)
+        	return run_result(false, post_shell[:message]) unless post_shell[:ok]
+        end
+
+        return run_result(true, "Log files rotated succesfully")
+   	end
+end	

data/lib/plugins/rotater.rb

@@ -31,6 +31,8 @@ class Rotater < QuartzPlugin
         	move_shell = run_shell("mv #{f} #{dest_path}")
         	return run_result(false, move_shell[:message]) unless move_shell[:ok]
 
+                return run_result(true, "Files moved successfully with no rotation") if keep == 0
+
         	# find all the files from this one's rotations
         	rotated_pattern = "#{File.join(archive, fname)}.*"
         	@log.debug "Looking for rotated files #{rotated_pattern}"
@@ -38,16 +40,16 @@ class Rotater < QuartzPlugin
         	if all_rotated.count > keep
         		remove_count = all_rotated.count - keep
 	        	to_remote = all_rotated.sort! { |a,b| File.mtime(a) <=> File.mtime(b) }[0...remove_count]
-				@log.debug "Removing extra files"
-				to_remote.each do |tr|
-					@log.debug "Removing #{tr}"
-					remove_shell = run_shell("rm #{tr}")
-					return run_result(false, remove_shell[:message]) unless remove_shell[:ok]
-				end
+			@log.debug "Removing extra files"
+			to_remote.each do |tr|
+				@log.debug "Removing #{tr}"
+				remove_shell = run_shell("rm #{tr}")
+				return run_result(false, remove_shell[:message]) unless remove_shell[:ok]
+			end
 	        end
         end
 
-        if post_rotate && rotated
+        if !post_rotate.empty? && rotated
         	@log.debug "Running post rotate step #{post_rotate}"
         	post_shell = run_shell(post_rotate)
         	return run_result(false, post_shell[:message]) unless post_shell[:ok]

data/lib/plugins/s3backup.rb

@@ -0,0 +1,101 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'fileutils'
+require 'fog'
+
+class S3backup < QuartzPlugin
+
+	def info
+		{ :uid => "d3533989f9d542f393566511e8eb2090", :name => "S3 Backup", :version => "0.0.0" }
+	end
+
+	def run(message)
+		pl = payload(message)
+
+		@log.debug "Pruned payload #{pl}"
+
+		@access_key_id 			= pl['access key']
+		@secret_access_key 		= pl['secret key']
+		@bucket 				= pl['bucket']
+		@remote_path 			= pl['remote path']
+		@region 				= pl['region']
+		@local_pattern 			= pl['local pattern']
+		@keep 					= pl['keep'].empty? ? 0 : pl['keep'].to_i
+
+		@testing				= pl['testing']
+
+		return transfer
+	end
+
+	private
+
+	def connection
+		Fog.mock! unless @testing.nil? || @testing == false
+		@connection ||= Fog::Storage.new(
+			:provider               => 'AWS',
+			:aws_access_key_id      => @access_key_id,
+			:aws_secret_access_key  => @secret_access_key,
+			:region                 => @region
+		)
+	end
+
+	def remote_path_for(filename)
+		filename.sub(/^\//, '')
+	end
+
+	def transfer
+		begin
+			remote_path = remote_path_for(@remote_path)
+			@log.debug "Remote path is #{remote_path}"
+
+			@log.debug "Syncronizing local and remote clocks"
+			connection.sync_clock
+
+			count = 0
+			# get local files
+			Dir.glob(@local_pattern).each do |f|
+				remote_file = File.join(@remote_path, File.basename(f))
+				next if File.directory?(f)
+				@log.debug "Copying #{f} to #{remote_file}"
+				count += 1
+				File.open(f, 'r') do |file|
+					connection.put_object(@bucket, File.join(remote_path, File.basename(f)), file)
+				end
+			end
+
+			return run_result(true, "Files copied to S3 bucket successfully with no rotation") if @keep == 0
+
+			directory = connection.directories.get(@bucket)
+			all_rotated = directory.files.reject { |m| File.dirname(m.key) != @remote_path }
+			@log.debug "Found #{all_rotated.count} in the remote bucket"
+			if all_rotated.count > @keep
+				remove_count = all_rotated.count - @keep
+				@log.debug "Removing #{remove_count} and keeping the most recent #{@keep}"
+				to_remove = all_rotated.sort { |a,b| a.last_modified <=> b.last_modified }.map{|m| m.key }[0...remove_count]
+				@log.debug "Removing extra files"
+				to_remove.each do |tr|
+					@log.debug "Removing #{tr}"
+					connection.delete_object(@bucket, tr)
+				end
+			end
+		rescue Excon::Errors::SocketError => exc
+			@log.error exc.message
+			return run_result(false, exc.message)
+		rescue Excon::Errors::Error => exc
+			@log.error exc.message
+			result = exc.response.body
+			message = result.match(/\<Message\>(.*)\<\/Message\>/)
+			if !message.nil?
+				message = message[1] 
+				return run_result(false, message)
+			elsif exc.response.status == 404
+				return run_result(false, "Remote S3 serivce or bucket not found (404)")
+			elsif exc.response.status != 0
+				return run_result(false, "Remote S3 serivce returned error #{exc.response.status} without any more details")
+			else
+				return run_result(false, exc.message)
+			end
+		end
+
+		run_result(true, "Successfully copied #{count} files to S3")
+	end
+end	

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloudblocks
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-17 00:00:00.000000000Z
+date: 2012-03-20 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
-  requirement: &70232128687800 !ruby/object:Gem::Requirement
+  requirement: &70274217198280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: 0.8.1
   type: :runtime
   prerelease: false
-  version_requirements: *70232128687800
+  version_requirements: *70274217198280
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70232128687140 !ruby/object:Gem::Requirement
+  requirement: &70274217197400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.3
   type: :runtime
   prerelease: false
-  version_requirements: *70232128687140
+  version_requirements: *70274217197400
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &70232128667840 !ruby/object:Gem::Requirement
+  requirement: &70274217196640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 0.12.10
   type: :runtime
   prerelease: false
-  version_requirements: *70232128667840
+  version_requirements: *70274217196640
 - !ruby/object:Gem::Dependency
   name: faye
-  requirement: &70232128667180 !ruby/object:Gem::Requirement
+  requirement: &70274217195940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *70232128667180
+  version_requirements: *70274217195940
 - !ruby/object:Gem::Dependency
   name: open4
-  requirement: &70232128666420 !ruby/object:Gem::Requirement
+  requirement: &70274217195080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,18 @@ dependencies:
         version: 1.3.0
   type: :runtime
   prerelease: false
-  version_requirements: *70232128666420
+  version_requirements: *70274217195080
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: &70274217194380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70274217194380
 description: See http://www.thecloudblocks.com for more info
 email: khash@thecloudblocks.com
 executables:
@@ -77,10 +88,12 @@ files:
 - lib/config-chief.rb
 - lib/cloud-quartz.rb
 - lib/plugins/broken.rb
+- lib/plugins/logrotate.rb
 - lib/plugins/mysql.rb
 - lib/plugins/quartz_plugin.rb
 - lib/plugins/rake.rb
 - lib/plugins/rotater.rb
+- lib/plugins/s3backup.rb
 - lib/plugins/shell.rb
 - lib/plugins/tester.rb
 - lib/plugins/webget.rb