yore 0.0.2 → 0.0.3

data/History.txt

@@ -2,3 +2,15 @@
 
 * 1 major enhancement:
   * Initial release
+  
+== 0.0.2 2009-2-04
+
+* now using popen4
+* raises exceptions when commandline tools fail
+* now using shoulda for tests
+
+== 0.0.3 2009-02-20
+
+* much better logging, reporting and console output
+
+

data/Manifest.txt

@@ -3,8 +3,11 @@ History.txt
 lib/ihl_ruby/enum.rb
 lib/ihl_ruby/extend_base_classes.rb
 lib/ihl_ruby/misc_utils.rb
+lib/ihl_ruby/config.rb
+lib/ihl_ruby/logging.rb
 lib/ihl_ruby/string_utils.rb
 lib/ihl_ruby/xml_utils.rb
+lib/ihl_ruby/shell_extras.rb
 lib/yore.orig.rb
 lib/yore/yore_core.rb
 Manifest.txt

data/bin/yore

@@ -24,9 +24,10 @@ def command(aParser,aController,aAction,aShortDescription=nil,aOptionParser=nil,
 	c.options = aOptionParser if aOptionParser
   c.set_execution_block do |args|
     job = args.first
+		aController.logger.info "Job file: #{File.expand_path(job)}"
     xmlRoot = XmlUtils.get_file_root(job)
-    aController.configure(xmlRoot,CMD_OPTIONS,{:basepath => File.dirname(File.expand_path(job))})
-    aController.send(aAction,args)
+		aController.configure(xmlRoot,CMD_OPTIONS,{:basepath => File.dirname(File.expand_path(job))})
+		aController.do_action(aAction,args)
   end
   aParser.add_command(c)
 end
@@ -63,3 +64,6 @@ command(cmd,yore,:db_dump,"Dump database by name in job\n")
 
 cmd.parse
 
+yore.logger.info "\nComplete.\n"
+yore.report
+

data/lib/ihl_ruby/config.rb

@@ -0,0 +1,107 @@
+class ConfigClass < Hash
+
+	attr_reader :default_values
+
+	def initialize(aDefaultValues,aNewValues=nil,&aBlock)
+    self.merge!(@default_values = aDefaultValues.clone)
+		if aNewValues
+			block_given? ? read(aNewValues,&aBlock) : read(aNewValues) 
+		end
+	end
+
+	# aBlock allows values to be filtered based on key,default and new values
+	def read(aSource,&aBlock)
+		default_values.each do |k,v|
+			done = false
+			if block_given? && ((newv = yield(k,v,aSource && aSource[k])) != nil)
+				self[k] = newv
+				done = true
+			end
+			copy_item(aSource,k) if !done && aSource && aSource[k]
+		end
+		self
+	end
+	
+	# reset values back to defaults
+	def reset
+		self.clear
+		self.merge!(default_values)
+	end
+
+	def set_int(aKey,aValue)
+		case aValue
+			when String then    self[aKey] = aValue.to_integer(self[aKey]);
+			when Fixnum then    self[aKey] = aValue;
+			when Float then     self[aKey] = aValue.to_i;
+		end
+	end
+
+	def set_float(aKey,aValue)
+		case aValue
+			when String then    self[aKey] = aValue.to_float(self[aKey]);
+			when Fixnum then    self[aKey] = aValue.to_f;
+			when Float then     self[aKey] = aValue;
+		end
+	end
+
+	def set_boolean(aKey,aValue)
+		case aValue
+			when TrueClass,FalseClass then   self[aKey] = aValue;
+			when String then    self[aKey] = (['1','yes','y','true','on'].include?(aValue.downcase))
+		else
+			set_boolean(aKey,aValue.to_s)
+		end
+	end
+	
+	def set_symbol(aKey,aValue)
+		case aValue
+			when String then    self[aKey] = (aValue.to_sym rescue nil);
+			when Symbol then    self[aKey] = aValue;
+		end
+	end
+
+	def copy_item(aHash,aKey)
+		case default_values[aKey]
+			when NilClass then ;
+			when String then self[aKey] = aHash[aKey].to_s unless aHash[aKey].nil?
+			when Float then set_float(aKey,aHash[aKey]);
+			when Fixnum then set_int(aKey,aHash[aKey]);
+			when TrueClass, FalseClass then set_boolean(aKey,aHash[aKey]);
+			when Symbol then self[aKey] = (aHash[aKey].to_sym rescue nil)
+			else
+				raise Error.new('unsupported type')
+		end
+	end
+
+	def copy_strings(aHash,*aKeys)
+		aKeys.each do |k|
+			self[k] = aHash[k].to_s unless aHash[k].nil?
+		end
+	end
+
+	def copy_ints(*aDb)
+		aHash = aDb.shift
+		aKeys = aDb
+		aKeys.each do |k|
+			set_int(k,aHash[k])
+		end
+	end
+
+	def copy_floats(aHash,*aKeys)
+		aKeys.each do |k|
+			set_float(k,aHash[k])
+		end
+	end
+
+	def copy_booleans(aHash,*aKeys)
+		aKeys.each do |k|
+			set_boolean(k,aHash[k])
+		end
+	end
+
+	def to_hash
+		{}.merge(self)
+	end
+
+end
+

data/lib/ihl_ruby/logging.rb

@@ -0,0 +1,159 @@
+require 'logger'
+require 'ihl_ruby/misc_utils'
+
+class Logger
+  attr_reader :logdev
+end
+
+module LogUtils
+
+	# eg.
+	# {
+	# 	'destination' => 'STDERR|STDOUT|FILE',
+	# 	'filename' => '/path/to/file.ext',
+	# 	'level' => 'DEBUG|INFO|...',
+	#
+	# 	'age' = 'daily|weekly|monthly',
+	# 	OR
+	# 	'max_files' => 3,
+	# 	'max_bytes' => 1024000
+	# }
+	def self.create_logger_from_config(aConfigHash)
+		if not aConfigHash
+			result = Logger.new(STDERR)
+			result.level = Logger::INFO
+			return result
+		end
+	
+		result = nil
+		case aConfigHash['destination']
+			when 'STDERR' then
+				result = Logger.new(STDERR)
+			when 'STDOUT' then
+				result = Logger.new(STDOUT)
+			when 'FILE' then
+        result = aConfigHash['age'] ?
+          Logger.new(aConfigHash['filename'],aConfigHash['age']) :
+          Logger.new(
+            aConfigHash['filename'],
+						(aConfigHash['max_files'] || 3).to_i,
+						(aConfigHash['max_bytes'] || 1024000).to_i
+					)
+			else
+				result = Logger.new(STDERR)
+		end
+    puts valstr = "Logger::#{(aConfigHash['level'] || 'INFO').upcase}"
+		result.level = eval(valstr)
+		return result
+	end
+
+	# use this to trunc a log file to 0 bytes
+	def self.trunc(aFilename)
+		f = File.open(aFilename, "w")
+		f.close
+	end
+
+  class ReportFormatter < Logger::Formatter
+    def call(severity, time, progname, msg)
+     "|%s %1s %s\n" % [(time.strftime('%H%M%S.')<<"%03d" % (time.usec/1000)),severity[0..0],msg2str(msg)]
+    end
+  end
+
+  class Reporter < Logger
+    def initialize(logdev)
+      super(logdev)
+    end
+
+  end
+
+  def self.create_reporter(aFilename=nil)
+    aFilename ||= MiscUtils::temp_file()
+    result = Logger.new(aFilename)
+    result.formatter = ReportFormatter.new
+    result
+  end
+end
+
+
+
+class MultiLogger < Logger
+
+  attr_reader :loggers
+
+  def initialize(aLoggers)
+    @loggers = aLoggers.is_a?(Array) ? aLoggers : [aLoggers]
+  end
+
+  def add(severity, message = nil, progname = nil, &block)
+		return true if !@loggers
+		severity ||= UNKNOWN
+    @loggers.each do |lr|
+      block_given? ? lr.add(severity,message,progname,&block) : lr.add(severity,message,progname)
+    end
+    true
+  end
+  alias log add
+
+  def <<(msg)
+    @loggers.each do |lr|
+      lr << msg
+    end
+  end
+
+  def close
+    @loggers.each do |lr|
+      lr.close
+    end
+  end
+
+end
+
+#DEBUG D
+#INFO
+#WARN ?
+#ERROR !
+#FATAL F
+#UNKNOWN U
+
+# Logger that mostly works like a STDOUT logger, except that warnings and above get sent to STDERR instead
+class ConsoleLogger < Logger
+
+  class ReportFormatter < Logger::Formatter
+    def call(severity, time, progname, msg)
+     msg2str(msg)+"\n"
+    end
+  end
+
+  def initialize(aErrLevel = Severity::WARN)
+		super(STDOUT)
+    self.formatter = ReportFormatter.new
+		self.level = Severity::INFO
+		self << "\n"
+		@err_logger = Logger.new(STDERR)
+		@err_level = aErrLevel
+		@err_logger.formatter = ReportFormatter.new
+  end
+
+	alias_method :orig_add, :add
+  def add(severity, message = nil, progname = nil, &block)	
+		if severity >= @err_level
+			block_given? ? @err_logger.add(severity,message,progname,&block) : @err_logger.add(severity,message,progname)
+		else
+			block_given? ? orig_add(severity,message,progname,&block) : orig_add(severity,message,progname)
+		end
+  end
+  alias log add
+
+  #
+  # Close the logging device.
+  #
+  def close
+		begin
+			@logdev.close if @logdev
+		ensure
+			@err_logger.close
+		end
+  end
+	
+end
+

data/lib/ihl_ruby/misc_utils.rb

@@ -2,6 +2,8 @@ require 'tmpdir'
 require 'logger'
 require 'pathname'
 
+require 'ihl_ruby/logging'
+
 module MiscUtils
 
 	def self.logger
@@ -346,81 +348,6 @@ END_OF_MESSAGE
 
 end
 
-class Logger
-  attr_reader :logdev
-end
-
-module LogUtils
-
-	# eg.
-	# {
-	# 	'destination' => 'STDERR|STDOUT|FILE',
-	# 	'filename' => '/path/to/file.ext',
-	# 	'level' => 'DEBUG|INFO|...',
-	#
-	# 	'age' = 'daily|weekly|monthly',
-	# 	OR
-	# 	'max_files' => 3,
-	# 	'max_bytes' => 1024000
-	# }
-	def self.create_logger_from_config(aConfigHash)
-		if not aConfigHash
-			result = Logger.new(STDERR)
-			result.level = Logger::DEBUG
-			return result
-		end
-	
-		result = nil
-		case aConfigHash['destination']
-			when 'STDERR' then
-				result = Logger.new(STDERR)
-			when 'STDOUT' then
-				result = Logger.new(STDOUT)
-			when 'FILE' then
-        result = aConfigHash['age'] ?
-          Logger.new(aConfigHash['filename'],aConfigHash['age']) :
-          Logger.new(
-            aConfigHash['filename'],
-						(aConfigHash['max_files'] || 3).to_i,
-						(aConfigHash['max_bytes'] || 1024000).to_i
-					)
-			else
-				result = Logger.new(STDERR)
-		end
-    puts valstr = "Logger::#{(aConfigHash['level'] || 'INFO').upcase}"
-		result.level = eval(valstr)
-		return result
-	end
-
-	# use this to trunc a log file to 0 bytes
-	def self.trunc(aFilename)
-		f = File.open(aFilename, "w")
-		f.close
-	end
-
-  class ReportFormatter < Logger::Formatter
-    def call(severity, time, progname, msg)
-     "|%s %1s %s\n" % [(time.strftime('%H%M%S.')<<"%03d" % (time.usec/1000)),severity[0..0],msg2str(msg)]
-    end
-  end
-
-  class Reporter < Logger
-    def initialize(logdev)
-      super(logdev)
-    end
-
-  end
-
-  def self.create_reporter(aFilename=nil)
-    aFilename ||= MiscUtils::temp_file()
-    result = Logger.new(aFilename)
-    result.formatter = ReportFormatter.new
-    result
-  end
-end
-
-
-
 # include this at the top of a class to protect it from baddies.
 # eg. 
 # + nearly all ancestor public_instance_methods will be hidden

data/lib/ihl_ruby/shell_extras.rb

@@ -0,0 +1,84 @@
+require 'rubygems'
+gem 'RequirePaths'; require 'require_paths'
+require_paths '.','..'
+
+gem 'Platform'; require 'platform'
+gem 'shairontoledo-popen4'; require 'popen4'
+
+module POpen4
+
+	class ExecuteError < StandardError
+
+		attr_reader :result #,:stderr,:stdout,:exitcode,:pid
+
+    def initialize(aArg)
+      if aArg.is_a? Hash
+				msg = ([aArg[:stderr],aArg[:stdout],"Error #{aArg[:exitcode].to_s}"].find {|i| i && !i.empty?})
+        super(msg)
+        @result = aArg
+      else
+        super(aArg)
+      end
+    end
+		
+		def inspect
+			"#{self.class.to_s}: #{@result.inspect}"
+		end
+
+	end
+	
+	def self.pump_thread(aIn,aOut)
+		Thread.new do
+			loop { aOut.puts aIn.gets }
+		end
+	end
+
+	# Usage :
+	# result = POpen4::shell('somebinary') do |r|   # block gives opportunity to adjust result, and avoid exception raised from non-zero exit codes
+	# 	if r[:exitcode]==254		# eg. say this binary returns 254 to mean something special but not an error
+	#     r[:stdout] = 'some correct output'
+	#     r[:stderr] = ''
+	#     r[:exitcode] = 0
+	#   end
+	# end
+  #
+  # OR
+  #
+  # result = POpen4::shell('somebinary');
+  # puts result[:stdout]
+	#
+	# Giving aStdOut,aStdErr causes the command output to be connected to the given stream, and that stream to not be given in the result hash
+	def self.shell(aCommand,aWorkingDir=nil,aTimeout=nil,aStdOut=nil,aStdErr=nil)
+		raise ExecuteError.new('aWorkingDir doesnt exist') unless !aWorkingDir || File.exists?(aWorkingDir)
+		orig_wd = Dir.getwd
+		result = {:command => aCommand, :dir => (aWorkingDir || orig_wd)}
+    status = nil
+		begin
+			Dir.chdir(aWorkingDir) if aWorkingDir
+			Timeout.timeout(aTimeout,ExecuteError) do	# nil aTimeout will not time out
+				status = POpen4::popen4(aCommand) do |stdout, stderr, stdin, pid|
+					thrOut = aStdOut ? Thread.new { aStdOut.puts stdout.read } : nil
+					thrErr = aStdErr ? Thread.new { aStdErr.puts stderr.read } : nil
+					thrOut.join if thrOut
+					thrErr.join if thrErr
+
+					result[:stdout] = stdout.read unless aStdOut
+					result[:stderr] = stderr.read unless aStdErr
+					result[:pid] = pid
+				end
+			end
+		ensure
+			Dir.chdir(orig_wd)
+		end
+		result[:exitcode] = (status && status.exitstatus) || 1
+		yield(result) if block_given?
+		raise ExecuteError.new(result) if result[:exitcode] != 0
+		return result
+	end
+	
+	def self.shell_out(aCommand,aWorkingDir=nil,aTimeout=nil,&block)
+		block_given? ? POpen4::shell(aCommand,aWorkingDir,aTimeout,STDOUT,STDERR,&block) : POpen4::shell(aCommand,aWorkingDir,aTimeout,STDOUT,STDERR)
+	end
+
+end
+

data/lib/yore.orig.rb

@@ -2,5 +2,5 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module Yore
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

data/lib/yore/yore_core.rb

@@ -2,78 +2,21 @@ require 'rubygems'
 gem 'RequirePaths'; require 'require_paths'
 require_paths '.','..'
 
-gem 'Platform'; require 'platform'
-gem 'shairontoledo-popen4'; require 'popen4'
 
 require 'fileutils'
 require 'net/smtp'
 
 require 'ihl_ruby/misc_utils'
+require 'ihl_ruby/logging'
 require 'ihl_ruby/string_utils'
 require 'ihl_ruby/xml_utils'
 require 'ihl_ruby/extend_base_classes'
+require 'ihl_ruby/shell_extras'
+require 'ihl_ruby/config'
 
 THIS_FILE = __FILE__
 THIS_DIR = File.dirname(THIS_FILE)
 
-module POpen4
-
-	class ExecuteError < StandardError
-
-		attr_reader :result #,:stderr,:stdout,:exitcode,:pid
-
-    def initialize(aArg)
-      if aArg.is_a? Hash
-        super(aArg[:stdout] || '')
-        @result = aArg
-#        @stderr = aArg[:stderr]
-#        @stdout = aArg[:stdout]
-#        @exitcode = aArg[:exitcode]
-#        @pid = aArg[:pid]
-      else
-        super(aArg)
-      end
-    end
-
-	end
-
-	# Usage :
-	# result = POpen4::shell('somebinary') do |r|   # block gives opportunity to adjust result, and avoid exception raised from non-zero exit codes
-	# 	if r[:exitcode]==254		# eg. say this binary returns 254 to mean something special but not an error
-	#     r[:stdout] = 'some correct output'
-	#     r[:stderr] = ''
-	#     r[:exitcode] = 0
-	#   end
-	# end
-  #
-  # OR
-  #
-  # result = POpen4::shell('somebinary');
-  # puts result[:stdout]
-	def self.shell(aCommand,aWorkingDir=nil,aTimeout=nil)
-		raise ExecuteError.new('aWorkingDir doesnt exist') unless !aWorkingDir || File.exists?(aWorkingDir)
-		orig_wd = Dir.getwd
-		result = {:command => aCommand, :dir => (aWorkingDir || orig_wd)}
-    status = nil
-		begin
-			Dir.chdir(aWorkingDir) if aWorkingDir
-			Timeout.timeout(aTimeout,ExecuteError) do	# nil aTimeout will not time out
-				status = POpen4::popen4(aCommand) do |stdout, stderr, stdin, pid|
-					result[:stdout] = stdout.read
-					result[:stderr] = stderr.read
-					result[:pid] = pid
-				end
-			end
-		ensure
-			Dir.chdir(orig_wd)
-		end
-		result[:exitcode] = (status && status.exitstatus) || 1
-		yield(result) if block_given?
-		raise ExecuteError.new(result) if result[:exitcode] != 0
-		return result
-	end
-
-end
 
 
 module YoreCore
@@ -137,88 +80,6 @@ module YoreCore
   end
 
 
-  class ConfigClass < Hash
-
-    def initialize(aDefaultHash=nil)
-      self.merge!(aDefaultHash) if aDefaultHash
-    end
-
-    def set_int(aKey,aValue)
-      case aValue
-        when String then    self[aKey] = aValue.to_integer(self[aKey]);
-        when Fixnum then    self[aKey] = aValue;
-        when Float then     self[aKey] = aValue.to_i;
-      end
-    end
-
-    def set_float(aKey,aValue)
-      case aValue
-        when String then    self[aKey] = aValue.to_float(self[aKey]);
-        when Fixnum then    self[aKey] = aValue.to_f;
-        when Float then     self[aKey] = aValue;
-      end
-    end
-
-    def set_boolean(aKey,aValue)
-      case aValue
-        when TrueClass,FalseClass then   self[aKey] = aValue;
-        when String then    self[aKey] = (['1','yes','y','true','on'].include?(aValue.downcase))
-      else
-        default        set_boolean(aKey,aValue.to_s)
-      end
-    end
-		
-    def set_symbol(aKey,aValue)
-      case aValue
-        when String then    self[aKey] = (aValue.to_sym rescue nil);
-        when Symbol then    self[aKey] = aValue;
-      end
-    end
-
-		def copy_item(aHash,aKey)
-			case self[aKey]
-				when NilClass then ;
-				when String then self[aKey] = aHash[aKey].to_s unless aHash[aKey].nil?
-				when Float then set_float(aKey,aHash[aKey]);
-				when Fixnum then set_int(aKey,aHash[aKey]);
-				when TrueClass, FalseClass then set_boolean(aKey,aHash[aKey]);
-				when Symbol then self[aKey] = (aHash[aKey].to_sym rescue nil)
-				else
-					raise Error.new('unsupported type')
-			end
-		end
-
-    def copy_strings(aHash,*aKeys)
-      aKeys.each do |k|
-        self[k] = aHash[k].to_s unless aHash[k].nil?
-      end
-    end
-
-    def copy_ints(*aDb)
-      aHash = aDb.shift
-      aKeys = aDb
-      aKeys.each do |k|
-        set_int(k,aHash[k])
-      end
-    end
-
-    def copy_floats(aHash,*aKeys)
-      aKeys.each do |k|
-        set_float(k,aHash[k])
-      end
-    end
-
-    def copy_booleans(aHash,*aKeys)
-      aKeys.each do |k|
-        set_boolean(k,aHash[k])
-      end
-    end
-
-    def to_hash
-      {}.merge(self)
-    end
-
-  end
 
   class Yore
 
@@ -230,7 +91,7 @@ module YoreCore
       :crypto_key => "07692FC8656F04AE5518B80D38681E038A3C12050DF6CC97CEEC33D800D5E2FE",   # apparently a string of up to 64 random hex digits
       :first_hour => 4,
       :prefix => 'backup',
-      :log_level => 'DEBUG',
+      :log_level => 'INFO',
       :bucket => '',
 			:email_report => false,
       :mail_host => '',
@@ -254,17 +115,26 @@ module YoreCore
 
     def initialize(aConfig=nil)
       DEFAULT_CONFIG[:email_report] = false  # fixes some bug where this was nil
-      configure(aConfig || DEFAULT_CONFIG)
-    end
+			@config = ConfigClass.new(DEFAULT_CONFIG,aConfig)
 
-    def self.clean_config(aConfig)
-      result = ConfigClass.new(DEFAULT_CONFIG)
-			DEFAULT_CONFIG.each do |k,v|
-        result.copy_item(aConfig,k) if aConfig[k]
-      end
-      #result.copy_ints(aConfig,:keep_daily,:keep_weekly,:keep_monthly)
-      #result.copy_strings(aConfig,:crypto_iv,:crypto_key,:prefix,:bucket,:log_level,:basepath)
-      result.to_hash
+			cons = ConsoleLogger.new()
+			cons.level = Logger::Severity.const_get(config[:log_level]) rescue Logger::Severity::INFO
+
+      report_file = MiscUtils::temp_file
+			@reporter = Logger.new(report_file)
+			@reporter.formatter = ConsoleLogger::ReportFormatter.new
+			@reporter.level = cons.level
+
+			@logger = MultiLogger.new([cons,@reporter])
+			#require 'ruby-debug'; debugger
+			@logger.info "Yore file and database backup tool for Amazon S3 "
+			@logger.info "(c) 2009 Buzzware Solutions (www.buzzware.com.au)"
+			@logger.info "-------------------------------------------------"
+			@logger.info ""
+
+      @logger.info "report file: #{report_file}"
+
+      configure(@config)
     end
 
     # read the config however its given and return a hash with values in their correct type, and either valid or nil
@@ -272,45 +142,47 @@ module YoreCore
     def configure(aConfig,aCmdOptions = nil,aOptions = nil)
       config_h = nil
       case aConfig
-        when Hash then config_h = aConfig
+        when Hash,::ConfigClass then config_h = aConfig
         when REXML::Element then config_h = XmlUtils.read_simple_items(aConfig,'/Yore/SimpleItems')
         else
-          raise Error.new('unsupported type')
+          raise StandardError.new('unsupported type')
       end
       config_i = {}
       config_h.each{|n,v| config_i[n.to_sym] = v} if config_h
       aCmdOptions.each{|k,v| config_i[k.to_sym] = v} if aCmdOptions
       config_i.merge!(aOptions) if aOptions
-      @config = Yore.clean_config(config_i)
+			config.read(config_i)
+
       @keepers = Array.new
       @keepers << KeepDaily.new(config[:keep_daily])
       @keepers << KeepWeekly.new(config[:keep_weekly])
       @keepers << KeepMonthly.new(config[:keep_monthly])
-
-      @log_file = MiscUtils::temp_file()
-
-      @logger = LogUtils::create_logger_from_config({
-          'destination' => 'STDOUT',
-          'level' => config[:log_level]
-#        'destination' => 'FILE',
-#        'filename' => @log_file,
-#        'level' => config[:log_level]
-      })
-      report_file = MiscUtils::temp_file
-      logger.info "report file: #{report_file}"
-      @reporter = LogUtils::create_reporter(report_file)
+			
       @basepath = config_h[:basepath]
-      #sources = aConfig
-      #
-      #@filelist =
     end
+		
+		def do_action(aAction,aArgs)
+			logger.info "Executing command: #{aAction} ...\n"
+			begin
+				send(aAction,aArgs)
+			rescue Exception => e
+				logger.warn "#{e.class.to_s}: during #{aAction}(#{aArgs.inspect}): #{e.message}"
+			end			
+		end
 
 		def shell(aCommandline,&aBlock)
-			reporter.debug "To shell: " + aCommandline
+			#require 'ruby-debug'; debugger
+			logger.debug "To shell: " + aCommandline
       result = block_given? ? POpen4::shell(aCommandline,nil,nil,&aBlock) : POpen4::shell(aCommandline)
-      reporter.debug "From shell: '#{result[:stdout]}'"
+      logger.debug "From shell: '#{result.inspect}'"
       return result[:stdout]
 		end
+		
+		def s3shell(aCommandline)
+			shell(aCommandline) do |r|
+				r[:exitcode] = 1 if r[:stderr].length > 0
+			end
+		end
 
     def get_log
       logger.close
@@ -318,7 +190,7 @@ module YoreCore
     end
 
     def get_report
-      MiscUtils::string_from_file(reporter.logdev.filename)
+      MiscUtils::string_from_file(@reporter.logdev.filename)
     end
 
     def self.filemap_from_filelist(aFiles)
@@ -334,11 +206,21 @@ module YoreCore
 
     end
 
+		#def self.nice_format(aNumber)
+		#	if aNumber >= 100
+		#		sprintf('%.0f', aNumber)
+		#	else
+		#		sprintf('%.3f', aNumber).sub(/\.0{1,3}$/, '')
+		#	end
+		#end
+
     # By default, GNU tar suppresses a leading slash on absolute pathnames while creating or reading a tar archive. (You can suppress this with the -p option.)
     # tar : http://my.safaribooksonline.com/0596102461/I_0596102461_CHP_3_SECT_9#snippet
 
     # get files from wherever they are into a single file
     def collect(aSourceFiles,aDestFile,aParentDir=nil)
+			logger.info "Collecting files ..."
+			logger.info aSourceFiles.join("\n")
       filelist = filemap = nil
       if aSourceFiles.is_a?(Hash)
         filelist = aSourceFiles.keys
@@ -354,12 +236,17 @@ module YoreCore
         listfile
       )
       tarfile = MiscUtils.file_change_ext(aDestFile, 'tar')
+			
 			shell("tar cv --directory=#{aParentDir} --file=#{tarfile} --files-from=#{listfile}")			
       shell("tar --append --directory=#{aParentDir} --file=#{tarfile} .contents")
+			logger.info "Compressing ..."
+			tarfile_size = File.size(tarfile)
       shell("bzip2 #{tarfile}; mv #{tarfile}.bz2 #{aDestFile}")
+			logger.info "Compressed #{'%.1f' % (tarfile_size*1.0/2**10)} KB to #{'%.1f' % (File.size(aDestFile)*1.0/2**10)} KB"
     end
 
     def pack(aFileIn,aFileOut)
+			logger.info "Encrypting ..."
       shell "openssl enc -aes-256-cbc -K #{config[:crypto_key]} -iv #{config[:crypto_iv]} -in #{aFileIn} -out #{aFileOut}"
     end
 
@@ -369,18 +256,20 @@ module YoreCore
 
     def ensure_bucket(aBucket=nil)
       aBucket ||= config[:bucket]
-      shell "s3cmd createbucket #{aBucket}"
+			logger.info "Ensuring S3 bucket #{aBucket} exists ..."
+      s3shell "s3cmd createbucket #{aBucket}"
     end
 
     # uploads the given file to the current bucket as its basename
     def upload(aFile)
       ensure_bucket()
-      shell "s3cmd put #{config[:bucket]}:#{File.basename(aFile)} #{aFile}"
+			logger.info "Uploading #{File.basename(aFile)} to S3 bucket #{config[:bucket]} ..."
+      s3shell "s3cmd put #{config[:bucket]}:#{File.basename(aFile)} #{aFile}"
     end
 
     # downloads the given file from the current bucket as its basename
     def download(aFile)
-      shell "s3cmd get #{config[:bucket]}:#{File.basename(aFile)} #{aFile}"
+      s3shell "s3cmd get #{config[:bucket]}:#{File.basename(aFile)} #{aFile}"
     end
 
     # calculate the date (with no time component) based on :day_begins_hour and the local time
@@ -407,7 +296,6 @@ module YoreCore
     #end
 
     def backup_process(aSourceFiles,aTimeNow=Time.now,aTempDir=nil)
-      #require 'ruby-debug'; debugger
       aTempDir ||= MiscUtils.make_temp_dir('yore_')
       temp_file = File.expand_path('backup.tar',aTempDir)
       collect(aSourceFiles,temp_file)
@@ -418,6 +306,7 @@ module YoreCore
 		
 		# aDb : Hash containing :db_host,db_user,db_password,db_name,		
 		def db_to_file(aDb,aFile)
+			logger.info "Dumping database #{aDb[:db_name]} ..."
 			shell "#{config[:mysqldump]} --host=#{aDb[:db_host]} --user=#{aDb[:db_user]} --password=#{aDb[:db_password]} --databases --skip-extended-insert --add-drop-database #{aDb[:db_name]} > #{aFile}"
 		end
 		
@@ -449,6 +338,7 @@ module YoreCore
     def report
       return unless config[:email_report]
       msg = get_report()
+			logger.info "Sending report via email to #{config[:mail_to]} ..."
       MiscUtils::send_email(
         :host => config[:mail_host],
         :port => config[:mail_port],
@@ -476,7 +366,6 @@ module YoreCore
     end
 
     def backup(aJobFiles)
-      #require 'ruby-debug'; debugger
       return unless job = aJobFiles.is_a?(Array) ? aJobFiles.first : aJobFiles # just use first job
 
       xmlRoot = XmlUtils.get_file_root(job)
@@ -501,16 +390,16 @@ module YoreCore
               args = Yore::database_from_xml(xmlDb)
               file = args.delete(:file)
 							unless args[:db_host] && args[:db_user] && args[:db_password] && args[:db_name] && file
-								raise Error.new("Invalid or missing parameter")
+								raise StandardError.new("Invalid or missing parameter")
 							end
 							db_to_file(args,file)
-							filelist += file
+							filelist << file
               sourceFound = true
 						end
 				end
       end
 
-      raise Error.new("Backup source found but file list empty") if sourceFound && filelist.empty?
+      raise StandardError.new("Backup source found but file list empty") if sourceFound && filelist.empty?
 
       filelist.uniq!
       filelist.sort!
@@ -520,7 +409,6 @@ module YoreCore
 
       backup_process(filelist,time,tempdir)
       #clean
-      report
     end
 		
 		def test_email(*aDb)
@@ -553,11 +441,11 @@ module YoreCore
       else
         xmlDb = XmlUtils::single_node(xmlRoot,"/Yore/Sources/Source[@Type='MySql']/Database")
       end
-      raise Error.new("No database") unless xmlDb
+      raise StandardError.new("No database") unless xmlDb
       args = Yore.database_from_xml(xmlDb)
       file = args.delete(:file)
       unless args[:db_host] && args[:db_user] && args[:db_password] && args[:db_name] && file
-        raise Error.new("Invalid or missing parameter")
+        raise StandardError.new("Invalid or missing parameter")
       end
       db_to_file(args,file)
     end

data/test/test_job_a.xml

@@ -8,7 +8,7 @@
         <Item Name="crypto_key">07692FC8656F04AE5518B80D38681E038A3C12050DF6CC97CEEC33D800D5E2FE</Item>
         <Item Name="first_hour">4</Item>
         <Item Name="prefix">backup</Item>
-        <Item Name="log_level">DEBUG</Item>
+        <Item Name="log_level">INFO</Item>
     </SimpleItems>
 	<Sources>
         <Source>

data/test/test_job_b.xml

@@ -9,7 +9,7 @@
 		<Item Name="bucket">yoretest</Item>
 		<Item Name="first_hour">4</Item>
 		<Item Name="prefix">yoretest</Item>
-		<Item Name="log_level">DEBUG</Item>
+		<Item Name="log_level">INFO</Item>
 		<Item Name="email_report">false</Item>
 
 		<Item Name="mail_host">mail.authsmtp.com</Item>

data/test/yore_test.rb

@@ -7,6 +7,7 @@ gem 'Shoulda'; require 'shoulda'
 
 	
 require 'ihl_ruby/misc_utils'
+require 'ihl_ruby/shell_extras'
 
 require 'fileutils'
 
@@ -126,7 +127,7 @@ job_template = <<EOF
         <Item Name="crypto_iv">3A63775C1E3F291B0925578165EB917E</Item>
         <Item Name="crypto_key">07692FC8656F04AE5518B80D38681E038A3C12050DF6CC97CEEC33D800D5E2FE</Item>
         <Item Name="prefix">backup</Item>
-        <Item Name="log_level">DEBUG</Item>
+        <Item Name="log_level">INFO</Item>
         <Item Name="bucket">${BUCKET}</Item>
     </SimpleItems>
 	<Sources>
@@ -186,6 +187,14 @@ EOF
 		aController = YoreCore::Yore.new      # main program object
     job = File.expand_path('../../test/test_job_b.xml',THIS_DIR)
     xmlRoot = XmlUtils.get_file_root(job)
+    srcdir = '/tmp/yoretest'
+    FileUtils.rm_rf srcdir+'/*'
+    FileUtils.mkdir_p srcdir
+    ['a','a/1','b','c'].each {|p| FileUtils.mkdir_p(File.expand_path(p,srcdir))}
+    %w(a/blah.txt a/1/blahblah.txt b/apples.txt c/carrots.txt).each do |f|
+      MiscUtils::make_temp_file(f,srcdir)
+    end
+
     aController.configure(xmlRoot,nil,{:basepath => File.dirname(File.expand_path(job))})
     aController.backup(job)
 	end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: yore
 version: !ruby/object:Gem::Version 
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors: 
 - Gary McGhee
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-02-04 00:00:00 +09:00
+date: 2009-02-20 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -90,8 +90,11 @@ files:
 - lib/ihl_ruby/enum.rb
 - lib/ihl_ruby/extend_base_classes.rb
 - lib/ihl_ruby/misc_utils.rb
+- lib/ihl_ruby/config.rb
+- lib/ihl_ruby/logging.rb
 - lib/ihl_ruby/string_utils.rb
 - lib/ihl_ruby/xml_utils.rb
+- lib/ihl_ruby/shell_extras.rb
 - lib/yore.orig.rb
 - lib/yore/yore_core.rb
 - Manifest.txt