yore 0.0.3 → 0.0.4

data/History.txt

@@ -13,4 +13,10 @@
 
 * much better logging, reporting and console output
 
+== 0.0.4 2009-07-30
+
+* now a more general tool for archiving web application data.
+* supports specific web applications via the kind option eg. rails and spree initially
+* refactored config functionality
+
 

data/Manifest.txt

@@ -8,6 +8,7 @@ lib/ihl_ruby/logging.rb
 lib/ihl_ruby/string_utils.rb
 lib/ihl_ruby/xml_utils.rb
 lib/ihl_ruby/shell_extras.rb
+lib/ihl_ruby/database_utils.rb
 lib/yore.orig.rb
 lib/yore/yore_core.rb
 Manifest.txt
@@ -20,3 +21,6 @@ script/generate
 test/test_job_a.xml
 test/test_job_b.xml
 test/yore_test.rb
+test/loadsave_yore_test.rb
+test/S3_test.rb
+

data/README.rdoc

@@ -5,19 +5,46 @@
 
 == DESCRIPTION:
 
-yore (as in "days of yore") is hands-off scheduled backup utility designed for web servers.
-It does backups of files and databases (MySQL only at present) to Amazon S3.
+yore (as in "days of yore") is a data management utility for web applications.
+It provides hands-off scheduled backup functions, combines file and database 
+data into a single file, knows certain applications, particularly Rails related ones,
+and can use Amazon S3 for storage.
 
 == FEATURES/PROBLEMS:
 
 * Compressed, encrypted, single file backups of folders and mysql databases
-* Designed to be called regularly
-* Backups are uploaded to Amazon S3
-* will later remove old files that don't match the configurable scheme for backup history
+* Can be called regularly eg. by cron
+* Backups can be uploaded to Amazon S3
+* will later remove old files that don't match the configurable scheme for backup history,
+but keeping a useful history eg. 
+- backup every day
+- keep 2 weeks of daily backups
+- keep 12 weeks of friday backups
+- keep the first friday backup of each month forever
+* Can automatically collect and compress database all user data from particular applications
+to a single file, and restore to another server. Known applications are Rails-centric but others
+can be manually configured. 
+* Rails database credentials are read from database.yml
+
+Amazon S3
+
+* Yore relies on s3cmd for S3 access which is contained in the s3sync gem.  
+s3cmd must be configured with keys for a registered Amazon S3 account.  
+Use something like "s3cmd listbuckets" to check that you have that setup 
+correctly before launching yore.  
+
+* You can and probably should create a seperate Amazon account for 1 or more web servers, 
+and then grant write access to a single bucket for their use, rather than using your main
+S3 keys. You don't need to supply your credit card details for this account - your main 
+account will be charged. This avoids your main Amazon AWS keys getting stolen by someone 
+hacking into the webserver, or even other legitimate users using them.
 
 == SYNOPSIS:
 
-yore backup yore_config.xml
+yore [global options] command [command options] [arguments]
+eg.
+* yore backup yore_config.xml
+* cd my_rails_app; yore save --kind=spree data.tgz
 
 == REQUIREMENTS:
 

data/bin/yore

@@ -4,7 +4,7 @@ require 'fileutils'
 
 require 'rubygems'
 gem 'RequirePaths'; require 'require_paths'
-require_paths '.','../../..','/Users/gary/repos/Buzzware/projects_2009/yore/lib'
+require_paths '.','../../..'
 
 gem 'cmdparse'; require 'cmdparse'
 
@@ -23,18 +23,17 @@ def command(aParser,aController,aAction,aShortDescription=nil,aOptionParser=nil,
 	c.description = aOther[:description] if aOther[:description]
 	c.options = aOptionParser if aOptionParser
   c.set_execution_block do |args|
-    job = args.first
-		aController.logger.info "Job file: #{File.expand_path(job)}"
-    xmlRoot = XmlUtils.get_file_root(job)
-		aController.configure(xmlRoot,CMD_OPTIONS,{:basepath => File.dirname(File.expand_path(job))})
-		aController.do_action(aAction,args)
+		if job = CMD_OPTIONS[:config]
+			aController.configure(job,CMD_OPTIONS)
+		end
+		aController.do_action(aAction,args,CMD_OPTIONS)
   end
   aParser.add_command(c)
 end
 
 cmd = CmdParse::CommandParser.new( true )
 cmd.program_name = "yore"
-cmd.program_version = [0, 0, 1]
+cmd.program_version = [0, 0, 4]
 # Options are given after a command and before arguments on the command line
 # so global options are given first, before the first command
 # ie ruby yore.rb --global_option command --command_option argument1 argument2 argumentn
@@ -43,6 +42,9 @@ cmd.options = CmdParse::OptionParserWrapper.new do |opt|
   opt.on("--verbose", "Be verbose when outputting info") do |t|
     CMD_OPTIONS[:verbose] = t
   end
+  opt.on("--config", "Configuration XML File") do |t|
+    CMD_OPTIONS[:config] = t
+  end
 end
 cmd.add_command( CmdParse::HelpCommand.new )
 cmd.add_command( CmdParse::VersionCommand.new )
@@ -56,8 +58,21 @@ option_parser = CmdParse::OptionParserWrapper.new do |opt|
   end
 end
 
+load_save_option_parser = CmdParse::OptionParserWrapper.new do |opt|
+  opt.on( '--kind=rails|spree', String, 'Specify application to configure for' ) do |value|
+    CMD_OPTIONS[:kind] = value
+  end
+  opt.on( '--RAILS_ENV=development|test|production', String, 'Specify Rails environment to use database credentials for' ) do |value|
+    CMD_OPTIONS[:RAILS_ENV] = value
+  end
+end
+
 command(cmd,yore,:backup,"Backup filelist to S3",option_parser)
 
+command(cmd,yore,:save,"Save application data to local file",load_save_option_parser)
+
+command(cmd,yore,:load,"Load application data from local file",load_save_option_parser)
+
 command(cmd,yore,:test_email,"Test email sending\n")
 
 command(cmd,yore,:db_dump,"Dump database by name in job\n")

data/lib/ihl_ruby/config.rb

@@ -1,3 +1,8 @@
+require 'rubygems'
+gem 'RequirePaths'; require 'require_paths'
+require_paths '..'
+require 'ihl_ruby/xml_utils'
+
 class ConfigClass < Hash
 
 	attr_reader :default_values
@@ -105,3 +110,93 @@ class ConfigClass < Hash
 
 end
 
+class ConfigXmlClass < ConfigClass
+	attr_accessor :xmlRoot
+	def initialize(aDefaultValues,aConfig)
+		return super(aDefaultValues,aConfig) unless aConfig.is_a?(REXML::Element)
+		@xmlRoot = aConfig.deep_clone
+		super(aDefaultValues,XmlUtils.read_simple_items(@xmlRoot,'/Yore/SimpleItems'))
+	end
+end
+
+# credentials files look like :
+#<?xml version="1.0" encoding="UTF-8"?>
+#<Credentials>
+#	<SimpleItems namespace="global">
+#		<Item name=""></Item>
+#		<Item name=""></Item>
+#		<Item name=""></Item>
+#	</SimpleItems>
+#	<SimpleItems namespace="yore_test">
+#		<Item name=""></Item>
+#		<Item name=""></Item>
+#		<Item name=""></Item>
+#	</SimpleItems>
+#</Credentials>
+#
+# global .credentials.xml file  
+# local .credentials.xml file
+# cred = Credentials.new()	# optionally specify filename or path or hash. if nil then use Dir.pwd
+#
+# def initialize(aSource)
+#		# load global namespace from ~/.credentials.xml
+#		# load global namespace from local .credentials.xml
+#		# load given namespace from ~/.credentials.xml
+#		# load given namespace from local .credentials.xml
+#		# merge all top to bottom
+class Credentials < Hash
+
+	CRED_FILENAME = ".credentials.xml"
+
+	def find_file_upwards(aFilename,aStartPath=nil)
+		aStartPath ||= Dir.pwd
+		return nil if aFilename.nil? || aFilename.empty?
+		arrPath = aStartPath.split(File::SEPARATOR)
+		while arrPath.length > 0
+			path = File.join(arrPath.join(File::SEPARATOR),aFilename)
+			return path if File.exists?(path)
+			arrPath.pop
+		end
+		return nil
+	end
+	
+	def get_all_credentials(aXmlRoot)
+		return nil unless aXmlRoot
+		result = {}
+		REXML::XPath.each(aXmlRoot, '/Credentials/SimpleItems') do |si|
+			ns = si.attributes['Namespace']
+			values = XmlUtils.read_simple_items(si)
+			result[ns.to_sym] = values.symbolize_keys if ns && values
+		end
+		return result
+	end
+
+	#XmlUtils.read_simple_items(@xmlRoot,'/Yore/SimpleItems')
+	def get_user_credentials
+		return get_all_credentials(XmlUtils.get_file_root(File.join(HOME_PATH,CRED_FILENAME)))
+	end
+
+	def get_local_credentials(aSource=nil)
+		aSource ||= Dir.pwd
+		# assume source is a directory path, but other types could be supported later
+		return nil unless file=find_file_upwards(CRED_FILENAME,aSource)
+		return get_all_credentials(XmlUtils.get_file_root(file))
+	end
+
+	def initialize(aNamespace=nil,aSource=nil)
+		#HOME_PATH can be preset by tests eg. ::Credentials.const_set('HOME_PATH',@user_dir)
+		Credentials.const_set("HOME_PATH", ENV['HOME']) unless Credentials.const_defined? "HOME_PATH"
+		arrCredentials = []
+		user_credentials = get_user_credentials()
+		local_credentials = get_local_credentials(aSource)
+		arrCredentials << user_credentials[:global] if user_credentials
+		arrCredentials << local_credentials[:global] if local_credentials
+		arrCredentials << user_credentials[aNamespace.to_sym] if aNamespace && user_credentials
+		arrCredentials << local_credentials[aNamespace.to_sym] if aNamespace && local_credentials
+		arrCredentials.compact!
+		arrCredentials.each do |c|
+			self.merge!(c)
+		end
+	end
+end
+

data/lib/ihl_ruby/database_utils.rb

@@ -0,0 +1,89 @@
+require 'rubygems'
+gem 'RequirePaths'; require 'require_paths'
+require_paths '..'
+require 'ihl_ruby/shell_extras'
+
+module DatabaseUtils
+  def self.execute_sql_file(filename,aUser=nil,aPassword=nil)
+    conf = ActiveRecord::Base.configurations[RAILS_ENV]
+    pw = aPassword || conf['password'].to_s || ''
+    user = aUser || conf['username'].to_s || ''
+    cmd_line = "mysql -h #{conf['host']} -D #{conf['database']} #{user.empty? ? '' : '-u '+user} #{pw.empty? ? '' : '-p'+pw} <#{filename}"
+    if !system(cmd_line)
+      raise Exception, "Error executing "+cmd_line
+    end
+  end
+	
+	## http://www.cyberciti.biz/faq/how-do-i-empty-mysql-database/
+	#
+	#
+	## drop all tables :
+	##			mysqldump -uusername -ppassword -hhost \
+	##--add-drop-table --no-data database | grep ^DROP | \
+	##mysql -uusername -ppassword -hhost database
+	#
+
+	def self.database_exists(aDbDetails,aDatabase=nil)
+		aDbDetails[:database] = aDatabase if aDatabase
+		return false if !aDbDetails[:database]
+		response = POpen4::shell("mysql -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} -e 'use #{aDbDetails[:database]}'") do |r|
+			if r[:stderr] && r[:stderr].index("ERROR 1049 ")==0		# Unknown database
+				r[:exitcode] = 0 
+				return false
+			end
+		end
+		return (response && response[:exitcode]==0)
+	end
+
+	def self.clear_database(aDbDetails)
+		response = POpen4::shell("mysqldump -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} --add-drop-table --no-data #{aDbDetails[:database]} | grep ^DROP | mysql -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} #{aDbDetails[:database]}")
+	end
+
+	def self.create_database(aDbDetails,aDatabase=nil)
+		aDbDetails[:database] = aDatabase if aDatabase
+		return false if !aDbDetails[:database]
+		response = POpen4::shell("mysqladmin -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} create #{aDbDetails[:database]}")
+	end
+
+	def self.ensure_empty_database(aDbDetails,aDatabase=nil)
+		aDbDetails[:database] = aDatabase if aDatabase
+		if database_exists(aDbDetails)
+			clear_database(aDbDetails)
+		else
+			create_database(aDbDetails)
+		end
+	end
+	
+	def self.load_database(aDbDetails,aSqlFile)
+		ensure_empty_database(aDbDetails)
+		response = POpen4::shell("mysql -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} #{aDbDetails[:database]} < #{aSqlFile}")
+	end
+
+	def self.save_database(aDbDetails,aSqlFile)
+		response = POpen4::shell("mysqldump --user=#{aDbDetails[:username]} --password=#{aDbDetails[:password]} --skip-extended-insert #{aDbDetails[:database]} > #{aSqlFile}")
+	end
+
+	#
+	## eg. rake metas:spree:data:load from=/tmp/spree_data.tgz to=mysql:fresco_server_d:root:password
+	#desc 'load spree data from a file'
+	#task :load do
+	#	from = ENV['from']
+	#	to=ENV['to']
+	#	db_server,db,user,password = to.split(':')
+	#	tmpdir = make_temp_dir('metas')
+	#	cmd = "tar -xvzf #{from} -C #{tmpdir}"
+	#	puts CapUtilsClass.shell(cmd)
+	#
+	#	ensure_empty_database(db_server,db,user,password)
+	#
+	#	puts CapUtilsClass.shell("mysql -u #{user} -p#{password} #{db} < #{File.join(tmpdir,'db/dumps/db.sql')}")
+	#	FileUtils.mkdir_p('public/assets')
+	#	puts CapUtilsClass.shell("cp -rf #{File.join(tmpdir,'public/assets/products')} public/assets/products")
+	#end
+
+
+
+
+end
+
+

data/lib/ihl_ruby/extend_base_classes.rb

@@ -224,6 +224,12 @@ module HashUtils
 		end
 		return result
 	end
+	
+	def symbolize_keys
+		result = {}
+		self.each { |k,v| k.is_a?(String) ? result[k.to_sym] = v : result[k] = v  }
+		return result
+	end
 
 end
 

data/lib/ihl_ruby/misc_utils.rb

@@ -138,7 +138,7 @@ module MiscUtils
 	def self.path_debase(aPath,aBase)
 		aBase = MiscUtils::append_slash(aBase)
 		aPath = MiscUtils::remove_slash(aPath) unless aPath=='/'
-		aPath[aBase.length,aPath.length-aBase.length]
+		aPath[0,aBase.length]==aBase ? aPath[aBase.length,aPath.length-aBase.length] : aPath
 	end
 	
 	def self.path_rebase(aPath,aOldBase,aNewBase)
@@ -230,6 +230,7 @@ module MiscUtils
 			abssrcpath = aRootPath = aPath
 			aPath = nil
 		end
+		return aArray if !File.exists?(abssrcpath)
 		#abssrcpath is real path to query
 		#aRootPath is highest level path
 		#aPath is current path relative to aRootPath

data/lib/ihl_ruby/xml_utils.rb

@@ -20,6 +20,7 @@ module XmlUtils
   end
 	
 	def self.get_file_root(aFilename)
+		return nil unless File.exists?(aFilename)
 		get_xml_root(MiscUtils.string_from_file(aFilename))
   end
 
@@ -35,8 +36,10 @@ module XmlUtils
     return val.nil? ? default : val.to_s
   end
 
-  def self.peek_node_value(node,xpath,default=nil)
-		peek_node(node,xpath+'/text()',default)
+  def self.peek_node_value(aNode,aXPath,aDefault=nil)
+		node = single_node(aNode,aXPath)
+		return node.to_s if node.is_a?(REXML::Attribute)
+		return node.nil? ? aDefault : node.text()
 	end
 
 	# convert <root><a>one</a><b>two</b></root> to {'a' => 'one', 'b' => 'two'}
@@ -66,6 +69,15 @@ module XmlUtils
 		result += aText ? " >#{aText}</#{aName}>" : " />"
 	end
 
+	def self.add_xml_from_string(aString,aNode)
+		return nil unless xdoc = REXML::Document.new('<?xml version="1.0" encoding="UTF-8"?>'+aString)
+		r = xdoc.root
+		r.remove()
+		r.parent = nil
+		aNode.add_element(r)
+		return r
+	end
+
 	def self.hash_to_xml(aHash,aRootName,aDocHeader=true)
     xdoc = REXML::Document.new(BASIC_HEADER)
 		root = xdoc.add_element(aRootName)
@@ -75,9 +87,10 @@ module XmlUtils
 		return xdoc
 	end
 
-	def self.read_simple_items(aRoot,aParentXPath)
+	def self.read_simple_items(aRoot,aParentXPath=nil)
 		result = {}
-		REXML::XPath.each(aRoot, aParentXPath+'/Item') do |item|
+		xp = aParentXPath ? File.join(aParentXPath,'Item') : 'Item'
+		REXML::XPath.each(aRoot, xp) do |item|
 			result[item.attribute('Name').to_s] = item.text
     end
 		return result
@@ -91,8 +104,9 @@ module XmlUtils
 		return result
 	end
 
+	# reads the simple items format given either a filename or xml node
 	def self.read_config_values(aXmlConfig)
-		xmlRoot = get_file_root(aXmlConfig)
+		xmlRoot = aXmlConfig.is_a?(REXML::Element) ? aXmlConfig : get_file_root(aXmlConfig)
 		return read_simple_items(xmlRoot,'/Config/SimpleItems')
   end
 

data/lib/yore.orig.rb

@@ -2,5 +2,5 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module Yore
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

data/lib/yore/yore_core.rb

@@ -13,12 +13,11 @@ require 'ihl_ruby/xml_utils'
 require 'ihl_ruby/extend_base_classes'
 require 'ihl_ruby/shell_extras'
 require 'ihl_ruby/config'
+require 'ihl_ruby/database_utils'
 
-THIS_FILE = __FILE__
+THIS_FILE = File.expand_path(__FILE__)
 THIS_DIR = File.dirname(THIS_FILE)
 
-
-
 module YoreCore
 
   class KeepDaily
@@ -79,11 +78,11 @@ module YoreCore
       end
   end
 
-
-
   class Yore
 
     DEFAULT_CONFIG = {
+			:kind => '',
+			:basepath => '',
       :keep_daily => 14,
       :keep_weekly => 12,
       :keep_monthly => 12,
@@ -104,18 +103,17 @@ module YoreCore
 			:mail_to => '',
 			:mail_to_alias => '',
 			:mail_auth => :plain,
-			:mysqldump => 'mysqldump'
+			:mysqldump => 'mysqldump',
+			:RAILS_ENV => ''
 		}
 
     attr_reader :config
     attr_reader :logger
     attr_reader :reporter
     attr_reader :keepers
-    attr_reader :basepath
 
     def initialize(aConfig=nil)
       DEFAULT_CONFIG[:email_report] = false  # fixes some bug where this was nil
-			@config = ConfigClass.new(DEFAULT_CONFIG,aConfig)
 
 			cons = ConsoleLogger.new()
 			cons.level = Logger::Severity.const_get(config[:log_level]) rescue Logger::Severity::INFO
@@ -126,7 +124,6 @@ module YoreCore
 			@reporter.level = cons.level
 
 			@logger = MultiLogger.new([cons,@reporter])
-			#require 'ruby-debug'; debugger
 			@logger.info "Yore file and database backup tool for Amazon S3 "
 			@logger.info "(c) 2009 Buzzware Solutions (www.buzzware.com.au)"
 			@logger.info "-------------------------------------------------"
@@ -134,44 +131,126 @@ module YoreCore
 
       @logger.info "report file: #{report_file}"
 
-      configure(@config)
+      configure(aConfig)
     end
+		
+		#aOptions may require {:basepath => File.dirname(File.expand_path(job))}
+		def self.launch(aConfigXml,aCmdOptions=nil,aOptions=nil)
+			result = Yore.new()
+			result.configure(aConfigXml,aCmdOptions,aOptions)
+			return result
+		end
+
+		def create_empty_config_xml()
+			s = <<-EOS
+				<?xml version="1.0" encoding="UTF-8"?>
+				<?xml version="1.0" encoding="UTF-8"?>
+				<Yore>
+					<SimpleItems>
+					</SimpleItems>
+					<Sources>
+					</Sources>
+				</Yore>
+			EOS
+			xdoc = REXML::Document.new(s)
+			return xdoc.root
+		end
+		
+		def get_rails_db_details(aRailsPath,aRailsEnv)
+			return nil unless aRailsPath && aRailsEnv && aRailsEnv!=''
+			return nil unless dbyml = (YAML::load(File.open(File.expand_path('config/database.yml',aRailsPath))) rescue nil)	
+			return dbyml[aRailsEnv] && dbyml[aRailsEnv].symbolize_keys
+		end
+
+		def expand_app_option(kind=nil)
+			kind = config[:kind] unless kind && !kind.empty?
+			return nil unless kind && !kind.empty?
+			config.xmlRoot = create_empty_config_xml() if !config.xmlRoot
+			case kind
+				when 'spree'
+					# add file source
+					xmlSources = XmlUtils.single_node(config.xmlRoot,'/Yore/Sources')
+					if xmlSources
+						strSource = <<-EOS
+							<Source Type="File">
+								<IncludePath>public/assets/products</IncludePath>
+							</Source>
+						EOS
+						XmlUtils.add_xml_from_string(strSource,xmlSources)
+					end
+					expand_app_option('rails')	# do again
+				when 'rails'
+					# * add db source from database.yml
+					# load database from config[:basepath],'config/database.yml'
+					#if (dbyml = YAML::load(File.open(File.expand_path('config/database.yml',config[:basepath]))) rescue nil)
+					#	if env = (config[:RAILS_ENV] && config[:RAILS_ENV]!='' && config[:RAILS_ENV])
+					#		if (db_details = dbyml[env]) &&
+					db_details = get_rails_db_details(config[:basepath],config[:RAILS_ENV])
+					xmlSources = XmlUtils.single_node(config.xmlRoot,'/Yore/Sources')
+					if db_details && xmlSources
+						strSource = <<-EOS
+							<Source Type="MySql" >
+								<Database Name="#{db_details[:database]}" Host="#{db_details[:host]}" User="#{db_details[:username]}" Password="#{db_details[:password]}">
+									<ArchiveFile>rails_app.sql</ArchiveFile>
+								</Database>
+							</Source>
+						EOS
+						XmlUtils.add_xml_from_string(strSource,xmlSources)
+					end
+			end
+		end
 
     # read the config however its given and return a hash with values in their correct type, and either valid or nil
     # keys must be :symbols for aOptions. aConfig and aCmdOptions can be strings
     def configure(aConfig,aCmdOptions = nil,aOptions = nil)
-      config_h = nil
-      case aConfig
-        when Hash,::ConfigClass then config_h = aConfig
-        when REXML::Element then config_h = XmlUtils.read_simple_items(aConfig,'/Yore/SimpleItems')
-        else
-          raise StandardError.new('unsupported type')
-      end
-      config_i = {}
-      config_h.each{|n,v| config_i[n.to_sym] = v} if config_h
-      aCmdOptions.each{|k,v| config_i[k.to_sym] = v} if aCmdOptions
-      config_i.merge!(aOptions) if aOptions
-			config.read(config_i)
+      config_to_read = {}
+			if aConfig.is_a?(String)
+				aConfig = File.expand_path(aConfig)
+				logger.info "Job file: #{aConfig}"
+				op = {:basepath => File.dirname(aConfig)}
+				xml = XmlUtils.get_file_root(aConfig)
+				return configure(xml,aCmdOptions,op)
+			end
+
+			if @config
+				config_as_hash = nil
+				case aConfig
+					when nil then ;	# do nothing
+					when Hash,::ConfigClass then config_as_hash = aConfig
+					when REXML::Element then
+						config_as_hash = XmlUtils.read_simple_items(aConfig,'/Yore/SimpleItems')
+						config.xmlRoot = aConfig 	# overwriting previous! perhaps should merge
+					else
+						raise StandardError.new('unsupported type')
+				end
+				config_as_hash.each{|n,v| config_to_read[n.to_sym] = v} if config_as_hash					# merge given new values
+			else
+				@config = ConfigXmlClass.new(DEFAULT_CONFIG,aConfig)
+			end		
+      aCmdOptions.each{|k,v| config_to_read[k.to_sym] = v} if aCmdOptions		# merge command options
+      config_to_read.merge!(aOptions) if aOptions														# merge options
+			config.read(config_to_read)
+			config[:basepath] = File.expand_path(Dir.pwd) if !config[:basepath] || config[:basepath]==''
+
+			expand_app_option()
 
       @keepers = Array.new
       @keepers << KeepDaily.new(config[:keep_daily])
       @keepers << KeepWeekly.new(config[:keep_weekly])
       @keepers << KeepMonthly.new(config[:keep_monthly])
-			
-      @basepath = config_h[:basepath]
     end
 		
-		def do_action(aAction,aArgs)
+		def do_action(aAction,aArgs,aCmdOptions)
 			logger.info "Executing command: #{aAction} ...\n"
 			begin
-				send(aAction,aArgs)
+				send(aAction,aArgs,aCmdOptions)
 			rescue Exception => e
-				logger.warn "#{e.class.to_s}: during #{aAction}(#{aArgs.inspect}): #{e.message}"
+				logger.info {e.backtrace.join("\n")}				
+				logger.warn "#{e.class.to_s}: during #{aAction.to_s}(#{(aArgs && aArgs.inspect).to_s}): #{e.message.to_s}"
 			end			
 		end
 
 		def shell(aCommandline,&aBlock)
-			#require 'ruby-debug'; debugger
 			logger.debug "To shell: " + aCommandline
       result = block_given? ? POpen4::shell(aCommandline,nil,nil,&aBlock) : POpen4::shell(aCommandline)
       logger.debug "From shell: '#{result.inspect}'"
@@ -192,6 +271,11 @@ module YoreCore
     def get_report
       MiscUtils::string_from_file(@reporter.logdev.filename)
     end
+		
+		def temp_path
+			@temp_path = MiscUtils.make_temp_dir('yore') unless @temp_path
+			return @temp_path
+		end
 
     def self.filemap_from_filelist(aFiles)
       ancestor_path = MiscUtils.file_list_ancestor(aFiles)
@@ -206,45 +290,52 @@ module YoreCore
 
     end
 
-		#def self.nice_format(aNumber)
-		#	if aNumber >= 100
-		#		sprintf('%.0f', aNumber)
-		#	else
-		#		sprintf('%.3f', aNumber).sub(/\.0{1,3}$/, '')
-		#	end
-		#end
 
     # By default, GNU tar suppresses a leading slash on absolute pathnames while creating or reading a tar archive. (You can suppress this with the -p option.)
     # tar : http://my.safaribooksonline.com/0596102461/I_0596102461_CHP_3_SECT_9#snippet
 
     # get files from wherever they are into a single file
-    def collect(aSourceFiles,aDestFile,aParentDir=nil)
+    def compress(aSourceFiles,aDestFile,aParentDir=nil)
 			logger.info "Collecting files ..."
-			logger.info aSourceFiles.join("\n")
-      filelist = filemap = nil
-      if aSourceFiles.is_a?(Hash)
-        filelist = aSourceFiles.keys
-        filemap = aSourceFiles
-      else  # assume array
-        filelist = aSourceFiles
-        filemap = Yore.filemap_from_filelist(aSourceFiles)
-      end
-      aParentDir ||= MiscUtils.file_list_ancestor(filelist)
-      listfile = File.join(aParentDir,'.contents')
+			#logger.info aSourceFiles.join("\n")
+      #filelist = filemap = nil
+      #if aSourceFiles.is_a?(Hash)
+      #  filelist = aSourceFiles.keys
+      #  filemap = aSourceFiles
+      #else  # assume array
+      #  filelist = aSourceFiles
+      #  filemap = Yore.filemap_from_filelist(aSourceFiles)
+      #end
+      #aParentDir ||= MiscUtils.file_list_ancestor(filelist)
+      listfile = MiscUtils.temp_file
       MiscUtils.string_to_file(
-        filelist.sort.map{|p| MiscUtils.path_debase(p, aParentDir)}.join("\n"),
+        aSourceFiles.join("\n"),			#filelist.sort.map{|p| MiscUtils.path_debase(p, aParentDir)}.join("\n"),
         listfile
       )
       tarfile = MiscUtils.file_change_ext(aDestFile, 'tar')
 			
-			shell("tar cv --directory=#{aParentDir} --file=#{tarfile} --files-from=#{listfile}")			
-      shell("tar --append --directory=#{aParentDir} --file=#{tarfile} .contents")
+			shell("tar cv #{aParentDir ? '--directory='+aParentDir.to_s : ''} --file=#{tarfile} --files-from=#{listfile}")			
 			logger.info "Compressing ..."
 			tarfile_size = File.size(tarfile)
       shell("bzip2 #{tarfile}; mv #{tarfile}.bz2 #{aDestFile}")
 			logger.info "Compressed #{'%.1f' % (tarfile_size*1.0/2**10)} KB to #{'%.1f' % (File.size(aDestFile)*1.0/2**10)} KB"
     end
 
+    def uncompress(aArchive,aDestination=nil,aArchiveContent=nil)			
+      #tarfile = File.expand_path(MiscUtils.file_change_ext(File.basename(aArchive),'tar'),temp_dir)
+      #shell("bunzip2 #{tarfile}; mv #{tarfile}.bz2 #{aDestFile}")
+      #
+			#shell("tar cv #{aParentDir ? '--directory='+aParentDir.to_s : ''} --file=#{tarfile} --files-from=#{listfile}")
+			#logger.info "Compressing ..."
+			#tarfile_size = File.size(tarfile)
+      #shell("bzip2 #{tarfile}; mv #{tarfile}.bz2 #{aDestFile}")
+			#logger.info "Compressed #{'%.1f' % (tarfile_size*1.0/2**10)} KB to #{'%.1f' % (File.size(aDestFile)*1.0/2**10)} KB"
+			aDestination ||= MiscUtils.make_temp_dir('uncompress')
+			FileUtils.mkdir_p(aDestination)
+			shell("tar xvf #{aArchive} #{aArchiveContent.to_s} --directory=#{aDestination} --bzip2")
+    end
+
+
     def pack(aFileIn,aFileOut)
 			logger.info "Encrypting ..."
       shell "openssl enc -aes-256-cbc -K #{config[:crypto_key]} -iv #{config[:crypto_iv]} -in #{aFileIn} -out #{aFileOut}"
@@ -262,7 +353,7 @@ module YoreCore
 
     # uploads the given file to the current bucket as its basename
     def upload(aFile)
-      ensure_bucket()
+      #ensure_bucket()
 			logger.info "Uploading #{File.basename(aFile)} to S3 bucket #{config[:bucket]} ..."
       s3shell "s3cmd put #{config[:bucket]}:#{File.basename(aFile)} #{aFile}"
     end
@@ -291,36 +382,6 @@ module YoreCore
       return Time.from_date_numeric(date)
     end
 
-    #def set_file_name(aFile,aNewName)#
-    #
-    #end
-
-    def backup_process(aSourceFiles,aTimeNow=Time.now,aTempDir=nil)
-      aTempDir ||= MiscUtils.make_temp_dir('yore_')
-      temp_file = File.expand_path('backup.tar',aTempDir)
-      collect(aSourceFiles,temp_file)
-      backup_file = File.expand_path(encode_file_name(aTimeNow),aTempDir)
-      pack(temp_file,backup_file)
-      upload(backup_file)
-    end
-		
-		# aDb : Hash containing :db_host,db_user,db_password,db_name,		
-		def db_to_file(aDb,aFile)
-			logger.info "Dumping database #{aDb[:db_name]} ..."
-			shell "#{config[:mysqldump]} --host=#{aDb[:db_host]} --user=#{aDb[:db_user]} --password=#{aDb[:db_password]} --databases --skip-extended-insert --add-drop-database #{aDb[:db_name]} > #{aFile}"
-		end
-		
-		def file_to_db(aFile,aDatabase)
-			#run "mysql --user=root --password=prot123ection </root/joomla_db_snapshot/joomla_db.sql"
-		end
-    
-		#		
-    #
-    # 											COMMANDLINE COMMANDS
-		#
-		#
-
-
 
     def clean
       
@@ -357,59 +418,142 @@ module YoreCore
 
     def self.database_from_xml(aDatabaseNode)
       return {
-        :db_host => aDatabaseNode.attributes['Host'],
-        :db_user => aDatabaseNode.attributes['User'],
-        :db_password => aDatabaseNode.attributes['Password'],
-        :db_name => aDatabaseNode.attributes['Name'],
-        :file => XmlUtils::peek_node_value(aDatabaseNode, "ToFile")
+        :host => aDatabaseNode.attributes['Host'],
+        :username => aDatabaseNode.attributes['User'],
+        :password => aDatabaseNode.attributes['Password'],
+        :database => aDatabaseNode.attributes['Name'],
+        :file => XmlUtils::peek_node_value(aDatabaseNode, "ToFile"),
+        :archive_file => XmlUtils::peek_node_value(aDatabaseNode, "ArchiveFile")
       }
     end
-
-    def backup(aJobFiles)
-      return unless job = aJobFiles.is_a?(Array) ? aJobFiles.first : aJobFiles # just use first job
-
-      xmlRoot = XmlUtils.get_file_root(job)
-
+		
+		
+		def collect_file_list(aSourcesXml,aTempFolder)
       filelist = []
       sourceFound = false
 
-      REXML::XPath.each(xmlRoot, '/Yore/Sources/Source') do |xmlSource|
-        case xmlSource.attributes['Type']
-					when 'File' then 
-						REXML::XPath.each(xmlSource, 'IncludePath') do |xmlPath|
-							filelist += MiscUtils::recursive_file_list(MiscUtils::path_combine(config[:basepath],xmlPath.text))
-              sourceFound = true
-						end
-					when 'MySql' then
-						#<Source Type="MySql" >
-						#	<Database Host="" Name="" User="" Password="">
-						#		<ToFile>~/dbdump.sql</ToFile>
-						#	</Database>
-						#</Source>
-						REXML::XPath.each(xmlSource, 'Database') do |xmlDb|
-              args = Yore::database_from_xml(xmlDb)
-              file = args.delete(:file)
-							unless args[:db_host] && args[:db_user] && args[:db_password] && args[:db_name] && file
-								raise StandardError.new("Invalid or missing parameter")
+      if aSourcesXml
+				REXML::XPath.each(aSourcesXml,'Source') do |xmlSource|
+					case xmlSource.attributes['Type']
+						when 'File' then
+							# BasePath tag provides base path for IncludePaths to be relative to. Also indicates root folder for archive 
+							bp = MiscUtils.path_combine(config[:basepath],XmlUtils::peek_node_value(xmlSource, "@BasePath"))
+							filelist << '-C'+bp
+							REXML::XPath.each(xmlSource, 'IncludePath') do |xmlPath|
+								files = MiscUtils::recursive_file_list(MiscUtils::path_combine(bp,xmlPath.text))
+								files.map!{|f| MiscUtils.path_debase(f,bp)}
+								filelist += files
+								sourceFound = true
 							end
-							db_to_file(args,file)
-							filelist << file
-              sourceFound = true
-						end
+						when 'MySql' then
+							#<Source Type="MySql" >
+							#	<Database Host="" Name="" User="" Password="">
+							#		<ToFile>~/dbdump.sql</ToFile>
+							#	</Database>
+							#</Source>
+							REXML::XPath.each(xmlSource, 'Database') do |xmlDb|
+								args = Yore::database_from_xml(xmlDb)
+								file = args.delete(:file)							#legacy, absolute path
+								arc_file = args.delete(:archive_file)	#path in archive
+								unless args[:host] && args[:username] && args[:password] && args[:database] && (file||arc_file)
+									raise StandardError.new("Invalid or missing parameter")
+								end
+								if arc_file
+									arc_file = MiscUtils.path_debase(arc_file,'/')
+									sql_file = File.expand_path(arc_file,aTempFolder)
+									FileUtils.mkdir_p(File.dirname(sql_file))  						# create folders as necessry							
+									DatabaseUtils::save_database(args,sql_file)					#db_to_file(args,sql_file)
+									filelist << '-C'+aTempFolder
+									filelist << arc_file
+									sourceFound = true
+								else
+									DatabaseUtils::save_database(args,sql_file)
+									filelist << file
+									sourceFound = true
+								end
+							end
+					end
 				end
-      end
-
+			end
       raise StandardError.new("Backup source found but file list empty") if sourceFound && filelist.empty?
+			return filelist	
+		end
+
+		def rails_tmp_path
+			return @rails_tmp_path if @rails_tmp_path
+			@rails_tmp_path = File.join(config[:basepath],'tmp/yore',Time.now.strftime('%Y%m%d-%H%M%S'))
+		end
+		
+		def self.move_folder(aPath1,aPath2)
+			path2Parent = MiscUtils.path_parent(aPath2)
+			FileUtils.mkdir_p(path2Parent)
+			FileUtils.mv(aPath1, path2Parent, :force => true)
+		end
+
+		def self.copy_folder(aPath1,aPath2)
+			path2Parent = MiscUtils.path_parent(aPath2)
+			FileUtils.mkdir_p(path2Parent)
+			FileUtils.cp_r(aPath1, path2Parent)
+		end
+
+		def save_internal(aFilename)
+			FileUtils.mkdir_p(files_path = File.join(temp_path,'files'))
+			filelist = collect_file_list(XmlUtils.single_node(config.xmlRoot,'/Yore/Sources'),files_path)
+      compress(filelist,aFilename)
+		end
 
-      filelist.uniq!
-      filelist.sort!
+		#
+		#		ACTIONS
+		#
 
-      tempdir = MiscUtils.make_temp_dir('yore')
-      time = Time.now
+		def save(aArgs,aCmdOptions=nil)
+			fnArchive = aArgs.is_a?(Array) ? aArgs.first : aArgs	#only supported argument
+			configure(nil,aCmdOptions)
+			save_internal(fnArchive)
+		end
 
-      backup_process(filelist,time,tempdir)
+    def backup(aArgs,aCmdOptions=nil)	# was aJobFiles
+			unless aCmdOptions && aCmdOptions[:config]	# assume already configured if config option specified, but back supports first arg being config file
+				job = aArgs.first
+				configure(job,aCmdOptions || {})
+			end			
+      temp_file = File.expand_path('backup.tar',temp_path)
+			save_internal(temp_file)
+      backup_file = File.expand_path(encode_file_name(),temp_path)
+      pack(temp_file,backup_file)
+      upload(backup_file)
       #clean
     end
+				
+		def load(aArgs,aCmdOptions=nil)
+			fnArchive = aArgs.is_a?(Array) ? aArgs.first : aArgs	#only supported argument
+			configure(nil,aCmdOptions)
+
+			FileUtils.mkdir_p(archive_path = File.join(temp_path,'archive'))
+			uncompress(fnArchive,archive_path)
+			xmlSources = XmlUtils.single_node(config.xmlRoot,'/Yore/Sources')
+      REXML::XPath.each(xmlSources,'Source') do |xmlSource|
+        case xmlSource.attributes['Type']
+					when 'File' then
+						#<Source Type="File">
+						#	<IncludePath>public/assets/products</IncludePath>
+						#</Source>
+						REXML::XPath.each(xmlSource,'IncludePath') do |xmlIncludePath|
+							pathArchive = xmlIncludePath.text()
+							pathUncompressed = File.join(archive_path,pathArchive)
+							pathTmp = File.join(rails_tmp_path,pathArchive)
+							pathDest = File.join(config[:basepath],pathArchive)
+							# move basepath/relativepath to tmp/yore/090807-010203/relativepath
+							Yore::move_folder(pathDest,pathTmp) if File.exists?(pathDest)
+							# get <IncludeFiles> and copy to basepath/relativepath
+							Yore::copy_folder(pathUncompressed,pathDest) if File.exists?(pathUncompressed)
+						end
+					when 'MySql' then
+						db_details = Yore::database_from_xml(XmlUtils.single_node(xmlSource,'Database'))
+						DatabaseUtils.load_database(db_details,File.join(archive_path,db_details[:archive_file]))
+				end
+			end						
+		end		
 		
 		def test_email(*aDb)
 			args = {
@@ -430,26 +574,6 @@ module YoreCore
       MiscUtils::send_email(args)
 		end
 
-    def db_dump(aArgs)
-      return nil unless aArgs
-      return nil unless job = aArgs[0]
-
-      xmlRoot = XmlUtils.get_file_root(job)
-      xmlDb = nil
-      if db_name = aArgs[1]
-        xmlDb = XmlUtils::single_node(xmlRoot,"/Yore/Sources/Source[@Type='MySql']/Database[@Name='#{db_name}']")
-      else
-        xmlDb = XmlUtils::single_node(xmlRoot,"/Yore/Sources/Source[@Type='MySql']/Database")
-      end
-      raise StandardError.new("No database") unless xmlDb
-      args = Yore.database_from_xml(xmlDb)
-      file = args.delete(:file)
-      unless args[:db_host] && args[:db_user] && args[:db_password] && args[:db_name] && file
-        raise StandardError.new("Invalid or missing parameter")
-      end
-      db_to_file(args,file)
-    end
-
   end
     
 end