yore 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/History.txt CHANGED
@@ -13,4 +13,10 @@
13
13
 
14
14
  * much better logging, reporting and console output
15
15
 
16
+ == 0.0.4 2009-07-30
17
+
18
+ * now a more general tool for archiving web application data.
19
+ * supports specific web applications via the kind option eg. rails and spree initially
20
+ * refactored config functionality
21
+
16
22
 
data/Manifest.txt CHANGED
@@ -8,6 +8,7 @@ lib/ihl_ruby/logging.rb
8
8
  lib/ihl_ruby/string_utils.rb
9
9
  lib/ihl_ruby/xml_utils.rb
10
10
  lib/ihl_ruby/shell_extras.rb
11
+ lib/ihl_ruby/database_utils.rb
11
12
  lib/yore.orig.rb
12
13
  lib/yore/yore_core.rb
13
14
  Manifest.txt
@@ -20,3 +21,6 @@ script/generate
20
21
  test/test_job_a.xml
21
22
  test/test_job_b.xml
22
23
  test/yore_test.rb
24
+ test/loadsave_yore_test.rb
25
+ test/S3_test.rb
26
+
data/README.rdoc CHANGED
@@ -5,19 +5,46 @@
5
5
 
6
6
  == DESCRIPTION:
7
7
 
8
- yore (as in "days of yore") is hands-off scheduled backup utility designed for web servers.
9
- It does backups of files and databases (MySQL only at present) to Amazon S3.
8
+ yore (as in "days of yore") is a data management utility for web applications.
9
+ It provides hands-off scheduled backup functions, combines file and database
10
+ data into a single file, knows certain applications, particularly Rails related ones,
11
+ and can use Amazon S3 for storage.
10
12
 
11
13
  == FEATURES/PROBLEMS:
12
14
 
13
15
  * Compressed, encrypted, single file backups of folders and mysql databases
14
- * Designed to be called regularly
15
- * Backups are uploaded to Amazon S3
16
- * will later remove old files that don't match the configurable scheme for backup history
16
+ * Can be called regularly eg. by cron
17
+ * Backups can be uploaded to Amazon S3
18
+ * will later remove old files that don't match the configurable scheme for backup history,
19
+ but keeping a useful history eg.
20
+ - backup every day
21
+ - keep 2 weeks of daily backups
22
+ - keep 12 weeks of friday backups
23
+ - keep the first friday backup of each month forever
24
+ * Can automatically collect and compress database all user data from particular applications
25
+ to a single file, and restore to another server. Known applications are Rails-centric but others
26
+ can be manually configured.
27
+ * Rails database credentials are read from database.yml
28
+
29
+ Amazon S3
30
+
31
+ * Yore relies on s3cmd for S3 access which is contained in the s3sync gem.
32
+ s3cmd must be configured with keys for a registered Amazon S3 account.
33
+ Use something like "s3cmd listbuckets" to check that you have that setup
34
+ correctly before launching yore.
35
+
36
+ * You can and probably should create a seperate Amazon account for 1 or more web servers,
37
+ and then grant write access to a single bucket for their use, rather than using your main
38
+ S3 keys. You don't need to supply your credit card details for this account - your main
39
+ account will be charged. This avoids your main Amazon AWS keys getting stolen by someone
40
+ hacking into the webserver, or even other legitimate users using them.
17
41
 
18
42
  == SYNOPSIS:
19
43
 
20
- yore backup yore_config.xml
44
+ yore [global options] command [command options] [arguments]
45
+ eg.
46
+ * yore backup yore_config.xml
47
+ * cd my_rails_app; yore save --kind=spree data.tgz
21
48
 
22
49
  == REQUIREMENTS:
23
50
 
data/bin/yore CHANGED
@@ -4,7 +4,7 @@ require 'fileutils'
4
4
 
5
5
  require 'rubygems'
6
6
  gem 'RequirePaths'; require 'require_paths'
7
- require_paths '.','../../..','/Users/gary/repos/Buzzware/projects_2009/yore/lib'
7
+ require_paths '.','../../..'
8
8
 
9
9
  gem 'cmdparse'; require 'cmdparse'
10
10
 
@@ -23,18 +23,17 @@ def command(aParser,aController,aAction,aShortDescription=nil,aOptionParser=nil,
23
23
  c.description = aOther[:description] if aOther[:description]
24
24
  c.options = aOptionParser if aOptionParser
25
25
  c.set_execution_block do |args|
26
- job = args.first
27
- aController.logger.info "Job file: #{File.expand_path(job)}"
28
- xmlRoot = XmlUtils.get_file_root(job)
29
- aController.configure(xmlRoot,CMD_OPTIONS,{:basepath => File.dirname(File.expand_path(job))})
30
- aController.do_action(aAction,args)
26
+ if job = CMD_OPTIONS[:config]
27
+ aController.configure(job,CMD_OPTIONS)
28
+ end
29
+ aController.do_action(aAction,args,CMD_OPTIONS)
31
30
  end
32
31
  aParser.add_command(c)
33
32
  end
34
33
 
35
34
  cmd = CmdParse::CommandParser.new( true )
36
35
  cmd.program_name = "yore"
37
- cmd.program_version = [0, 0, 1]
36
+ cmd.program_version = [0, 0, 4]
38
37
  # Options are given after a command and before arguments on the command line
39
38
  # so global options are given first, before the first command
40
39
  # ie ruby yore.rb --global_option command --command_option argument1 argument2 argumentn
@@ -43,6 +42,9 @@ cmd.options = CmdParse::OptionParserWrapper.new do |opt|
43
42
  opt.on("--verbose", "Be verbose when outputting info") do |t|
44
43
  CMD_OPTIONS[:verbose] = t
45
44
  end
45
+ opt.on("--config", "Configuration XML File") do |t|
46
+ CMD_OPTIONS[:config] = t
47
+ end
46
48
  end
47
49
  cmd.add_command( CmdParse::HelpCommand.new )
48
50
  cmd.add_command( CmdParse::VersionCommand.new )
@@ -56,8 +58,21 @@ option_parser = CmdParse::OptionParserWrapper.new do |opt|
56
58
  end
57
59
  end
58
60
 
61
+ load_save_option_parser = CmdParse::OptionParserWrapper.new do |opt|
62
+ opt.on( '--kind=rails|spree', String, 'Specify application to configure for' ) do |value|
63
+ CMD_OPTIONS[:kind] = value
64
+ end
65
+ opt.on( '--RAILS_ENV=development|test|production', String, 'Specify Rails environment to use database credentials for' ) do |value|
66
+ CMD_OPTIONS[:RAILS_ENV] = value
67
+ end
68
+ end
69
+
59
70
  command(cmd,yore,:backup,"Backup filelist to S3",option_parser)
60
71
 
72
+ command(cmd,yore,:save,"Save application data to local file",load_save_option_parser)
73
+
74
+ command(cmd,yore,:load,"Load application data from local file",load_save_option_parser)
75
+
61
76
  command(cmd,yore,:test_email,"Test email sending\n")
62
77
 
63
78
  command(cmd,yore,:db_dump,"Dump database by name in job\n")
@@ -1,3 +1,8 @@
1
+ require 'rubygems'
2
+ gem 'RequirePaths'; require 'require_paths'
3
+ require_paths '..'
4
+ require 'ihl_ruby/xml_utils'
5
+
1
6
  class ConfigClass < Hash
2
7
 
3
8
  attr_reader :default_values
@@ -105,3 +110,93 @@ class ConfigClass < Hash
105
110
 
106
111
  end
107
112
 
113
+ class ConfigXmlClass < ConfigClass
114
+ attr_accessor :xmlRoot
115
+ def initialize(aDefaultValues,aConfig)
116
+ return super(aDefaultValues,aConfig) unless aConfig.is_a?(REXML::Element)
117
+ @xmlRoot = aConfig.deep_clone
118
+ super(aDefaultValues,XmlUtils.read_simple_items(@xmlRoot,'/Yore/SimpleItems'))
119
+ end
120
+ end
121
+
122
+ # credentials files look like :
123
+ #<?xml version="1.0" encoding="UTF-8"?>
124
+ #<Credentials>
125
+ # <SimpleItems namespace="global">
126
+ # <Item name=""></Item>
127
+ # <Item name=""></Item>
128
+ # <Item name=""></Item>
129
+ # </SimpleItems>
130
+ # <SimpleItems namespace="yore_test">
131
+ # <Item name=""></Item>
132
+ # <Item name=""></Item>
133
+ # <Item name=""></Item>
134
+ # </SimpleItems>
135
+ #</Credentials>
136
+ #
137
+ # global .credentials.xml file
138
+ # local .credentials.xml file
139
+ # cred = Credentials.new() # optionally specify filename or path or hash. if nil then use Dir.pwd
140
+ #
141
+ # def initialize(aSource)
142
+ # # load global namespace from ~/.credentials.xml
143
+ # # load global namespace from local .credentials.xml
144
+ # # load given namespace from ~/.credentials.xml
145
+ # # load given namespace from local .credentials.xml
146
+ # # merge all top to bottom
147
+ class Credentials < Hash
148
+
149
+ CRED_FILENAME = ".credentials.xml"
150
+
151
+ def find_file_upwards(aFilename,aStartPath=nil)
152
+ aStartPath ||= Dir.pwd
153
+ return nil if aFilename.nil? || aFilename.empty?
154
+ arrPath = aStartPath.split(File::SEPARATOR)
155
+ while arrPath.length > 0
156
+ path = File.join(arrPath.join(File::SEPARATOR),aFilename)
157
+ return path if File.exists?(path)
158
+ arrPath.pop
159
+ end
160
+ return nil
161
+ end
162
+
163
+ def get_all_credentials(aXmlRoot)
164
+ return nil unless aXmlRoot
165
+ result = {}
166
+ REXML::XPath.each(aXmlRoot, '/Credentials/SimpleItems') do |si|
167
+ ns = si.attributes['Namespace']
168
+ values = XmlUtils.read_simple_items(si)
169
+ result[ns.to_sym] = values.symbolize_keys if ns && values
170
+ end
171
+ return result
172
+ end
173
+
174
+ #XmlUtils.read_simple_items(@xmlRoot,'/Yore/SimpleItems')
175
+ def get_user_credentials
176
+ return get_all_credentials(XmlUtils.get_file_root(File.join(HOME_PATH,CRED_FILENAME)))
177
+ end
178
+
179
+ def get_local_credentials(aSource=nil)
180
+ aSource ||= Dir.pwd
181
+ # assume source is a directory path, but other types could be supported later
182
+ return nil unless file=find_file_upwards(CRED_FILENAME,aSource)
183
+ return get_all_credentials(XmlUtils.get_file_root(file))
184
+ end
185
+
186
+ def initialize(aNamespace=nil,aSource=nil)
187
+ #HOME_PATH can be preset by tests eg. ::Credentials.const_set('HOME_PATH',@user_dir)
188
+ Credentials.const_set("HOME_PATH", ENV['HOME']) unless Credentials.const_defined? "HOME_PATH"
189
+ arrCredentials = []
190
+ user_credentials = get_user_credentials()
191
+ local_credentials = get_local_credentials(aSource)
192
+ arrCredentials << user_credentials[:global] if user_credentials
193
+ arrCredentials << local_credentials[:global] if local_credentials
194
+ arrCredentials << user_credentials[aNamespace.to_sym] if aNamespace && user_credentials
195
+ arrCredentials << local_credentials[aNamespace.to_sym] if aNamespace && local_credentials
196
+ arrCredentials.compact!
197
+ arrCredentials.each do |c|
198
+ self.merge!(c)
199
+ end
200
+ end
201
+ end
202
+
@@ -0,0 +1,89 @@
1
+ require 'rubygems'
2
+ gem 'RequirePaths'; require 'require_paths'
3
+ require_paths '..'
4
+ require 'ihl_ruby/shell_extras'
5
+
6
+ module DatabaseUtils
7
+ def self.execute_sql_file(filename,aUser=nil,aPassword=nil)
8
+ conf = ActiveRecord::Base.configurations[RAILS_ENV]
9
+ pw = aPassword || conf['password'].to_s || ''
10
+ user = aUser || conf['username'].to_s || ''
11
+ cmd_line = "mysql -h #{conf['host']} -D #{conf['database']} #{user.empty? ? '' : '-u '+user} #{pw.empty? ? '' : '-p'+pw} <#{filename}"
12
+ if !system(cmd_line)
13
+ raise Exception, "Error executing "+cmd_line
14
+ end
15
+ end
16
+
17
+ ## http://www.cyberciti.biz/faq/how-do-i-empty-mysql-database/
18
+ #
19
+ #
20
+ ## drop all tables :
21
+ ## mysqldump -uusername -ppassword -hhost \
22
+ ##--add-drop-table --no-data database | grep ^DROP | \
23
+ ##mysql -uusername -ppassword -hhost database
24
+ #
25
+
26
+ def self.database_exists(aDbDetails,aDatabase=nil)
27
+ aDbDetails[:database] = aDatabase if aDatabase
28
+ return false if !aDbDetails[:database]
29
+ response = POpen4::shell("mysql -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} -e 'use #{aDbDetails[:database]}'") do |r|
30
+ if r[:stderr] && r[:stderr].index("ERROR 1049 ")==0 # Unknown database
31
+ r[:exitcode] = 0
32
+ return false
33
+ end
34
+ end
35
+ return (response && response[:exitcode]==0)
36
+ end
37
+
38
+ def self.clear_database(aDbDetails)
39
+ response = POpen4::shell("mysqldump -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} --add-drop-table --no-data #{aDbDetails[:database]} | grep ^DROP | mysql -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} #{aDbDetails[:database]}")
40
+ end
41
+
42
+ def self.create_database(aDbDetails,aDatabase=nil)
43
+ aDbDetails[:database] = aDatabase if aDatabase
44
+ return false if !aDbDetails[:database]
45
+ response = POpen4::shell("mysqladmin -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} create #{aDbDetails[:database]}")
46
+ end
47
+
48
+ def self.ensure_empty_database(aDbDetails,aDatabase=nil)
49
+ aDbDetails[:database] = aDatabase if aDatabase
50
+ if database_exists(aDbDetails)
51
+ clear_database(aDbDetails)
52
+ else
53
+ create_database(aDbDetails)
54
+ end
55
+ end
56
+
57
+ def self.load_database(aDbDetails,aSqlFile)
58
+ ensure_empty_database(aDbDetails)
59
+ response = POpen4::shell("mysql -u #{aDbDetails[:username]} -p#{aDbDetails[:password]} #{aDbDetails[:database]} < #{aSqlFile}")
60
+ end
61
+
62
+ def self.save_database(aDbDetails,aSqlFile)
63
+ response = POpen4::shell("mysqldump --user=#{aDbDetails[:username]} --password=#{aDbDetails[:password]} --skip-extended-insert #{aDbDetails[:database]} > #{aSqlFile}")
64
+ end
65
+
66
+ #
67
+ ## eg. rake metas:spree:data:load from=/tmp/spree_data.tgz to=mysql:fresco_server_d:root:password
68
+ #desc 'load spree data from a file'
69
+ #task :load do
70
+ # from = ENV['from']
71
+ # to=ENV['to']
72
+ # db_server,db,user,password = to.split(':')
73
+ # tmpdir = make_temp_dir('metas')
74
+ # cmd = "tar -xvzf #{from} -C #{tmpdir}"
75
+ # puts CapUtilsClass.shell(cmd)
76
+ #
77
+ # ensure_empty_database(db_server,db,user,password)
78
+ #
79
+ # puts CapUtilsClass.shell("mysql -u #{user} -p#{password} #{db} < #{File.join(tmpdir,'db/dumps/db.sql')}")
80
+ # FileUtils.mkdir_p('public/assets')
81
+ # puts CapUtilsClass.shell("cp -rf #{File.join(tmpdir,'public/assets/products')} public/assets/products")
82
+ #end
83
+
84
+
85
+
86
+
87
+ end
88
+
89
+
@@ -224,6 +224,12 @@ module HashUtils
224
224
  end
225
225
  return result
226
226
  end
227
+
228
+ def symbolize_keys
229
+ result = {}
230
+ self.each { |k,v| k.is_a?(String) ? result[k.to_sym] = v : result[k] = v }
231
+ return result
232
+ end
227
233
 
228
234
  end
229
235
 
@@ -138,7 +138,7 @@ module MiscUtils
138
138
  def self.path_debase(aPath,aBase)
139
139
  aBase = MiscUtils::append_slash(aBase)
140
140
  aPath = MiscUtils::remove_slash(aPath) unless aPath=='/'
141
- aPath[aBase.length,aPath.length-aBase.length]
141
+ aPath[0,aBase.length]==aBase ? aPath[aBase.length,aPath.length-aBase.length] : aPath
142
142
  end
143
143
 
144
144
  def self.path_rebase(aPath,aOldBase,aNewBase)
@@ -230,6 +230,7 @@ module MiscUtils
230
230
  abssrcpath = aRootPath = aPath
231
231
  aPath = nil
232
232
  end
233
+ return aArray if !File.exists?(abssrcpath)
233
234
  #abssrcpath is real path to query
234
235
  #aRootPath is highest level path
235
236
  #aPath is current path relative to aRootPath
@@ -20,6 +20,7 @@ module XmlUtils
20
20
  end
21
21
 
22
22
  def self.get_file_root(aFilename)
23
+ return nil unless File.exists?(aFilename)
23
24
  get_xml_root(MiscUtils.string_from_file(aFilename))
24
25
  end
25
26
 
@@ -35,8 +36,10 @@ module XmlUtils
35
36
  return val.nil? ? default : val.to_s
36
37
  end
37
38
 
38
- def self.peek_node_value(node,xpath,default=nil)
39
- peek_node(node,xpath+'/text()',default)
39
+ def self.peek_node_value(aNode,aXPath,aDefault=nil)
40
+ node = single_node(aNode,aXPath)
41
+ return node.to_s if node.is_a?(REXML::Attribute)
42
+ return node.nil? ? aDefault : node.text()
40
43
  end
41
44
 
42
45
  # convert <root><a>one</a><b>two</b></root> to {'a' => 'one', 'b' => 'two'}
@@ -66,6 +69,15 @@ module XmlUtils
66
69
  result += aText ? " >#{aText}</#{aName}>" : " />"
67
70
  end
68
71
 
72
+ def self.add_xml_from_string(aString,aNode)
73
+ return nil unless xdoc = REXML::Document.new('<?xml version="1.0" encoding="UTF-8"?>'+aString)
74
+ r = xdoc.root
75
+ r.remove()
76
+ r.parent = nil
77
+ aNode.add_element(r)
78
+ return r
79
+ end
80
+
69
81
  def self.hash_to_xml(aHash,aRootName,aDocHeader=true)
70
82
  xdoc = REXML::Document.new(BASIC_HEADER)
71
83
  root = xdoc.add_element(aRootName)
@@ -75,9 +87,10 @@ module XmlUtils
75
87
  return xdoc
76
88
  end
77
89
 
78
- def self.read_simple_items(aRoot,aParentXPath)
90
+ def self.read_simple_items(aRoot,aParentXPath=nil)
79
91
  result = {}
80
- REXML::XPath.each(aRoot, aParentXPath+'/Item') do |item|
92
+ xp = aParentXPath ? File.join(aParentXPath,'Item') : 'Item'
93
+ REXML::XPath.each(aRoot, xp) do |item|
81
94
  result[item.attribute('Name').to_s] = item.text
82
95
  end
83
96
  return result
@@ -91,8 +104,9 @@ module XmlUtils
91
104
  return result
92
105
  end
93
106
 
107
+ # reads the simple items format given either a filename or xml node
94
108
  def self.read_config_values(aXmlConfig)
95
- xmlRoot = get_file_root(aXmlConfig)
109
+ xmlRoot = aXmlConfig.is_a?(REXML::Element) ? aXmlConfig : get_file_root(aXmlConfig)
96
110
  return read_simple_items(xmlRoot,'/Config/SimpleItems')
97
111
  end
98
112
 
data/lib/yore.orig.rb CHANGED
@@ -2,5 +2,5 @@ $:.unshift(File.dirname(__FILE__)) unless
2
2
  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
3
3
 
4
4
  module Yore
5
- VERSION = '0.0.3'
5
+ VERSION = '0.0.4'
6
6
  end
@@ -13,12 +13,11 @@ require 'ihl_ruby/xml_utils'
13
13
  require 'ihl_ruby/extend_base_classes'
14
14
  require 'ihl_ruby/shell_extras'
15
15
  require 'ihl_ruby/config'
16
+ require 'ihl_ruby/database_utils'
16
17
 
17
- THIS_FILE = __FILE__
18
+ THIS_FILE = File.expand_path(__FILE__)
18
19
  THIS_DIR = File.dirname(THIS_FILE)
19
20
 
20
-
21
-
22
21
  module YoreCore
23
22
 
24
23
  class KeepDaily
@@ -79,11 +78,11 @@ module YoreCore
79
78
  end
80
79
  end
81
80
 
82
-
83
-
84
81
  class Yore
85
82
 
86
83
  DEFAULT_CONFIG = {
84
+ :kind => '',
85
+ :basepath => '',
87
86
  :keep_daily => 14,
88
87
  :keep_weekly => 12,
89
88
  :keep_monthly => 12,
@@ -104,18 +103,17 @@ module YoreCore
104
103
  :mail_to => '',
105
104
  :mail_to_alias => '',
106
105
  :mail_auth => :plain,
107
- :mysqldump => 'mysqldump'
106
+ :mysqldump => 'mysqldump',
107
+ :RAILS_ENV => ''
108
108
  }
109
109
 
110
110
  attr_reader :config
111
111
  attr_reader :logger
112
112
  attr_reader :reporter
113
113
  attr_reader :keepers
114
- attr_reader :basepath
115
114
 
116
115
  def initialize(aConfig=nil)
117
116
  DEFAULT_CONFIG[:email_report] = false # fixes some bug where this was nil
118
- @config = ConfigClass.new(DEFAULT_CONFIG,aConfig)
119
117
 
120
118
  cons = ConsoleLogger.new()
121
119
  cons.level = Logger::Severity.const_get(config[:log_level]) rescue Logger::Severity::INFO
@@ -126,7 +124,6 @@ module YoreCore
126
124
  @reporter.level = cons.level
127
125
 
128
126
  @logger = MultiLogger.new([cons,@reporter])
129
- #require 'ruby-debug'; debugger
130
127
  @logger.info "Yore file and database backup tool for Amazon S3 "
131
128
  @logger.info "(c) 2009 Buzzware Solutions (www.buzzware.com.au)"
132
129
  @logger.info "-------------------------------------------------"
@@ -134,44 +131,126 @@ module YoreCore
134
131
 
135
132
  @logger.info "report file: #{report_file}"
136
133
 
137
- configure(@config)
134
+ configure(aConfig)
138
135
  end
136
+
137
+ #aOptions may require {:basepath => File.dirname(File.expand_path(job))}
138
+ def self.launch(aConfigXml,aCmdOptions=nil,aOptions=nil)
139
+ result = Yore.new()
140
+ result.configure(aConfigXml,aCmdOptions,aOptions)
141
+ return result
142
+ end
143
+
144
+ def create_empty_config_xml()
145
+ s = <<-EOS
146
+ <?xml version="1.0" encoding="UTF-8"?>
147
+ <?xml version="1.0" encoding="UTF-8"?>
148
+ <Yore>
149
+ <SimpleItems>
150
+ </SimpleItems>
151
+ <Sources>
152
+ </Sources>
153
+ </Yore>
154
+ EOS
155
+ xdoc = REXML::Document.new(s)
156
+ return xdoc.root
157
+ end
158
+
159
+ def get_rails_db_details(aRailsPath,aRailsEnv)
160
+ return nil unless aRailsPath && aRailsEnv && aRailsEnv!=''
161
+ return nil unless dbyml = (YAML::load(File.open(File.expand_path('config/database.yml',aRailsPath))) rescue nil)
162
+ return dbyml[aRailsEnv] && dbyml[aRailsEnv].symbolize_keys
163
+ end
164
+
165
+ def expand_app_option(kind=nil)
166
+ kind = config[:kind] unless kind && !kind.empty?
167
+ return nil unless kind && !kind.empty?
168
+ config.xmlRoot = create_empty_config_xml() if !config.xmlRoot
169
+ case kind
170
+ when 'spree'
171
+ # add file source
172
+ xmlSources = XmlUtils.single_node(config.xmlRoot,'/Yore/Sources')
173
+ if xmlSources
174
+ strSource = <<-EOS
175
+ <Source Type="File">
176
+ <IncludePath>public/assets/products</IncludePath>
177
+ </Source>
178
+ EOS
179
+ XmlUtils.add_xml_from_string(strSource,xmlSources)
180
+ end
181
+ expand_app_option('rails') # do again
182
+ when 'rails'
183
+ # * add db source from database.yml
184
+ # load database from config[:basepath],'config/database.yml'
185
+ #if (dbyml = YAML::load(File.open(File.expand_path('config/database.yml',config[:basepath]))) rescue nil)
186
+ # if env = (config[:RAILS_ENV] && config[:RAILS_ENV]!='' && config[:RAILS_ENV])
187
+ # if (db_details = dbyml[env]) &&
188
+ db_details = get_rails_db_details(config[:basepath],config[:RAILS_ENV])
189
+ xmlSources = XmlUtils.single_node(config.xmlRoot,'/Yore/Sources')
190
+ if db_details && xmlSources
191
+ strSource = <<-EOS
192
+ <Source Type="MySql" >
193
+ <Database Name="#{db_details[:database]}" Host="#{db_details[:host]}" User="#{db_details[:username]}" Password="#{db_details[:password]}">
194
+ <ArchiveFile>rails_app.sql</ArchiveFile>
195
+ </Database>
196
+ </Source>
197
+ EOS
198
+ XmlUtils.add_xml_from_string(strSource,xmlSources)
199
+ end
200
+ end
201
+ end
139
202
 
140
203
  # read the config however its given and return a hash with values in their correct type, and either valid or nil
141
204
  # keys must be :symbols for aOptions. aConfig and aCmdOptions can be strings
142
205
  def configure(aConfig,aCmdOptions = nil,aOptions = nil)
143
- config_h = nil
144
- case aConfig
145
- when Hash,::ConfigClass then config_h = aConfig
146
- when REXML::Element then config_h = XmlUtils.read_simple_items(aConfig,'/Yore/SimpleItems')
147
- else
148
- raise StandardError.new('unsupported type')
149
- end
150
- config_i = {}
151
- config_h.each{|n,v| config_i[n.to_sym] = v} if config_h
152
- aCmdOptions.each{|k,v| config_i[k.to_sym] = v} if aCmdOptions
153
- config_i.merge!(aOptions) if aOptions
154
- config.read(config_i)
206
+ config_to_read = {}
207
+ if aConfig.is_a?(String)
208
+ aConfig = File.expand_path(aConfig)
209
+ logger.info "Job file: #{aConfig}"
210
+ op = {:basepath => File.dirname(aConfig)}
211
+ xml = XmlUtils.get_file_root(aConfig)
212
+ return configure(xml,aCmdOptions,op)
213
+ end
214
+
215
+ if @config
216
+ config_as_hash = nil
217
+ case aConfig
218
+ when nil then ; # do nothing
219
+ when Hash,::ConfigClass then config_as_hash = aConfig
220
+ when REXML::Element then
221
+ config_as_hash = XmlUtils.read_simple_items(aConfig,'/Yore/SimpleItems')
222
+ config.xmlRoot = aConfig # overwriting previous! perhaps should merge
223
+ else
224
+ raise StandardError.new('unsupported type')
225
+ end
226
+ config_as_hash.each{|n,v| config_to_read[n.to_sym] = v} if config_as_hash # merge given new values
227
+ else
228
+ @config = ConfigXmlClass.new(DEFAULT_CONFIG,aConfig)
229
+ end
230
+ aCmdOptions.each{|k,v| config_to_read[k.to_sym] = v} if aCmdOptions # merge command options
231
+ config_to_read.merge!(aOptions) if aOptions # merge options
232
+ config.read(config_to_read)
233
+ config[:basepath] = File.expand_path(Dir.pwd) if !config[:basepath] || config[:basepath]==''
234
+
235
+ expand_app_option()
155
236
 
156
237
  @keepers = Array.new
157
238
  @keepers << KeepDaily.new(config[:keep_daily])
158
239
  @keepers << KeepWeekly.new(config[:keep_weekly])
159
240
  @keepers << KeepMonthly.new(config[:keep_monthly])
160
-
161
- @basepath = config_h[:basepath]
162
241
  end
163
242
 
164
- def do_action(aAction,aArgs)
243
+ def do_action(aAction,aArgs,aCmdOptions)
165
244
  logger.info "Executing command: #{aAction} ...\n"
166
245
  begin
167
- send(aAction,aArgs)
246
+ send(aAction,aArgs,aCmdOptions)
168
247
  rescue Exception => e
169
- logger.warn "#{e.class.to_s}: during #{aAction}(#{aArgs.inspect}): #{e.message}"
248
+ logger.info {e.backtrace.join("\n")}
249
+ logger.warn "#{e.class.to_s}: during #{aAction.to_s}(#{(aArgs && aArgs.inspect).to_s}): #{e.message.to_s}"
170
250
  end
171
251
  end
172
252
 
173
253
  def shell(aCommandline,&aBlock)
174
- #require 'ruby-debug'; debugger
175
254
  logger.debug "To shell: " + aCommandline
176
255
  result = block_given? ? POpen4::shell(aCommandline,nil,nil,&aBlock) : POpen4::shell(aCommandline)
177
256
  logger.debug "From shell: '#{result.inspect}'"
@@ -192,6 +271,11 @@ module YoreCore
192
271
  def get_report
193
272
  MiscUtils::string_from_file(@reporter.logdev.filename)
194
273
  end
274
+
275
+ def temp_path
276
+ @temp_path = MiscUtils.make_temp_dir('yore') unless @temp_path
277
+ return @temp_path
278
+ end
195
279
 
196
280
  def self.filemap_from_filelist(aFiles)
197
281
  ancestor_path = MiscUtils.file_list_ancestor(aFiles)
@@ -206,45 +290,52 @@ module YoreCore
206
290
 
207
291
  end
208
292
 
209
- #def self.nice_format(aNumber)
210
- # if aNumber >= 100
211
- # sprintf('%.0f', aNumber)
212
- # else
213
- # sprintf('%.3f', aNumber).sub(/\.0{1,3}$/, '')
214
- # end
215
- #end
216
293
 
217
294
  # By default, GNU tar suppresses a leading slash on absolute pathnames while creating or reading a tar archive. (You can suppress this with the -p option.)
218
295
  # tar : http://my.safaribooksonline.com/0596102461/I_0596102461_CHP_3_SECT_9#snippet
219
296
 
220
297
  # get files from wherever they are into a single file
221
- def collect(aSourceFiles,aDestFile,aParentDir=nil)
298
+ def compress(aSourceFiles,aDestFile,aParentDir=nil)
222
299
  logger.info "Collecting files ..."
223
- logger.info aSourceFiles.join("\n")
224
- filelist = filemap = nil
225
- if aSourceFiles.is_a?(Hash)
226
- filelist = aSourceFiles.keys
227
- filemap = aSourceFiles
228
- else # assume array
229
- filelist = aSourceFiles
230
- filemap = Yore.filemap_from_filelist(aSourceFiles)
231
- end
232
- aParentDir ||= MiscUtils.file_list_ancestor(filelist)
233
- listfile = File.join(aParentDir,'.contents')
300
+ #logger.info aSourceFiles.join("\n")
301
+ #filelist = filemap = nil
302
+ #if aSourceFiles.is_a?(Hash)
303
+ # filelist = aSourceFiles.keys
304
+ # filemap = aSourceFiles
305
+ #else # assume array
306
+ # filelist = aSourceFiles
307
+ # filemap = Yore.filemap_from_filelist(aSourceFiles)
308
+ #end
309
+ #aParentDir ||= MiscUtils.file_list_ancestor(filelist)
310
+ listfile = MiscUtils.temp_file
234
311
  MiscUtils.string_to_file(
235
- filelist.sort.map{|p| MiscUtils.path_debase(p, aParentDir)}.join("\n"),
312
+ aSourceFiles.join("\n"), #filelist.sort.map{|p| MiscUtils.path_debase(p, aParentDir)}.join("\n"),
236
313
  listfile
237
314
  )
238
315
  tarfile = MiscUtils.file_change_ext(aDestFile, 'tar')
239
316
 
240
- shell("tar cv --directory=#{aParentDir} --file=#{tarfile} --files-from=#{listfile}")
241
- shell("tar --append --directory=#{aParentDir} --file=#{tarfile} .contents")
317
+ shell("tar cv #{aParentDir ? '--directory='+aParentDir.to_s : ''} --file=#{tarfile} --files-from=#{listfile}")
242
318
  logger.info "Compressing ..."
243
319
  tarfile_size = File.size(tarfile)
244
320
  shell("bzip2 #{tarfile}; mv #{tarfile}.bz2 #{aDestFile}")
245
321
  logger.info "Compressed #{'%.1f' % (tarfile_size*1.0/2**10)} KB to #{'%.1f' % (File.size(aDestFile)*1.0/2**10)} KB"
246
322
  end
247
323
 
324
+ def uncompress(aArchive,aDestination=nil,aArchiveContent=nil)
325
+ #tarfile = File.expand_path(MiscUtils.file_change_ext(File.basename(aArchive),'tar'),temp_dir)
326
+ #shell("bunzip2 #{tarfile}; mv #{tarfile}.bz2 #{aDestFile}")
327
+ #
328
+ #shell("tar cv #{aParentDir ? '--directory='+aParentDir.to_s : ''} --file=#{tarfile} --files-from=#{listfile}")
329
+ #logger.info "Compressing ..."
330
+ #tarfile_size = File.size(tarfile)
331
+ #shell("bzip2 #{tarfile}; mv #{tarfile}.bz2 #{aDestFile}")
332
+ #logger.info "Compressed #{'%.1f' % (tarfile_size*1.0/2**10)} KB to #{'%.1f' % (File.size(aDestFile)*1.0/2**10)} KB"
333
+ aDestination ||= MiscUtils.make_temp_dir('uncompress')
334
+ FileUtils.mkdir_p(aDestination)
335
+ shell("tar xvf #{aArchive} #{aArchiveContent.to_s} --directory=#{aDestination} --bzip2")
336
+ end
337
+
338
+
248
339
  def pack(aFileIn,aFileOut)
249
340
  logger.info "Encrypting ..."
250
341
  shell "openssl enc -aes-256-cbc -K #{config[:crypto_key]} -iv #{config[:crypto_iv]} -in #{aFileIn} -out #{aFileOut}"
@@ -262,7 +353,7 @@ module YoreCore
262
353
 
263
354
  # uploads the given file to the current bucket as its basename
264
355
  def upload(aFile)
265
- ensure_bucket()
356
+ #ensure_bucket()
266
357
  logger.info "Uploading #{File.basename(aFile)} to S3 bucket #{config[:bucket]} ..."
267
358
  s3shell "s3cmd put #{config[:bucket]}:#{File.basename(aFile)} #{aFile}"
268
359
  end
@@ -291,36 +382,6 @@ module YoreCore
291
382
  return Time.from_date_numeric(date)
292
383
  end
293
384
 
294
- #def set_file_name(aFile,aNewName)#
295
- #
296
- #end
297
-
298
- def backup_process(aSourceFiles,aTimeNow=Time.now,aTempDir=nil)
299
- aTempDir ||= MiscUtils.make_temp_dir('yore_')
300
- temp_file = File.expand_path('backup.tar',aTempDir)
301
- collect(aSourceFiles,temp_file)
302
- backup_file = File.expand_path(encode_file_name(aTimeNow),aTempDir)
303
- pack(temp_file,backup_file)
304
- upload(backup_file)
305
- end
306
-
307
- # aDb : Hash containing :db_host,db_user,db_password,db_name,
308
- def db_to_file(aDb,aFile)
309
- logger.info "Dumping database #{aDb[:db_name]} ..."
310
- shell "#{config[:mysqldump]} --host=#{aDb[:db_host]} --user=#{aDb[:db_user]} --password=#{aDb[:db_password]} --databases --skip-extended-insert --add-drop-database #{aDb[:db_name]} > #{aFile}"
311
- end
312
-
313
- def file_to_db(aFile,aDatabase)
314
- #run "mysql --user=root --password=prot123ection </root/joomla_db_snapshot/joomla_db.sql"
315
- end
316
-
317
- #
318
- #
319
- # COMMANDLINE COMMANDS
320
- #
321
- #
322
-
323
-
324
385
 
325
386
  def clean
326
387
 
@@ -357,59 +418,142 @@ module YoreCore
357
418
 
358
419
  def self.database_from_xml(aDatabaseNode)
359
420
  return {
360
- :db_host => aDatabaseNode.attributes['Host'],
361
- :db_user => aDatabaseNode.attributes['User'],
362
- :db_password => aDatabaseNode.attributes['Password'],
363
- :db_name => aDatabaseNode.attributes['Name'],
364
- :file => XmlUtils::peek_node_value(aDatabaseNode, "ToFile")
421
+ :host => aDatabaseNode.attributes['Host'],
422
+ :username => aDatabaseNode.attributes['User'],
423
+ :password => aDatabaseNode.attributes['Password'],
424
+ :database => aDatabaseNode.attributes['Name'],
425
+ :file => XmlUtils::peek_node_value(aDatabaseNode, "ToFile"),
426
+ :archive_file => XmlUtils::peek_node_value(aDatabaseNode, "ArchiveFile")
365
427
  }
366
428
  end
367
-
368
- def backup(aJobFiles)
369
- return unless job = aJobFiles.is_a?(Array) ? aJobFiles.first : aJobFiles # just use first job
370
-
371
- xmlRoot = XmlUtils.get_file_root(job)
372
-
429
+
430
+
431
+ def collect_file_list(aSourcesXml,aTempFolder)
373
432
  filelist = []
374
433
  sourceFound = false
375
434
 
376
- REXML::XPath.each(xmlRoot, '/Yore/Sources/Source') do |xmlSource|
377
- case xmlSource.attributes['Type']
378
- when 'File' then
379
- REXML::XPath.each(xmlSource, 'IncludePath') do |xmlPath|
380
- filelist += MiscUtils::recursive_file_list(MiscUtils::path_combine(config[:basepath],xmlPath.text))
381
- sourceFound = true
382
- end
383
- when 'MySql' then
384
- #<Source Type="MySql" >
385
- # <Database Host="" Name="" User="" Password="">
386
- # <ToFile>~/dbdump.sql</ToFile>
387
- # </Database>
388
- #</Source>
389
- REXML::XPath.each(xmlSource, 'Database') do |xmlDb|
390
- args = Yore::database_from_xml(xmlDb)
391
- file = args.delete(:file)
392
- unless args[:db_host] && args[:db_user] && args[:db_password] && args[:db_name] && file
393
- raise StandardError.new("Invalid or missing parameter")
435
+ if aSourcesXml
436
+ REXML::XPath.each(aSourcesXml,'Source') do |xmlSource|
437
+ case xmlSource.attributes['Type']
438
+ when 'File' then
439
+ # BasePath tag provides base path for IncludePaths to be relative to. Also indicates root folder for archive
440
+ bp = MiscUtils.path_combine(config[:basepath],XmlUtils::peek_node_value(xmlSource, "@BasePath"))
441
+ filelist << '-C'+bp
442
+ REXML::XPath.each(xmlSource, 'IncludePath') do |xmlPath|
443
+ files = MiscUtils::recursive_file_list(MiscUtils::path_combine(bp,xmlPath.text))
444
+ files.map!{|f| MiscUtils.path_debase(f,bp)}
445
+ filelist += files
446
+ sourceFound = true
394
447
  end
395
- db_to_file(args,file)
396
- filelist << file
397
- sourceFound = true
398
- end
448
+ when 'MySql' then
449
+ #<Source Type="MySql" >
450
+ # <Database Host="" Name="" User="" Password="">
451
+ # <ToFile>~/dbdump.sql</ToFile>
452
+ # </Database>
453
+ #</Source>
454
+ REXML::XPath.each(xmlSource, 'Database') do |xmlDb|
455
+ args = Yore::database_from_xml(xmlDb)
456
+ file = args.delete(:file) #legacy, absolute path
457
+ arc_file = args.delete(:archive_file) #path in archive
458
+ unless args[:host] && args[:username] && args[:password] && args[:database] && (file||arc_file)
459
+ raise StandardError.new("Invalid or missing parameter")
460
+ end
461
+ if arc_file
462
+ arc_file = MiscUtils.path_debase(arc_file,'/')
463
+ sql_file = File.expand_path(arc_file,aTempFolder)
464
+ FileUtils.mkdir_p(File.dirname(sql_file)) # create folders as necessry
465
+ DatabaseUtils::save_database(args,sql_file) #db_to_file(args,sql_file)
466
+ filelist << '-C'+aTempFolder
467
+ filelist << arc_file
468
+ sourceFound = true
469
+ else
470
+ DatabaseUtils::save_database(args,sql_file)
471
+ filelist << file
472
+ sourceFound = true
473
+ end
474
+ end
475
+ end
399
476
  end
400
- end
401
-
477
+ end
402
478
  raise StandardError.new("Backup source found but file list empty") if sourceFound && filelist.empty?
479
+ return filelist
480
+ end
481
+
482
+ def rails_tmp_path
483
+ return @rails_tmp_path if @rails_tmp_path
484
+ @rails_tmp_path = File.join(config[:basepath],'tmp/yore',Time.now.strftime('%Y%m%d-%H%M%S'))
485
+ end
486
+
487
+ def self.move_folder(aPath1,aPath2)
488
+ path2Parent = MiscUtils.path_parent(aPath2)
489
+ FileUtils.mkdir_p(path2Parent)
490
+ FileUtils.mv(aPath1, path2Parent, :force => true)
491
+ end
492
+
493
+ def self.copy_folder(aPath1,aPath2)
494
+ path2Parent = MiscUtils.path_parent(aPath2)
495
+ FileUtils.mkdir_p(path2Parent)
496
+ FileUtils.cp_r(aPath1, path2Parent)
497
+ end
498
+
499
+ def save_internal(aFilename)
500
+ FileUtils.mkdir_p(files_path = File.join(temp_path,'files'))
501
+ filelist = collect_file_list(XmlUtils.single_node(config.xmlRoot,'/Yore/Sources'),files_path)
502
+ compress(filelist,aFilename)
503
+ end
403
504
 
404
- filelist.uniq!
405
- filelist.sort!
505
+ #
506
+ # ACTIONS
507
+ #
406
508
 
407
- tempdir = MiscUtils.make_temp_dir('yore')
408
- time = Time.now
509
+ def save(aArgs,aCmdOptions=nil)
510
+ fnArchive = aArgs.is_a?(Array) ? aArgs.first : aArgs #only supported argument
511
+ configure(nil,aCmdOptions)
512
+ save_internal(fnArchive)
513
+ end
409
514
 
410
- backup_process(filelist,time,tempdir)
515
+ def backup(aArgs,aCmdOptions=nil) # was aJobFiles
516
+ unless aCmdOptions && aCmdOptions[:config] # assume already configured if config option specified, but back supports first arg being config file
517
+ job = aArgs.first
518
+ configure(job,aCmdOptions || {})
519
+ end
520
+ temp_file = File.expand_path('backup.tar',temp_path)
521
+ save_internal(temp_file)
522
+ backup_file = File.expand_path(encode_file_name(),temp_path)
523
+ pack(temp_file,backup_file)
524
+ upload(backup_file)
411
525
  #clean
412
526
  end
527
+
528
+ def load(aArgs,aCmdOptions=nil)
529
+ fnArchive = aArgs.is_a?(Array) ? aArgs.first : aArgs #only supported argument
530
+ configure(nil,aCmdOptions)
531
+
532
+ FileUtils.mkdir_p(archive_path = File.join(temp_path,'archive'))
533
+ uncompress(fnArchive,archive_path)
534
+ xmlSources = XmlUtils.single_node(config.xmlRoot,'/Yore/Sources')
535
+ REXML::XPath.each(xmlSources,'Source') do |xmlSource|
536
+ case xmlSource.attributes['Type']
537
+ when 'File' then
538
+ #<Source Type="File">
539
+ # <IncludePath>public/assets/products</IncludePath>
540
+ #</Source>
541
+ REXML::XPath.each(xmlSource,'IncludePath') do |xmlIncludePath|
542
+ pathArchive = xmlIncludePath.text()
543
+ pathUncompressed = File.join(archive_path,pathArchive)
544
+ pathTmp = File.join(rails_tmp_path,pathArchive)
545
+ pathDest = File.join(config[:basepath],pathArchive)
546
+ # move basepath/relativepath to tmp/yore/090807-010203/relativepath
547
+ Yore::move_folder(pathDest,pathTmp) if File.exists?(pathDest)
548
+ # get <IncludeFiles> and copy to basepath/relativepath
549
+ Yore::copy_folder(pathUncompressed,pathDest) if File.exists?(pathUncompressed)
550
+ end
551
+ when 'MySql' then
552
+ db_details = Yore::database_from_xml(XmlUtils.single_node(xmlSource,'Database'))
553
+ DatabaseUtils.load_database(db_details,File.join(archive_path,db_details[:archive_file]))
554
+ end
555
+ end
556
+ end
413
557
 
414
558
  def test_email(*aDb)
415
559
  args = {
@@ -430,26 +574,6 @@ module YoreCore
430
574
  MiscUtils::send_email(args)
431
575
  end
432
576
 
433
- def db_dump(aArgs)
434
- return nil unless aArgs
435
- return nil unless job = aArgs[0]
436
-
437
- xmlRoot = XmlUtils.get_file_root(job)
438
- xmlDb = nil
439
- if db_name = aArgs[1]
440
- xmlDb = XmlUtils::single_node(xmlRoot,"/Yore/Sources/Source[@Type='MySql']/Database[@Name='#{db_name}']")
441
- else
442
- xmlDb = XmlUtils::single_node(xmlRoot,"/Yore/Sources/Source[@Type='MySql']/Database")
443
- end
444
- raise StandardError.new("No database") unless xmlDb
445
- args = Yore.database_from_xml(xmlDb)
446
- file = args.delete(:file)
447
- unless args[:db_host] && args[:db_user] && args[:db_password] && args[:db_name] && file
448
- raise StandardError.new("Invalid or missing parameter")
449
- end
450
- db_to_file(args,file)
451
- end
452
-
453
577
  end
454
578
 
455
579
  end