yore 0.0.1

data/lib/yore/yore_core.rb

@@ -0,0 +1,491 @@
+require 'rubygems'
+gem 'RequirePaths'; require 'require_paths'
+require_paths '.','..'
+
+require 'fileutils'
+require 'net/smtp'
+
+require 'ihl_ruby/misc_utils'
+require 'ihl_ruby/string_utils'
+require 'ihl_ruby/xml_utils'
+require 'ihl_ruby/extend_base_classes'
+
+THIS_FILE = __FILE__
+THIS_DIR = File.dirname(THIS_FILE)
+
+module YoreCore
+
+  class KeepDaily
+
+      attr_reader :keep_age
+
+      def initialize(aKeepAge=14)
+        @keep_age = aKeepAge
+      end
+
+      def is?
+        true
+      end
+
+      def age(aDate)
+
+      end
+      def keep?(aDate)
+
+      end
+  end
+
+  class KeepWeekly
+
+      attr_reader :keep_age
+
+      def initialize(aKeepAge=14)
+        @keep_age = aKeepAge
+      end
+
+      def is?
+
+      end
+      def age(aDate)
+
+      end
+      def keep?(aDate)
+
+      end
+  end
+
+  class KeepMonthly
+
+      attr_reader :keep_age
+
+      def initialize(aKeepAge=14)
+        @keep_age = aKeepAge
+      end
+
+      def is?
+
+      end
+      def age(aDate)
+
+      end
+      def keep?(aDate)
+
+      end
+  end
+
+
+  class ConfigClass < Hash
+
+    def initialize(aDefaultHash=nil)
+      self.merge!(aDefaultHash) if aDefaultHash
+    end
+
+    def set_int(aKey,aValue)
+      case aValue
+        when String then    self[aKey] = aValue.to_integer(self[aKey]);
+        when Fixnum then    self[aKey] = aValue;
+        when Float then     self[aKey] = aValue.to_i;
+      end
+    end
+
+    def set_float(aKey,aValue)
+      case aValue
+        when String then    self[aKey] = aValue.to_float(self[aKey]);
+        when Fixnum then    self[aKey] = aValue.to_f;
+        when Float then     self[aKey] = aValue;
+      end
+    end
+
+    def set_boolean(aKey,aValue)
+      case aValue
+        when TrueClass,FalseClass then   self[aKey] = aValue;
+        when String then    self[aKey] = (['1','yes','y','true','on'].include?(aValue.downcase))
+      else
+        default        set_boolean(aKey,aValue.to_s)
+      end
+    end
+		
+    def set_symbol(aKey,aValue)
+      case aValue
+        when String then    self[aKey] = (aValue.to_sym rescue nil);
+        when Symbol then    self[aKey] = aValue;
+      end
+    end
+
+		def copy_item(aHash,aKey)
+			case self[aKey]
+				when NilClass then ;
+				when String then self[aKey] = aHash[aKey].to_s unless aHash[aKey].nil?
+				when Float then set_float(aKey,aHash[aKey]);
+				when Fixnum then set_int(aKey,aHash[aKey]);
+				when TrueClass, FalseClass then set_boolean(aKey,aHash[aKey]);
+				when Symbol then self[aKey] = (aHash[aKey].to_sym rescue nil)
+				else
+					raise Error.new('unsupported type')
+			end
+		end
+
+    def copy_strings(aHash,*aKeys)
+      aKeys.each do |k|
+        self[k] = aHash[k].to_s unless aHash[k].nil?
+      end
+    end
+
+    def copy_ints(*aDb)
+      aHash = aDb.shift
+      aKeys = aDb
+      aKeys.each do |k|
+        set_int(k,aHash[k])
+      end
+    end
+
+    def copy_floats(aHash,*aKeys)
+      aKeys.each do |k|
+        set_float(k,aHash[k])
+      end
+    end
+
+    def copy_booleans(aHash,*aKeys)
+      aKeys.each do |k|
+        set_boolean(k,aHash[k])
+      end
+    end
+
+    def to_hash
+      {}.merge(self)
+    end
+
+  end
+
+  class Yore
+
+    DEFAULT_CONFIG = {
+      :keep_daily => 14,
+      :keep_weekly => 12,
+      :keep_monthly => 12,
+      :crypto_iv => "3A63775C1E3F291B0925578165EB917E",
+      :crypto_key => "07692FC8656F04AE5518B80D38681E038A3C12050DF6CC97CEEC33D800D5E2FE",
+      :first_hour => 4,
+      :prefix => 'backup',
+      :log_level => 'DEBUG',
+      :bucket => '',
+			:email_report => false,
+      :mail_host => '',
+      :mail_port => 25,
+      :mail_helodomain => '',
+      :mail_user => '',
+      :mail_password => '',
+			:mail_from => '',
+			:mail_from_alias => '',
+			:mail_to => '',
+			:mail_to_alias => '',
+			:mail_auth => :plain,
+			:mysqldump => 'mysqldump'
+		}
+
+    attr_reader :config
+    attr_reader :logger
+    attr_reader :reporter
+    attr_reader :keepers
+    attr_reader :basepath
+
+    def initialize(aConfig=nil)
+      DEFAULT_CONFIG[:email_report] = true  # fixes some bug where this was nil
+      configure(aConfig || DEFAULT_CONFIG)
+    end
+
+    def self.clean_config(aConfig)
+      result = ConfigClass.new(DEFAULT_CONFIG)
+			DEFAULT_CONFIG.each do |k,v|
+        result.copy_item(aConfig,k) if aConfig[k]
+      end
+      #result.copy_ints(aConfig,:keep_daily,:keep_weekly,:keep_monthly)
+      #result.copy_strings(aConfig,:crypto_iv,:crypto_key,:prefix,:bucket,:log_level,:basepath)
+      result.to_hash
+    end
+
+    # read the config however its given and return a hash with values in their correct type, and either valid or nil
+    # keys must be :symbols for aOptions. aConfig and aCmdOptions can be strings
+    def configure(aConfig,aCmdOptions = nil,aOptions = nil)
+      config_h = nil
+      case aConfig
+        when Hash then config_h = aConfig
+        when REXML::Element then config_h = XmlUtils.read_simple_items(aConfig,'/Yore/SimpleItems')
+        else
+          raise Error.new('unsupported type')
+      end
+      config_i = {}
+      config_h.each{|n,v| config_i[n.to_sym] = v} if config_h
+      aCmdOptions.each{|k,v| config_i[k.to_sym] = v} if aCmdOptions
+      config_i.merge!(aOptions) if aOptions
+      @config = Yore.clean_config(config_i)
+      @keepers = Array.new
+      @keepers << KeepDaily.new(config[:keep_daily])
+      @keepers << KeepWeekly.new(config[:keep_weekly])
+      @keepers << KeepMonthly.new(config[:keep_monthly])
+
+      @log_file = MiscUtils::temp_file()
+
+      @logger = LogUtils::create_logger_from_config({
+          'destination' => 'STDOUT',
+          'level' => config[:log_level]
+#        'destination' => 'FILE',
+#        'filename' => @log_file,
+#        'level' => config[:log_level]
+      })
+      report_file = MiscUtils::temp_file
+      logger.info "report file: #{report_file}"
+      @reporter = LogUtils::create_reporter(report_file)
+      @basepath = config_h[:basepath]
+      #sources = aConfig
+      #
+      #@filelist =
+    end
+
+		def shell(aCommandline)
+			reporter.debug aCommandline
+			reporter.debug response = `#{aCommandline}`
+			response
+		end
+
+    def get_log
+      logger.close
+      # read in log and return
+    end
+
+    def get_report
+      MiscUtils::string_from_file(reporter.logdev.filename)
+    end
+
+    def self.filemap_from_filelist(aFiles)
+      ancestor_path = MiscUtils.file_list_ancestor(aFiles)
+      filemap = {}
+      aFiles.each do |fp|
+        filemap[fp] = MiscUtils.path_debase(fp,ancestor_path)
+      end
+      filemap
+    end
+
+    def keep_file?(aFile)
+
+    end
+
+    # By default, GNU tar suppresses a leading slash on absolute pathnames while creating or reading a tar archive. (You can suppress this with the -p option.)
+    # tar : http://my.safaribooksonline.com/0596102461/I_0596102461_CHP_3_SECT_9#snippet
+
+    # get files from wherever they are into a single file
+    def collect(aSourceFiles,aDestFile,aParentDir=nil)
+      filelist = filemap = nil
+      if aSourceFiles.is_a?(Hash)
+        filelist = aSourceFiles.keys
+        filemap = aSourceFiles
+      else  # assume array
+        filelist = aSourceFiles
+        filemap = Yore.filemap_from_filelist(aSourceFiles)
+      end
+      aParentDir ||= MiscUtils.file_list_ancestor(filelist)
+      listfile = File.join(aParentDir,'.contents')
+      MiscUtils.string_to_file(
+        filelist.sort.map{|p| MiscUtils.path_debase(p, aParentDir)}.join("\n"),
+        listfile
+      )
+      tarfile = MiscUtils.file_change_ext(aDestFile, 'tar')
+			shell("tar cv --directory=#{aParentDir} --file=#{tarfile} --files-from=#{listfile}")			
+      shell("tar --append --directory=#{aParentDir} --file=#{tarfile} .contents")
+      shell("bzip2 #{tarfile}; mv #{tarfile}.bz2 #{aDestFile}")
+    end
+
+    def pack(aFileIn,aFileOut)
+      shell "openssl enc -aes-256-cbc -K #{config[:crypto_key]} -iv #{config[:crypto_iv]} -in #{aFileIn} -out #{aFileOut}"
+    end
+
+    def unpack(aFileIn,aFileOut)
+       shell "openssl enc -d -aes-256-cbc -K #{config[:crypto_key]} -iv #{config[:crypto_iv]} -in #{aFileIn} -out #{aFileOut}"
+    end
+
+    # uploads the given file to the current bucket as its basename
+    def upload(aFile)
+      shell "s3cmd put #{config[:bucket]}:#{File.basename(aFile)} #{aFile}"
+    end
+
+    # downloads the given file from the current bucket as its basename
+    def download(aFile)
+      shell "s3cmd get #{config[:bucket]}:#{File.basename(aFile)} #{aFile}"
+    end
+
+    # calculate the date (with no time component) based on :day_begins_hour and the local time
+    def backup_date(aTime)
+      (aTime.localtime - (config[:first_hour]*3600)).date
+    end
+
+    # generates filename based on date and config
+    # config :
+    #   :first_hour
+    #   :
+    def encode_file_name(aTimeNow=Time.now)
+      "#{config[:prefix]}-#{backup_date(aTimeNow).date_numeric}.rew"
+    end
+
+    # return date based on filename
+    def decode_file_name(aFilename)
+      prefix,date,ext = aFilename.scan(/(.*?)\-(.*?)\.(.*)/).flatten
+      return Time.from_date_numeric(date)
+    end
+
+    #def set_file_name(aFile,aNewName)#
+    #
+    #end
+
+    def backup_process(aSourceFiles,aTimeNow=Time.now,aTempDir=nil)
+      aTempDir ||= MiscUtils.make_temp_dir('yore_')
+      temp_file = File.expand_path('backup.tar',aTempDir)
+      collect(aSourceFiles,temp_file)
+      backup_file = File.expand_path(encode_file_name(aTimeNow),aTempDir)
+      pack(temp_file,backup_file)
+      upload(backup_file)
+    end
+		
+		# aDb : Hash containing :db_host,db_user,db_password,db_name,		
+		def db_to_file(aDb,aFile)
+			shell "#{config[:mysqldump]} --host=#{aDb[:db_host]} --user=#{aDb[:db_user]} --password=#{aDb[:db_password]} --databases --skip-extended-insert --add-drop-database #{aDb[:db_name]} > #{aFile}"
+		end
+		
+		def file_to_db(aFile,aDatabase)
+			#run "mysql --user=root --password=prot123ection </root/joomla_db_snapshot/joomla_db.sql"
+		end
+    
+		#		
+    #
+    # 											COMMANDLINE COMMANDS
+		#
+		#
+
+
+
+    def clean
+      
+    end
+
+		# "/usr/bin/env" sets normal vars
+		# eg. 30 14 * * * /usr/bin/env ruby /Users/kip/svn/thewall/script/runner /Users/kip/svn/thewall/app/delete_old_posts.rb
+		# http://www.ameravant.com/posts/recurring-tasks-in-ruby-on-rails-using-runner-and-cron-jobs
+
+		# install gems
+		# make folder with correct folder structure
+		# copy in files
+		# add to crontab, with just email sending, then call backup
+
+    def report
+      return unless config[:email_report]
+      msg = get_report()
+      MiscUtils::send_email(
+        :host => config[:mail_host],
+        :port => config[:mail_port],
+        :helodomain => config[:mail_helodomain],
+        :user => config[:mail_user],
+        :password => config[:mail_password],
+        :from => config[:mail_from],
+        :from_alias => config[:mail_from_alias],
+        :to => config[:mail_to],
+        :to_alias => config[:mail_to_alias],
+				:auth => config[:mail_auth],
+        :subject => 'backup report',
+        :message => msg
+      )
+    end
+
+    def self.database_from_xml(aDatabaseNode)
+      return {
+        :db_host => aDatabaseNode.attributes['Host'],
+        :db_user => aDatabaseNode.attributes['User'],
+        :db_password => aDatabaseNode.attributes['Password'],
+        :db_name => aDatabaseNode.attributes['Name'],
+        :file => XmlUtils::peek_node_value(aDatabaseNode, "ToFile")
+      }
+    end
+
+    def backup(aJobFiles)
+      #require 'ruby-debug'; debugger
+      return unless job = aJobFiles.is_a?(Array) ? aJobFiles.first : aJobFiles # just use first job
+
+      xmlRoot = XmlUtils.get_file_root(job)
+
+      filelist = []
+
+      REXML::XPath.each(xmlRoot, '/Yore/Sources/Source') do |xmlSource|
+				case xmlSource['Type']
+					when 'File' then 
+						REXML::XPath.each(xmlSource, 'IncludePath') do |xmlPath|
+							filelist += MiscUtils::recursive_file_list(File.expand_path(xmlPath.text,config[:basepath]))
+						end
+					when 'MySql' then
+						#<Source Type="MySql" >
+						#	<Database Host="" Name="" User="" Password="">
+						#		<ToFile>~/dbdump.sql</ToFile>
+						#	</Database>
+						#</Source>
+						REXML::XPath.each(xmlSource, 'Database') do |xmlDb|
+              args = Yore::database_from_xml(xmlDb)
+              file = args.delete(:file)
+							unless args[:db_host] && args[:db_user] && args[:db_password] && args[:db_name] && file
+								raise Error.new("Invalid or missing parameter")
+							end
+							db_to_file(args,file)
+							filelist += file						
+						end
+				end
+      end
+
+      filelist.uniq!
+      filelist.sort!
+
+      tempdir = MiscUtils.make_temp_dir('yore')
+      time = Time.now
+
+      backup_process(filelist,time,tempdir)
+      #clean
+      report
+    end
+		
+		def test_email(*aDb)
+			args = {
+        :host => config[:mail_host],
+        :port => config[:mail_port],
+        :helodomain => config[:mail_helodomain],
+        :user => config[:mail_user],
+        :password => config[:mail_password],
+        :from => config[:mail_from],
+        :from_alias => config[:mail_from_alias],
+        :to => config[:mail_to],
+        :to_alias => config[:mail_to_alias],
+				:auth => config[:mail_auth],
+        :subject => 'email test',
+        :message => 'Just testing email sending'
+			}
+			logger.debug args.inspect
+      MiscUtils::send_email(args)
+		end
+
+    def db_dump(aArgs)
+      return nil unless aArgs
+      return nil unless job = aArgs[0]
+
+      xmlRoot = XmlUtils.get_file_root(job)
+      xmlDb = nil
+      if db_name = aArgs[1]
+        xmlDb = XmlUtils::single_node(xmlRoot,"/Yore/Sources/Source[@Type='MySql']/Database[@Name='#{db_name}']")
+      else
+        xmlDb = XmlUtils::single_node(xmlRoot,"/Yore/Sources/Source[@Type='MySql']/Database")
+      end
+      raise Error.new("No database") unless xmlDb
+      args = Yore.database_from_xml(xmlDb)
+      file = args.delete(:file)
+      unless args[:db_host] && args[:db_user] && args[:db_password] && args[:db_name] && file
+        raise Error.new("Invalid or missing parameter")
+      end
+      db_to_file(args,file)
+    end
+
+  end
+    
+end

data/lib/yore.orig.rb

@@ -0,0 +1,6 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+module Yore
+  VERSION = '0.0.1'
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/yore.rb'}"
+puts "Loading yore gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)