yore 0.0.3 → 0.0.4

data/test/S3_test.rb

@@ -0,0 +1,141 @@
+require 'rubygems'
+gem 'RequirePaths'; require 'require_paths'
+require_paths '../../..','../lib' 
+require 'ihl_ruby/misc_utils'
+require 'ihl_ruby/shell_extras'
+require 'yore/yore_core'
+require 'test/unit'
+gem 'Shoulda'; require 'shoulda'
+require 'fileutils'
+
+def ensure_clean_bucket(aName)
+  `s3cmd createbucket #{aName}`
+  `s3cmd deleteall #{aName}`
+end
+
+# finwa example : yore backup /var/www/joomla/deploy-system/finwa_yore.xml >/dev/null 2>&1
+
+class S3Test < Test::Unit::TestCase
+
+	def setup
+    @yore = YoreCore::Yore.new
+  end
+
+	should "upload and download files intact" do
+		bucket = 'yore_test'
+		ensure_clean_bucket(bucket)
+		@yore.configure({
+			:bucket => bucket
+		})
+		content = 'this is my test content'
+		temp_dir = MiscUtils.make_temp_dir()
+		temp_file = MiscUtils.make_temp_file(nil, temp_dir, content)
+		puts @yore.upload(temp_file)
+		orig_file = temp_file+'.orig'
+		File.rename(temp_file, orig_file)
+		puts @yore.download(temp_file)
+		down_file_content = MiscUtils.string_from_file(temp_file)
+		assert_equal content, down_file_content
+	end
+
+	should "backup and restore multiple directories of file" do
+		# create a temp dir
+		temp_dir = MiscUtils::make_temp_dir('yore_test')
+		# create source and dest subfolders
+		source_dir = File.expand_path('source',temp_dir)
+		source1 = File.join(source_dir,'source1')
+		source2 = File.join(source_dir,'source2')
+		dest_dir = File.expand_path('dest',temp_dir)
+		bucket = 'yore_test'
+
+		FileUtils.mkdir_p([source1,source2,dest_dir])
+		# create some dirs and files in source
+		['a','a/1','b','c'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source1))}
+		%w(a/blah.txt a/1/blahblah.txt b/apples.txt c/carrots.txt).each do |f|
+			MiscUtils::make_temp_file(f,source1)
+		end
+		['w','x/1','y/2','z'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source2))}
+		%w(w/zonk.txt x/1/eggs.txt y/2/bloop.txt z/zax.txt).each do |f|
+			MiscUtils::make_temp_file(f,source2)
+		end
+
+
+		ensure_clean_bucket(bucket)
+
+		# create job file
+		job_template = <<-EOF
+		<?xml version="1.0" encoding="UTF-8"?>
+		<Yore>
+				<SimpleItems>
+						<Item Name="crypto_iv">3A63775C1E3F291B0925578165EB917E</Item>
+						<Item Name="crypto_key">07692FC8656F04AE5518B80D38681E038A3C12050DF6CC97CEEC33D800D5E2FE</Item>
+						<Item Name="prefix">backup</Item>
+						<Item Name="log_level">INFO</Item>
+						<Item Name="bucket">${BUCKET}</Item>
+				</SimpleItems>
+			<Sources>
+						<Source Type="File" BasePath="${SOURCE_DIR}">
+							<IncludePath>source1</IncludePath>
+							<IncludePath>source2</IncludePath>
+						</Source>
+				</Sources>
+		</Yore>
+		EOF
+
+		job_content = StringUtils::render_template(job_template,{
+			'SOURCE_DIR' => source_dir,
+			'BUCKET' => bucket
+		})
+		MiscUtils::string_to_file(job_content,job_file = MiscUtils::temp_file)
+
+		# call yore script with ruby from the command line, then download result and check contents
+		begin
+			_xmlRoot = XmlUtils.get_xml_root(job_content)
+			cmd_options = {:config => job_file}
+			@yore_upload = YoreCore::Yore::launch(_xmlRoot,cmd_options,{:basepath => File.dirname(File.expand_path(job_file))})
+			@yore_upload.backup([job_file],cmd_options)
+		rescue ::StandardError => e
+			flunk e.inspect
+		end
+
+		#puts result
+		yore = YoreCore::Yore.new()
+		yore.configure({
+			:bucket => bucket
+		})
+
+		retrieved_fname = File.expand_path(yore.encode_file_name(), dest_dir)
+		collection_fname = MiscUtils::temp_file
+
+		puts yore.download(retrieved_fname)
+		puts yore.unpack(retrieved_fname,collection_fname)
+
+		filelist = MiscUtils::recursive_file_list(source_dir,false)
+
+		# check contains filelist files
+		cmd = "tar --list --file=#{collection_fname}"
+		filelist_out = yore.shell(cmd)
+		filelist_out = filelist_out.split("\n").sort
+		assert_equal filelist, filelist_out
+	end
+
+	should "handle test_file_b" do
+		aController = YoreCore::Yore.new      # main program object
+		srcdir = '/tmp/yoretest'
+		FileUtils.rm_rf srcdir+'/*'
+		FileUtils.mkdir_p srcdir
+		['a','a/1','b','c'].each {|p| FileUtils.mkdir_p(File.expand_path(p,srcdir))}
+		%w(a/blah.txt a/1/blahblah.txt b/apples.txt c/carrots.txt).each do |f|
+			MiscUtils::make_temp_file(f,srcdir)
+		end
+
+		job = File.expand_path('../../test/test_job_b.xml',THIS_DIR)
+		aController.configure(job)
+		aController.backup([job])
+	end
+
+	should "provide configurable criteria for keeping old files"
+
+	should "clean a folder full of files, removing files that don't match configurable criteria for keeping old files"
+
+end

data/test/loadsave_yore_test.rb

@@ -0,0 +1,182 @@
+require 'rubygems'
+gem 'RequirePaths'; require 'require_paths'
+require_paths '../../..','../lib' 
+
+require 'ihl_ruby/misc_utils'
+require 'ihl_ruby/config'
+require 'ihl_ruby/shell_extras'
+require 'yore/yore_core'
+require 'test/unit'
+gem 'Shoulda'; require 'shoulda'
+require 'fileutils'
+
+
+class LoadsaveYoreTest < Test::Unit::TestCase
+
+	def setup
+    #@yore = YoreCore::Yore.new
+  end
+	
+	should "save a spree applications data to a file and then load it into a different project" do
+		require "mysql"
+		cred = Credentials.new()
+		db_details = {
+			:username => cred[:mysql_username],
+			:password => cred[:mysql_password]
+		}
+
+		def create_spree_db(aDbDetails)
+			assert mysql = Mysql.new("localhost",aDbDetails[:username],aDbDetails[:password])
+			mysql.query("DROP DATABASE IF EXISTS #{aDbDetails[:database]}")
+			mysql.query("CREATE DATABASE #{aDbDetails[:database]}")
+			#assert mysql.query("GRANT ALL ON #{aName}.* TO rubyuser@localhost IDENTIFIED by 'ruby'")
+			mysql.select_db(aDbDetails[:database])
+			mysql.query("
+				CREATE TABLE products (
+					id int(11) NOT NULL AUTO_INCREMENT,
+					name varchar(30),
+					description varchar(50),
+					PRIMARY KEY (id)
+				)
+			")
+			mysql.query("INSERT INTO products (name,description) VALUES('spoon', 'golden spoon -#{aDbDetails[:database]}')")
+			mysql.query("INSERT INTO products (name,description) VALUES('fork', 'silver fork -#{aDbDetails[:database]}')")
+			mysql.query("
+				CREATE TABLE customers (
+					id int(11) NOT NULL AUTO_INCREMENT,
+					name varchar(30),
+					address varchar(50),
+					PRIMARY KEY (id)
+				)
+			")
+			mysql.query("INSERT INTO customers (name,address) VALUES('fred', '1 some st -#{aDbDetails[:database]}')")
+			mysql.query("INSERT INTO customers (name,address) VALUES('mary', '2 another rd -#{aDbDetails[:database]}')")
+			return mysql
+		end
+		
+		#users = [{:name => 'Bob', :permissions => ['Read']},
+		#         {:name => 'Alice', :permissions => ['Read', 'Write']}]
+		#
+		## Serialize
+		#open('users', 'w') { |f| YAML.dump(users, f) }
+		#
+		## And deserialize
+		#users2 = open("users") { |f| YAML.load(f) }
+		## => [{:permissions=>["Read"], :name=>"Bob"},
+		##     {:permissions=>["Read", "Write"], :name=>"Alice"}]
+
+		def hash_from_yaml_file(aFile)
+			return YAML::load(File.open(aFile)) rescue nil
+		end
+		
+		def yaml_file_from_hash(aName,aHash)
+			open(aName, 'w') { |f| YAML.dump(aHash, f) }
+		end
+		
+		RAND_CHARS	=	"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+		def random_string(aLength)
+			rand_max = RAND_CHARS.size
+			ret = "" 
+			aLength.times{ ret << RAND_CHARS[rand(rand_max)] }
+			ret
+		end
+		
+		def random_filename(aPrefix=nil)
+			aPrefix.to_s+random_string(8)+'.'+random_string(3)
+		end
+		
+		def create_random_file(aPath,aLength,aName=nil)
+			aName ||= random_filename()
+			path = File.join(aPath,aName)
+			MiscUtils.string_to_file(random_string(aLength),path)
+		end
+		
+		# eg 
+		def create_spree_folder(aParentDir,aFolderName,aDatabaseParams)
+			path = File.expand_path(aFolderName,aParentDir)
+			FileUtils.mkdir_p(path)
+			FileUtils.mkdir_p(File.join(path,'config'))
+			FileUtils.mkdir_p(File.join(path,'kind'))
+			FileUtils.mkdir_p(File.join(path,'public/assets/products'))
+			# create database yml file
+			dbs = {
+				'bogus' => {
+					'adapter' => 'mysql',
+					'database' => 'sdsdsad',
+					'username' => 'asdsadasd',
+					'password' => 'dfdsgdfg'
+				}
+			}
+			params = {'adapter' => 'mysql'}
+			aDatabaseParams.values.first.each {|k,v| params[k.to_s] = v}
+			dbs[aDatabaseParams.keys.first.to_s] = params
+			yaml_file_from_hash(File.join(path,'config/database.yml'),dbs)
+
+			3.times { create_random_file(File.join(path,'kind'),1+rand(100),random_filename(aFolderName+'_')) }
+			3.times { create_random_file(File.join(path,'public'),1+rand(100),random_filename(aFolderName+'_')) }
+			3.times { create_random_file(File.join(path,'public/assets'),1+rand(100),random_filename(aFolderName+'_')) }
+			3.times { create_random_file(File.join(path,'public/assets/products'),1+rand(100),random_filename(aFolderName+'_')) }
+			return path
+		end
+		
+		def filelist_with_sizes(aPath)
+			result = MiscUtils.recursive_file_list(aPath,true)
+			result.map! do |f|
+				shortf = MiscUtils.path_debase(f,aPath)
+				shortf + "|#{File.size?(f).to_s}"
+			end
+			result
+		end
+
+		tempdir = MiscUtils.make_temp_dir("yore_spree_test")
+
+		# set up spree-like A&B (different) test databases & files
+		db_a = create_spree_db(db_details.merge(:database=>"spree_test_a"))
+		path_a = create_spree_folder(tempdir,"spree_test_a",:test => db_details.merge(:database=>"spree_test_a"))
+		db_b = create_spree_db(db_details.merge(:database=>"spree_test_b"))
+		path_b = create_spree_folder(tempdir,"spree_test_b",:test => db_details.merge(:database=>"spree_test_b"))
+
+		DatabaseUtils::save_database(db_details.merge(:database=>"spree_test_a"),File.join(tempdir,'spree_test_a.sql'))
+		db_a_s = MiscUtils.string_from_file(File.join(tempdir,'spree_test_a.sql')).gsub!(/\n--.*$/,'')
+		DatabaseUtils::save_database(db_details.merge(:database=>"spree_test_b"),File.join(tempdir,'spree_test_b.sql'))
+		db_b_s = MiscUtils.string_from_file(File.join(tempdir,'spree_test_b.sql')).gsub!(/\n--.*$/,'')
+		
+		files_a_s = filelist_with_sizes(path_a).join("\n")
+		files_products_a_s = filelist_with_sizes(File.join(path_a,'public/assets/products')).join("\n")
+		files_b_s = filelist_with_sizes(path_b).join("\n")
+		files_products_b_s = filelist_with_sizes(File.join(path_b,'public/assets/products')).join("\n")
+
+		assert_not_equal files_a_s,files_b_s
+		assert_not_equal files_products_a_s,files_products_b_s
+
+		# launch yore to save A to file system equivalent to : 
+		# cd spree_test_a; yore save --app=spree --RAILS_ENV=test spree_test_a.tgz
+		@yore_a = YoreCore::Yore.launch(nil,{:kind => 'spree',:RAILS_ENV => 'test'},{:basepath => path_a})
+		spree_test_a_tgz = File.join(tempdir,'spree_test_a.tgz')
+		@yore_a.save(spree_test_a_tgz)
+
+		# launch yore to load file into B
+		@yore_b = YoreCore::Yore.launch(nil,{:kind => 'spree',:RAILS_ENV => 'test'},{:basepath => path_b})
+		@yore_b.load(spree_test_a_tgz)
+
+		DatabaseUtils::save_database(db_details.merge(:database=>"spree_test_b"),File.join(tempdir,'spree_test_b_after.sql'))
+		db_b_after_s = MiscUtils.string_from_file(File.join(tempdir,'spree_test_b_after.sql')).gsub!(/\n--.*$/,'')
+		
+		# assert files in A & B are the same within public/assets/products, and different elsewhere
+		assert_not_equal db_a_s,db_b_s
+		
+		db_a_s.gsub!(/\n--.*$/,'')
+		db_b_after_s.gsub!(/\n--.*$/,'')
+		MiscUtils.string_to_file(db_a_s,"/Users/gary/temp/db_a_s.txt")
+		MiscUtils.string_to_file(db_b_after_s,"/Users/gary/temp/db_b_after_s.txt")
+		assert_equal db_a_s,db_b_after_s			
+
+		files_b_after_s = filelist_with_sizes(path_b).join("\n")
+		files_products_b_after_s = filelist_with_sizes(File.join(path_b,'public/assets/products')).join("\n")
+		
+		assert_not_equal files_b_s,files_b_after_s
+		assert_not_equal files_products_b_after_s,files_products_b_s
+		assert_equal files_products_a_s,files_products_b_after_s	
+	end
+
+end

data/test/test_job_b.xml

@@ -14,28 +14,21 @@
 
 		<Item Name="mail_host">mail.authsmtp.com</Item>
 		<Item Name="mail_port">25</Item>
-		<Item Name="mail_helodomain">finwa.org.au</Item>
-		<Item Name="mail_user">ac37391</Item>
-		<Item Name="mail_password">mbju7wwzv</Item>
+		<Item Name="mail_helodomain">blah.com</Item>
+		<Item Name="mail_user">erewrwer</Item>
+		<Item Name="mail_password">werewrewr</Item>
 		<Item Name="mail_from">noreply@buzzware.com.au</Item>
-		<Item Name="mail_from_alias">FINWA Web Server</Item>
-		<Item Name="mail_to">familyincnet@gmail.com</Item>
-		<Item Name="mail_to_alias">FINWA</Item>
+		<Item Name="mail_from_alias">Web Server</Item>
+		<Item Name="mail_to">sadsd@blah.com</Item>
+		<Item Name="mail_to_alias">sdsadsd</Item>
 		<Item Name="mail_auth">login</Item>
-		
 		<Item Name="mysqldump">/usr/local/mysql/bin/mysqldump</Item>
 		
 		</SimpleItems>
 	<Sources>
 		<Source Type="File">
 			<IncludePath>/tmp/yoretest</IncludePath>
-			<!-- <ExcludePath></ExcludePath> -->
 		</Source>
-<!--<Source Type="MySql" >																														 -->
-<!--	<Database Name="joomla_db" Host="localhost" User="root" Password="prot123ection">-->
-<!--		<ToFile>/Users/gary/temp/test.sql</ToFile>																		 -->
-<!--	</Database>																																			 -->
-<!--</Source>																																					 -->
   </Sources>
 </Yore>
 

data/test/yore_test.rb

@@ -1,22 +1,12 @@
 require 'rubygems'
 gem 'RequirePaths'; require 'require_paths'
-require_paths '../lib','../../..'
-
-require 'test/unit'
-gem 'Shoulda'; require 'shoulda'
-
-	
+require_paths '../../..','../lib' 
 require 'ihl_ruby/misc_utils'
 require 'ihl_ruby/shell_extras'
-
-require 'fileutils'
-
 require 'yore/yore_core'
-
-def ensure_clean_bucket(aName)
-  `s3cmd createbucket #{aName}`
-  `s3cmd deleteall #{aName}`
-end
+require 'test/unit'
+gem 'Shoulda'; require 'shoulda'
+require 'fileutils'
 
 class YoreTest < Test::Unit::TestCase
 
@@ -24,7 +14,7 @@ class YoreTest < Test::Unit::TestCase
     @yore = YoreCore::Yore.new
   end
 
-  should "collect a list of files into a single file with a .contents file" do
+  should "collect a list of files into a single file" do
     # create a temp dir
     temp_dir = MiscUtils::make_temp_dir('yore_test')
     # create source and dest subfolders
@@ -38,22 +28,20 @@ class YoreTest < Test::Unit::TestCase
     end
     # collect into dest file
     # get recursive file list, without carrots
-    filelist = MiscUtils::recursive_file_list(source_dir) {|p| not p =~ /carrots.txt/}
+    filelist = MiscUtils::recursive_file_list(source_dir,false) {|p| not p =~ /carrots.txt/}
     dest_file = File.join(dest_dir,'destfile.bzb')
 
-    @yore.collect(filelist,dest_file)
+    @yore.compress(filelist,dest_file,source_dir)
 
-    # check contains filelist files ignoring .contents
+    # check contains filelist files
     cmd = "tar --list --file=#{dest_file}"
     filelist_out = `#{cmd}`
     i = 0
     filelist_out.each_line {|line|
-      next if line==".contents\n"
       assert i < filelist.length
-      assert_equal line.chomp("\n"), MiscUtils.path_debase(filelist[i],source_dir) # .bite('/')
+      assert_equal line.chomp("\n"), filelist[i] # .bite('/')
       i += 1
     }
-    # could also check .contents file against actual contents
   end
 
   should "encrypt and unencrypt a file with a standard iv but a supplied key" do
@@ -81,125 +69,8 @@ class YoreTest < Test::Unit::TestCase
     assert_equal Time.local(2008, 12, 31), @yore.decode_file_name(calc_filename)
   end
 
-  should "upload and download files intact" do
-    bucket = 'yore_test'
-    ensure_clean_bucket(bucket)
-    @yore.configure({
-      :bucket => bucket
-    })
-    content = 'this is my test content'
-    temp_dir = MiscUtils.make_temp_dir()
-    temp_file = MiscUtils.make_temp_file(nil, temp_dir, content)
-    puts @yore.upload(temp_file)
-    orig_file = temp_file+'.orig'
-    File.rename(temp_file, orig_file)
-    puts @yore.download(temp_file)
-    down_file_content = MiscUtils.string_from_file(temp_file)
-    assert_equal content, down_file_content
-  end
-
-  should "backup and restore multiple directories of file" do
-    # create a temp dir
-    temp_dir = MiscUtils::make_temp_dir('yore_test')
-    # create source and dest subfolders
-    source_dir = File.expand_path('source',temp_dir)
-    source1 = File.join(source_dir,'source1')
-    source2 = File.join(source_dir,'source2')
-    dest_dir = File.expand_path('dest',temp_dir)
-    bucket = 'yore_test'
-
-    FileUtils.mkdir_p([source1,source2,dest_dir])
-    # create some dirs and files in source
-    ['a','a/1','b','c'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source1))}
-    %w(a/blah.txt a/1/blahblah.txt b/apples.txt c/carrots.txt).each do |f|
-      MiscUtils::make_temp_file(f,source1)
-    end
-    ['w','x/1','y/2','z'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source2))}
-    %w(w/zonk.txt x/1/eggs.txt y/2/bloop.txt z/zax.txt).each do |f|
-      MiscUtils::make_temp_file(f,source2)
-    end
-
-    # create job file
-job_template = <<EOF
-<?xml version="1.0" encoding="UTF-8"?>
-<Yore>
-    <SimpleItems>
-        <Item Name="crypto_iv">3A63775C1E3F291B0925578165EB917E</Item>
-        <Item Name="crypto_key">07692FC8656F04AE5518B80D38681E038A3C12050DF6CC97CEEC33D800D5E2FE</Item>
-        <Item Name="prefix">backup</Item>
-        <Item Name="log_level">INFO</Item>
-        <Item Name="bucket">${BUCKET}</Item>
-    </SimpleItems>
-	<Sources>
-        <Source Type="File">
-            <IncludePath>${SOURCE1}</IncludePath>
-            <IncludePath>${SOURCE2}</IncludePath>
-        </Source>
-    </Sources>
-</Yore>
-EOF
-
-    job_content = StringUtils::render_template(job_template,{
-      'SOURCE1' => source1,
-      'SOURCE2' => source2,
-      'BUCKET' => bucket
-    })
-  MiscUtils::string_to_file(job_content,job_file = MiscUtils::temp_file)
-
-    ensure_clean_bucket(bucket)
-    @yore.configure({
-      :bucket => bucket
-    })
-
-    # call yore script with ruby from the command line, then download result and check contents
-    begin
-      result = @yore.shell("ruby #{File.expand_path('../../bin/yore',THIS_DIR)} backup #{job_file}")
-    rescue ::StandardError => e
-      flunk e.inspect
-    end
-
-    puts result
-
-    retrieved_fname = File.expand_path(@yore.encode_file_name(), dest_dir)
-    collection_fname = MiscUtils::temp_file
-
-    puts @yore.download(retrieved_fname)
-    puts @yore.unpack(retrieved_fname,collection_fname)
-
-    filelist = MiscUtils::recursive_file_list(source_dir,false)
-
-    # check contains filelist files ignoring .contents
-    cmd = "tar --list --file=#{collection_fname}"
-    filelist_out = @yore.shell(cmd)
-    filelist_out = filelist_out.split("\n").sort
-    assert_equal filelist, filelist_out
-#    i = 0
-#    filelist_out.each {|line|
-#      next if line==".contents\n"
-#      i.should < filelist.length
-#      line.chomp("\n").should == MiscUtils.path_debase(filelist[i],source_dir) # .bite('/')
-#      i += 1
-#    }
-    # could also check .contents file against actual contents
-  end
-
-  should "handle test_file_b" do
-		aController = YoreCore::Yore.new      # main program object
-    job = File.expand_path('../../test/test_job_b.xml',THIS_DIR)
-    xmlRoot = XmlUtils.get_file_root(job)
-    srcdir = '/tmp/yoretest'
-    FileUtils.rm_rf srcdir+'/*'
-    FileUtils.mkdir_p srcdir
-    ['a','a/1','b','c'].each {|p| FileUtils.mkdir_p(File.expand_path(p,srcdir))}
-    %w(a/blah.txt a/1/blahblah.txt b/apples.txt c/carrots.txt).each do |f|
-      MiscUtils::make_temp_file(f,srcdir)
-    end
-
-    aController.configure(xmlRoot,nil,{:basepath => File.dirname(File.expand_path(job))})
-    aController.backup(job)
-	end
-
   should "provide configurable criteria for keeping old files"
 
   should "clean a folder full of files, removing files that don't match configurable criteria for keeping old files"
+
 end