yore 0.0.1

data/spec/rewind_spec.rb

@@ -0,0 +1,187 @@
+require 'rubygems'
+gem 'RequirePaths'; require 'require_paths'
+require_paths '../../..'
+
+require 'ihl_ruby/misc_utils'
+
+require 'fileutils'
+
+require 'yore_core'
+
+def ensure_clean_bucket(aName)
+  `s3cmd createbucket #{aName}`
+  `s3cmd deleteall #{aName}`
+end
+
+describe YoreCore::Yore do
+  before(:each) do
+    @yore = YoreCore::Yore.new
+  end
+
+  it "should collect a list of files into a single file with a .contents file" do
+    # create a temp dir
+    temp_dir = MiscUtils::make_temp_dir('yore_spec')
+    # create source and dest subfolders
+    source_dir = File.expand_path('source',temp_dir)
+    dest_dir = File.expand_path('dest',temp_dir)
+    FileUtils.mkdir_p([source_dir,dest_dir])
+    # create some dirs and files in source
+    ['a','a/1','b','c'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source_dir))}
+    %w(a/blah.txt a/1/blahblah.txt b/apples.txt c/carrots.txt).each do |f|
+      MiscUtils::make_temp_file(f,source_dir)
+    end
+    # collect into dest file
+    # get recursive file list, without carrots
+    filelist = MiscUtils::recursive_file_list(source_dir) {|p| not p =~ /carrots.txt/}
+    dest_file = File.join(dest_dir,'destfile.bzb')
+
+    @yore.collect(filelist,dest_file)
+
+    # check contains filelist files ignoring .contents
+    cmd = "tar --list --file=#{dest_file}"
+    filelist_out = `#{cmd}`
+    i = 0
+    filelist_out.each_line {|line|
+      next if line==".contents\n"
+      i.should < filelist.length
+      line.chomp("\n").should == MiscUtils.path_debase(filelist[i],source_dir) # .bite('/')
+      i += 1
+    }
+    # could also check .contents file against actual contents
+  end
+
+  it "should encrypt and unencrypt a file with a standard iv but a supplied key" do
+    orig_content = 'abcdef123456'
+    temp_file1 = MiscUtils.make_temp_file(nil,nil,orig_content)
+    temp_file2 = temp_file1+'.enc'
+    temp_file3 = temp_file1+'.dec'
+
+    @yore.pack(temp_file1,temp_file2)
+    @yore.unpack(temp_file2,temp_file3)
+
+    file3_content = MiscUtils.string_from_file(temp_file3)
+    file3_content.should == orig_content
+  end
+
+  it "should create a backup filename combining a prefix, date, day and standard extension, and be able to decode the filename back to a date" do
+    @yore.configure({
+      :prefix => 'spec',
+      :first_hour => 4
+    })
+    now = Time.local(2009, 1, 1, 0, 0, 0, 0)
+    exp_filename = "spec-20081231.rew"
+    calc_filename = @yore.encode_file_name(now)
+    calc_filename.should == exp_filename
+    @yore.decode_file_name(calc_filename).should == Time.local(2008, 12, 31)
+  end
+
+  it "should upload and download files intact" do
+    bucket = 'yore_spec'
+    ensure_clean_bucket(bucket)
+    @yore.configure({
+      :bucket => bucket
+    })
+    content = 'this is my test content'
+    temp_dir = MiscUtils.make_temp_dir()
+    temp_file = MiscUtils.make_temp_file(nil, temp_dir, content)
+    puts @yore.upload(temp_file)
+    orig_file = temp_file+'.orig'
+    File.rename(temp_file, orig_file)
+    puts @yore.download(temp_file)
+    down_file_content = MiscUtils.string_from_file(temp_file)
+    down_file_content.should == content
+  end
+
+  it "should backup and restore multiple directories of file" do
+    # create a temp dir
+    temp_dir = MiscUtils::make_temp_dir('yore_spec')
+    # create source and dest subfolders
+    source_dir = File.expand_path('source',temp_dir)
+    source1 = File.join(source_dir,'source1')
+    source2 = File.join(source_dir,'source2')
+    dest_dir = File.expand_path('dest',temp_dir)
+    bucket = 'yore_spec'
+    
+    FileUtils.mkdir_p([source1,source2,dest_dir])
+    # create some dirs and files in source
+    ['a','a/1','b','c'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source1))}
+    %w(a/blah.txt a/1/blahblah.txt b/apples.txt c/carrots.txt).each do |f|
+      MiscUtils::make_temp_file(f,source1)
+    end
+    ['w','x/1','y/2','z'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source2))}
+    %w(w/zonk.txt x/1/eggs.txt y/2/bloop.txt z/zax.txt).each do |f|
+      MiscUtils::make_temp_file(f,source2)
+    end
+
+    # create job file
+job_template = <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<Yore>
+    <SimpleItems>
+        <Item Name="crypto_iv">3A63775C1E3F291B0925578165EB917E</Item>
+        <Item Name="crypto_key">07692FC8656F04AE5518B80D38681E038A3C12050DF6CC97CEEC33D800D5E2FE</Item>
+        <Item Name="prefix">backup</Item>
+        <Item Name="log_level">DEBUG</Item>
+        <Item Name="bucket">${BUCKET}</Item>
+    </SimpleItems>
+	<Sources>
+        <Source>
+            <IncludePath>${SOURCE1}</IncludePath>
+            <IncludePath>${SOURCE2}</IncludePath>
+        </Source>
+    </Sources>
+</Yore>
+EOF
+
+    job_content = StringUtils::render_template(job_template,{
+      'SOURCE1' => source1,
+      'SOURCE2' => source2,
+      'BUCKET' => bucket
+    })
+  MiscUtils::string_to_file(job_content,job_file = MiscUtils::temp_file)
+
+    ensure_clean_bucket(bucket)
+    @yore.configure({
+      :bucket => bucket
+    })
+
+    # call yore script with ruby from the command line, then download result and check contents
+    puts cmd = "#{File.expand_path('../lib/yore',THIS_DIR)} backup #{job_file}"
+    puts result = `#{cmd}`
+
+    retrieved_fname = File.expand_path(@yore.encode_file_name(), dest_dir)
+    collection_fname = MiscUtils::temp_file
+
+    puts @yore.download(retrieved_fname)
+    puts @yore.unpack(retrieved_fname,collection_fname)
+
+    filelist = MiscUtils::recursive_file_list(source_dir,false)
+
+    # check contains filelist files ignoring .contents
+    cmd = "tar --list --file=#{collection_fname}"
+    filelist_out = `#{cmd}`
+    filelist_out = filelist_out.split("\n").sort
+    filelist_out.should == filelist
+#    i = 0
+#    filelist_out.each {|line|
+#      next if line==".contents\n"
+#      i.should < filelist.length
+#      line.chomp("\n").should == MiscUtils.path_debase(filelist[i],source_dir) # .bite('/')
+#      i += 1
+#    }
+    # could also check .contents file against actual contents
+  end
+
+  it "should assign proper boolean values" do
+    class Blah
+      SOMECONSTANT = {:a => 'AAA', :b => true, :c => false}
+    end
+
+    puts Blah::SOMECONSTANT.inspect
+  end
+
+  it "should provide configurable criteria for keeping old files"
+
+  it "should clean a folder full of files, removing files that don't match configurable criteria for keeping old files"
+end
+

data/spec/spec.opts

@@ -0,0 +1 @@
+--colour

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems'
+  gem 'rspec'
+  require 'spec'
+end
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'yore'

data/spec/test_job_a.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Yore>
+    <SimpleItems>
+        <Item Name="keep_daily">14</Item>
+        <Item Name="keep_weekly">12</Item>
+        <Item Name="keep_monthly">12</Item>
+        <Item Name="crypto_iv">3A63775C1E3F291B0925578165EB917E</Item>
+        <Item Name="crypto_key">07692FC8656F04AE5518B80D38681E038A3C12050DF6CC97CEEC33D800D5E2FE</Item>
+        <Item Name="first_hour">4</Item>
+        <Item Name="prefix">backup</Item>
+        <Item Name="log_level">DEBUG</Item>
+    </SimpleItems>
+	<Sources>
+        <Source>
+            <IncludePath>test_source</IncludePath>
+            <!-- <ExcludePath></ExcludePath> -->
+        </Source>
+    </Sources>
+</Yore>

data/spec/yore_spec.rb

@@ -0,0 +1,199 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+gem 'RequirePaths'; require 'require_paths'
+require_paths '../lib','../../..'
+
+
+## Time to add your specs!
+# http://rspec.info/
+#describe "Place your specs here" do
+#  
+#  it "find this spec in spec directory" do
+#    violated "Be sure to write your specs"
+#	end
+#end
+	
+require 'ihl_ruby/misc_utils'
+
+require 'fileutils'
+
+require 'yore/yore_core'
+
+def ensure_clean_bucket(aName)
+  `s3cmd createbucket #{aName}`
+  `s3cmd deleteall #{aName}`
+end
+
+describe YoreCore::Yore do
+  before(:each) do
+    @yore = YoreCore::Yore.new
+  end
+
+  it "should collect a list of files into a single file with a .contents file" do
+    # create a temp dir
+    temp_dir = MiscUtils::make_temp_dir('yore_spec')
+    # create source and dest subfolders
+    source_dir = File.expand_path('source',temp_dir)
+    dest_dir = File.expand_path('dest',temp_dir)
+    FileUtils.mkdir_p([source_dir,dest_dir])
+    # create some dirs and files in source
+    ['a','a/1','b','c'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source_dir))}
+    %w(a/blah.txt a/1/blahblah.txt b/apples.txt c/carrots.txt).each do |f|
+      MiscUtils::make_temp_file(f,source_dir)
+    end
+    # collect into dest file
+    # get recursive file list, without carrots
+    filelist = MiscUtils::recursive_file_list(source_dir) {|p| not p =~ /carrots.txt/}
+    dest_file = File.join(dest_dir,'destfile.bzb')
+
+    @yore.collect(filelist,dest_file)
+
+    # check contains filelist files ignoring .contents
+    cmd = "tar --list --file=#{dest_file}"
+    filelist_out = `#{cmd}`
+    i = 0
+    filelist_out.each_line {|line|
+      next if line==".contents\n"
+      i.should < filelist.length
+      line.chomp("\n").should == MiscUtils.path_debase(filelist[i],source_dir) # .bite('/')
+      i += 1
+    }
+    # could also check .contents file against actual contents
+  end
+
+  it "should encrypt and unencrypt a file with a standard iv but a supplied key" do
+    orig_content = 'abcdef123456'
+    temp_file1 = MiscUtils.make_temp_file(nil,nil,orig_content)
+    temp_file2 = temp_file1+'.enc'
+    temp_file3 = temp_file1+'.dec'
+
+    @yore.pack(temp_file1,temp_file2)
+    @yore.unpack(temp_file2,temp_file3)
+
+    file3_content = MiscUtils.string_from_file(temp_file3)
+    file3_content.should == orig_content
+  end
+
+  it "should create a backup filename combining a prefix, date, day and standard extension, and be able to decode the filename back to a date" do
+    @yore.configure({
+      :prefix => 'spec',
+      :first_hour => 4
+    })
+    now = Time.local(2009, 1, 1, 0, 0, 0, 0)
+    exp_filename = "spec-20081231.rew"
+    calc_filename = @yore.encode_file_name(now)
+    calc_filename.should == exp_filename
+    @yore.decode_file_name(calc_filename).should == Time.local(2008, 12, 31)
+  end
+
+  it "should upload and download files intact" do
+    bucket = 'yore_spec'
+    ensure_clean_bucket(bucket)
+    @yore.configure({
+      :bucket => bucket
+    })
+    content = 'this is my test content'
+    temp_dir = MiscUtils.make_temp_dir()
+    temp_file = MiscUtils.make_temp_file(nil, temp_dir, content)
+    puts @yore.upload(temp_file)
+    orig_file = temp_file+'.orig'
+    File.rename(temp_file, orig_file)
+    puts @yore.download(temp_file)
+    down_file_content = MiscUtils.string_from_file(temp_file)
+    down_file_content.should == content
+  end
+
+  it "should backup and restore multiple directories of file" do
+    # create a temp dir
+    temp_dir = MiscUtils::make_temp_dir('yore_spec')
+    # create source and dest subfolders
+    source_dir = File.expand_path('source',temp_dir)
+    source1 = File.join(source_dir,'source1')
+    source2 = File.join(source_dir,'source2')
+    dest_dir = File.expand_path('dest',temp_dir)
+    bucket = 'yore_spec'
+    
+    FileUtils.mkdir_p([source1,source2,dest_dir])
+    # create some dirs and files in source
+    ['a','a/1','b','c'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source1))}
+    %w(a/blah.txt a/1/blahblah.txt b/apples.txt c/carrots.txt).each do |f|
+      MiscUtils::make_temp_file(f,source1)
+    end
+    ['w','x/1','y/2','z'].each {|p| FileUtils.mkdir_p(File.expand_path(p,source2))}
+    %w(w/zonk.txt x/1/eggs.txt y/2/bloop.txt z/zax.txt).each do |f|
+      MiscUtils::make_temp_file(f,source2)
+    end
+
+    # create job file
+job_template = <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<Yore>
+    <SimpleItems>
+        <Item Name="crypto_iv">3A63775C1E3F291B0925578165EB917E</Item>
+        <Item Name="crypto_key">07692FC8656F04AE5518B80D38681E038A3C12050DF6CC97CEEC33D800D5E2FE</Item>
+        <Item Name="prefix">backup</Item>
+        <Item Name="log_level">DEBUG</Item>
+        <Item Name="bucket">${BUCKET}</Item>
+    </SimpleItems>
+	<Sources>
+        <Source>
+            <IncludePath>${SOURCE1}</IncludePath>
+            <IncludePath>${SOURCE2}</IncludePath>
+        </Source>
+    </Sources>
+</Yore>
+EOF
+
+    job_content = StringUtils::render_template(job_template,{
+      'SOURCE1' => source1,
+      'SOURCE2' => source2,
+      'BUCKET' => bucket
+    })
+  MiscUtils::string_to_file(job_content,job_file = MiscUtils::temp_file)
+
+    ensure_clean_bucket(bucket)
+    @yore.configure({
+      :bucket => bucket
+    })
+
+    # call yore script with ruby from the command line, then download result and check contents
+    puts cmd = "ruby #{File.expand_path('../bin/yore.rb',THIS_DIR)} backup #{job_file}"
+    puts result = `#{cmd}`
+
+    retrieved_fname = File.expand_path(@yore.encode_file_name(), dest_dir)
+    collection_fname = MiscUtils::temp_file
+
+    puts @yore.download(retrieved_fname)
+    puts @yore.unpack(retrieved_fname,collection_fname)
+
+    filelist = MiscUtils::recursive_file_list(source_dir,false)
+
+    # check contains filelist files ignoring .contents
+    cmd = "tar --list --file=#{collection_fname}"
+    filelist_out = `#{cmd}`
+    filelist_out = filelist_out.split("\n").sort
+    filelist_out.should == filelist
+#    i = 0
+#    filelist_out.each {|line|
+#      next if line==".contents\n"
+#      i.should < filelist.length
+#      line.chomp("\n").should == MiscUtils.path_debase(filelist[i],source_dir) # .bite('/')
+#      i += 1
+#    }
+    # could also check .contents file against actual contents
+  end
+
+  it "should assign proper boolean values" do
+    class Blah
+      SOMECONSTANT = {:a => 'AAA', :b => true, :c => false}
+    end
+
+    puts Blah::SOMECONSTANT.inspect
+  end
+  
+
+  it "should provide configurable criteria for keeping old files"
+
+  it "should clean a folder full of files, removing files that don't match configurable criteria for keeping old files"
+end

data/tasks/rspec.rake

@@ -0,0 +1,21 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems'
+  require 'spec'
+end
+begin
+  require 'spec/rake/spectask'
+rescue LoadError
+  puts <<-EOS
+To use rspec for testing you must install rspec gem:
+    gem install rspec
+EOS
+  exit(0)
+end
+
+desc "Run the specs under spec/models"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "spec/spec.opts"]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification 
+name: yore
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Gary McGhee
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-01-25 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: RequirePaths
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: cmdparse
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: s3sync
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.5
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: newgem
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.8.0
+    version: 
+description: FIX (describe your package)
+email: 
+- contact@buzz@ware@com@au
+executables: 
+- yore
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+- README.rdoc
+files: 
+- bin/yore
+- History.txt
+- lib/ihl_ruby/enum.rb
+- lib/ihl_ruby/extend_base_classes.rb
+- lib/ihl_ruby/misc_utils.rb
+- lib/ihl_ruby/string_utils.rb
+- lib/ihl_ruby/xml_utils.rb
+- lib/yore.orig.rb
+- lib/yore/yore_core.rb
+- Manifest.txt
+- PostInstall.txt
+- Rakefile
+- README.rdoc
+- script/console
+- script/destroy
+- script/generate
+- spec/rewind_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/test_job_a.xml
+- spec/yore_spec.rb
+- tasks/rspec.rake
+has_rdoc: true
+homepage: FIX (url)
+post_install_message: 
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: buzzware
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: FIX (describe your package)
+test_files: []
+