darkofabijan-astrails-safe 0.2.8 → 0.2.9

data/lib/astrails/safe/page.rb

@@ -0,0 +1,34 @@
+module Astrails
+  module Safe
+    class Page < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        #puts "Hi from Cut pipe"
+        #puts "CUT CONFIG #{@config.parent.inspect}"
+        #puts "PIPE: #{@backup.inspect}"
+        "|gzip|split -d -b #{page_size} - #{@backup.filename}#{@backup.extension}."
+      end
+
+      def extension
+        ""
+      end
+
+      def active?
+        page_size 
+      end
+
+      private
+
+      def page_size
+        @page_size ||= @config[:page, :page_size]
+      end
+
+    end
+  end
+end

data/spec/integration/archive_paging_integration_spec.rb

@@ -0,0 +1,92 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require "fileutils"
+include FileUtils
+
+describe "tar backup" do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = "/tmp/safe-test"
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = "#{@root}/src"
+    mkdir_p "#{@src}/q/w/e"
+    mkdir_p "#{@src}/a/s/d"
+
+    File.open("#{@src}/qwe1", "w") {|f| f.write("qwe") }
+    File.open("#{@src}/q/qwe2", "w") {|f| f.write("qwe"*2) }
+    File.open("#{@src}/q/w/qwe3", "w") {|f| f.write("qwe"*3) }
+    File.open("#{@src}/q/w/e/qwe4", "w") {|f| f.write("qwe"*4) }
+
+    File.open("#{@src}/asd1", "w") {|f| f.write("asd") }
+    File.open("#{@src}/a/asd2", "w") {|f| f.write("asd" * 2) }
+    File.open("#{@src}/a/s/asd3", "w") {|f| f.write("asd" * 3) }
+
+    @dst = dst = "#{@root}/backup"
+    mkdir_p @dst
+
+    @now = Time.now
+    @timestamp = @now.strftime("%y%m%d-%H%M")
+
+    stub(Time).now {@now} # Freeze
+
+    Astrails::Safe.safe do
+      local :path => "#{dst}/:kind"
+      page do 
+        page_size 20
+      end
+      tar do
+        archive :test1 do
+          files src
+          exclude "#{src}/q/w"
+          exclude "#{src}/q/w/e"
+        end
+      end
+    end
+
+    @backup = "#{dst}/archive/archive-test1.#{@timestamp}.tar"
+  end
+
+  describe "after extracting" do
+    before(:all) do
+      # prepare target dir
+      @target = "#{@root}/test"
+      mkdir_p @target
+
+      backup_file_path = File.dirname(@backup)
+      backup_file_prefix = File.basename(@backup)
+      system "cat #{backup_file_prefix}.* >> #{backup_file_prefix}"
+      
+      system "tar -zxvf #{@backup} -C #{@target}"
+
+      @test = "#{@target}/#{@root}/src"
+      puts @test
+    end
+
+    it "should include asd1/2/3" do
+      File.exists?("#{@test}/asd1").should be_true
+      File.exists?("#{@test}/a/asd2").should be_true
+      File.exists?("#{@test}/a/s/asd3").should be_true
+    end
+
+    it "should only include qwe 1 and 2 (no 3)" do
+      File.exists?("#{@test}/qwe1").should be_true
+      File.exists?("#{@test}/q/qwe2").should be_true
+      File.exists?("#{@test}/q/w/qwe3").should be_false
+      File.exists?("#{@test}/q/w/e/qwe4").should be_false
+    end
+
+    it "should preserve file content" do
+      File.read("#{@test}/qwe1").should == "qwe"
+      File.read("#{@test}/q/qwe2").should == "qweqwe"
+      File.read("#{@test}/a/s/asd3").should == "asdasdasd"
+    end
+  end
+
+end

data/spec/integration/s3_paging_integration_spec.rb

@@ -0,0 +1,71 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require "fileutils"
+include FileUtils
+
+describe "s3 upload with paging" do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = "/tmp/safe-s3"
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = File.join(root, "src")
+    mkdir_p File.join(src, 'dir1', 'dir2', 'dir3')
+    mkdir_p File.join(src, 'dir3', 'dir4', 'dir5')
+
+    File.open(File.join(src, 'file1'), "w") {|f| f.write("dir1") }
+    File.open(File.join(src, 'dir1', 'file2'), "w") {|f| f.write("dir2"*2) }
+    File.open(File.join(src, 'dir1', 'dir2', 'file3'), "w") {|f| f.write("dir3"*3) }
+
+    @dst = dst = File.join(@root, 'backup')
+    mkdir_p @dst
+
+    @now = Time.now
+    @timestamp = @now.strftime("%y%m%d-%H%M")
+
+    stub(Time).now {@now} # Freeze
+    
+    Astrails::Safe.safe do
+      local :path => "#{dst}/:kind"
+
+      s3 do
+        key SERVICES_CONFIG['s3']['key'] 
+        secret SERVICES_CONFIG['s3']['secret']
+        bucket SERVICES_CONFIG['s3']['bucket']
+       path ":kind/" # this is default
+      end
+
+      page do 
+        page_size 20
+      end
+
+      tar do
+        archive :test1 do
+          files src
+        end
+      end
+
+    end
+    @backup_file = "archive/archive-test1.#{@timestamp}.tar"
+    @backup_path = File.join(dst, @backup_file)
+  end
+  
+  it "should upload all files to s3" do
+    AWS::S3::Base.establish_connection!(:access_key_id => SERVICES_CONFIG['s3']['key'], :secret_access_key => SERVICES_CONFIG['s3']['secret'], :use_ssl => true)
+
+    files = Dir.glob("#{@backup_path}*") 
+
+    files.each do |file|
+      file.gsub!("#{@dst}/", '')
+      AWS::S3::S3Object.exists?(file, "df-dev-test").should be_true
+    end
+
+  end
+
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: darkofabijan-astrails-safe
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 5
   prerelease: false
   segments: 
   - 0
   - 2
-  - 8
-  version: 0.2.8
+  - 9
+  version: 0.2.9
 platform: ruby
 authors: 
 - Astrails Ltd.
@@ -100,6 +100,7 @@ files:
 - lib/astrails/safe/config/builder.rb
 - lib/astrails/safe/config/node.rb
 - lib/astrails/safe/gpg.rb
+- lib/astrails/safe/page.rb
 - lib/astrails/safe/gzip.rb
 - lib/astrails/safe/local.rb
 - lib/astrails/safe/mysqldump.rb
@@ -114,6 +115,8 @@ files:
 - lib/astrails/safe/tmp_file.rb
 - lib/extensions/mktmpdir.rb
 - spec/integration/archive_integration_spec.rb
+- spec/integration/archive_paging_integration_spec.rb
+- spec/integration/s3_paging_integration_spec.rb
 - spec/integration/cleanup_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb