encbs 0.1.3 → 0.2.0.alpha

data/Gemfile

@@ -3,8 +3,10 @@ source "http://rubygems.org"
 gem "fog"
 gem "slop"
 gem "ruby-progressbar"
+gem "lzoruby"
 
 group :development do
+  gem "rake", "0.8.4"
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.6.0"
   gem "rcov", ">= 0"

data/VERSION

@@ -1 +1 @@
-0.1.3
+0.2.0.alpha

data/bin/encbs

@@ -8,218 +8,261 @@ require 'fileutils'
 require 'openssl'
 require 'socket'
 require 'helpers'
-
-safe_require do
-  require 'slop'
-  require 'fog'
-  require 'progressbar'
-end
+require 'slop'
+require 'fog'
+require 'progressbar'
+require 'lzoruby'
+require 'zlib'
+require 'base64'
 
 require 'backup'
 require 'encbsconfig'
 
-opts = Slop.parse :help => true do
-  on :a, :add, "Add path to backup", true
-  on :b, :bucket, "Set Amazon S3 bucket to backup", true
-  on :c, :config, "Use config file to upload backup", true
-  on :colorize, "Colorize print to console"
-  on :d, :date, "Date for backup restore (default: last)", true
-  on :g, :generate, "Generate RSA keys (option: 4096, 2048)", true
-  on :h, :hostname, "Set hostname (default: system)", true
-  on :i, :increment, "Use increment mode for backup (default: false)"
-  on :j, :jar, "Versions of jar (option: hash or path)", true
-  on :k, :key, "Set API key to access Amazon S3", true
-  on :l, :local, "Backup in local directory", true
-  on :list, "List of jars"
-  on :r, :rescue, "Return data from backup (option: jar, path or filter)", true
-  on :s, :secret, "Set API secret to access Amazon S3", true
-  on :t, :to, "Path to recovery (default: /)", true
-  on :token, "RSA Key to encrypt/decrypt backup data", true
-  on :v, :verbose, "Verbose mode"
-
-  banner "Usage:\n    $ encbs [options]\n\nOptions:"
-end
-
-@config = EncbsConfig.new
-
-if ARGV.empty?
-  if File.exists?("#{Dir.getwd}/Encbsfile")
-    @config.load "#{Dir.getwd}/Encbsfile"
+begin
+  if lock_exists?
+    puts_fail "Lock file exists. Make sure that an encbs instance does not exists."
   else
-    puts opts.help
-    exit
-  end
-end
-
-@config.load opts[:config] if opts.config?
-
-$PRINT_VERBOSE = @config.verbose || opts.verbose?
-$COLORIZE = @config.colorize || opts.colorize?
-
-if opts.generate?
-  bits = opts[:generate].to_i
-  puts_fail "Unsupport #{bits} bits" unless bits == 4096 or bits == 2048
-  puts "Generate #{bits} bits RSA keys"
-  Crypto::create_keys(
-    File.join(Dir.getwd, "rsa_key"),
-  	File.join(Dir.getwd, "rsa_key.pub"),
-  	bits
-  )
-  puts "Done!"
-
-  exit
-end
-
-if opts.local?
-  try_create_dir opts[:local]
-  @backup = Backup::Instance.new opts[:local]
-else
-  [:key, :secret, :bucket].each do |arg|
-    if opts[arg].nil? and @config.send(arg).nil?
-      puts_fail "Argument '--#{arg}' should not be empty"
-    end
+    create_lock
   end
-  @backup = Backup::Instance.new(
-    "backups",
-    true,
-    :bucket => @config.bucket || opts[:bucket],
-    :key => @config.key || opts[:key],
-    :secret => @config.secret || opts[:secret]
-  )
-end
-
-@backup.hostname = @config.hostname || (opts[:hostname] if opts.hostname?)
-
-if opts.list?
-  jars_list = @backup.jars
-
-  unless jars_list.empty?
-    puts "List of jars:\n"
-    jars_list.keys.sort.each do |key|
-      puts "    #{key.dark_green}: #{jars_list[key]}"
-    end
-  else
-    puts "Nothing to listing."
+
+  opts = Slop.parse :help => true do
+    on :a, :add, "Add path to backup", true
+    on :b, :bucket, "Set Amazon S3 bucket to backup", true
+    on :c, :config, "Use config file to upload backup", true
+    on :colorize, "Colorize print to console"
+    on :compression, "Use gzip for file (de)compression (option: gzip, lzo)", true
+    on :d, :date, "Date for backup restore (default: last)", true
+    on :g, :generate, "Generate RSA keys (option: 4096, 2048)", true
+    on :h, :hostname, "Set hostname (default: system)", true
+    on :i, :increment, "Use increment mode for backup (default: false)"
+    on :j, :jar, "Versions of jar (option: hash or path)", true
+    on :k, :key, "Set API key to access Amazon S3", true
+    on :l, :local, "Backup in local directory", true
+    on :list, "List of jars"
+    on :r, :rescue, "Return data from backup (option: jar, path or filter)", true
+    on :s, :secret, "Set API secret to access Amazon S3", true
+    on :t, :to, "Path to recovery (default: /)", true
+    on :timeout, "Timeout(sec) to try pushing into cloud (default: 60s)", true
+    on :token, "RSA key to encrypt/decrypt backup data", true
+    on :size, "RSA private key length (default: 4096)", true
+    on :v, :verbose, "Verbose mode"
+
+    banner "Usage:\n    $ encbs [options]\n\nOptions:"
   end
 
-  exit
-end
+  @config = EncbsConfig.new
 
-if @config.token || opts.token?
-  key = @config.token || opts[:token]
+  if ARGV.empty?
+    if File.exists?("#{Dir.getwd}/Encbsfile")
+      @config.load "#{Dir.getwd}/Encbsfile"
+    else
+      puts opts.help
+      exit
+    end
+  end
 
-  puts_fail "Key #{key.dark_green} not found" unless File.exists? key
+  @config.load opts[:config] if opts.config?
 
-  @backup.key = key
-end
+  $PRINT_VERBOSE = @config.verbose || opts.verbose?
+  $COLORIZE = @config.colorize || opts.colorize?
 
-if opts.date?
-  date = opts[:date].split("-")
+  if opts.generate?
+    bits = opts[:generate].to_i
+    puts_fail "Unsupport #{bits} bits" unless bits == 4096 or bits == 2048
+    puts "Generate #{bits} bits RSA keys"
+    Crypto::create_keys(
+      File.join(Dir.getwd, "rsa_key"),
+    	File.join(Dir.getwd, "rsa_key.pub"),
+    	bits
+    )
+    puts "Done!"
 
-  unless date.length == 1
-    @start_date = Backup::Timestamp.parse_timestamp date[0]
-    @end_date = Backup::Timestamp.parse_timestamp date[1], true
+    exit
+  end
 
-    puts_fail "Last date less than start date" if start_date > end_date
+  if opts.local?
+    try_create_dir opts[:local]
+    @backup = Backup::Instance.new opts[:local]
   else
-    @start_date = Backup::Timestamp.parse_timestamp date[0]
-    @end_date = Backup::Timestamp.parse_timestamp date[0], true
+    [:key, :secret, :bucket].each do |arg|
+      if opts[arg].nil? and @config.send(arg).nil?
+        puts_fail "Argument '--#{arg}' should not be empty"
+      end
+    end
+    @backup = Backup::Instance.new(
+      "backups",
+      true,
+      :bucket => @config.bucket || opts[:bucket],
+      :key => @config.key || opts[:key],
+      :secret => @config.secret || opts[:secret]
+    )
   end
-else
-  @start_date = nil
-  @end_date = Time.now.utc
-end
 
-if opts.jar?
-  opts[:jar].split(" ").each do |jar|
-    versions = @backup.jar_versions(jar)
+  hostname = @config.hostname || opts[:hostname] if opts.hostname?
+  @backup.hostname = hostname unless hostname.nil?
+
+  timeout = @config.timeout || opts[:timeout] if opts.timeout?
+  @backup.file_item.timeout = timeout unless timeout.nil?
 
-    unless versions.empty?
-      puts "Versions of backup '#{jar}':"
+  compression = @config.compression || opts[:compression] if opts.compression?
+  @backup.compression = compression unless compression.nil?
 
-      versions.each do |version|
-        puts " => #{version.dark_green}: #{Backup::Timestamp.to_str(version)}"
+  if opts.list?
+    jars_list = @backup.jars
+
+    unless jars_list.empty?
+      puts "List of jars:\n"
+      jars_list.keys.sort.each do |key|
+        puts "    #{key.dark_green}: #{jars_list[key]}"
       end
     else
-      puts "Versions doesn't exists for jar: #{jar}"
+      puts "Nothing to listing."
     end
+
+    exit
   end
 
-  exit
-end
+  if !!@config.token || opts.token?
+    key = @config.token || opts[:token]
+    puts_fail "Key #{key.dark_green} not found" unless File.exists? key
 
-#TODO: Support rescue option as hash
-if opts.rescue?
-  paths = opts[:rescue].split(" ")
-  jars_list = @backup.jars
+    size = (@config.size || opts[:size]).to_i
+    puts_fail "Unsupport #{size} bits" unless size == 4096 or size == 2048
 
-  include_path = lambda {|path| jars_list.keys.include? path}
+    @backup.rsa_key(key, size)
+  end
 
-  jars_hashes = paths.map do |path|
-    path = File.expand_path path
+  if opts.date?
+    date = opts[:date].split("-")
 
-    unless include_path[path] or include_path["#{path}/"]
-      puts_fail "Jar \"#{path}\" not exists." 
-    end
+    unless date.length == 1
+      @start_date = Backup::Timestamp.parse_timestamp date[0]
+      @end_date = Backup::Timestamp.parse_timestamp date[1], true
 
-    jars_list[path] || jars_list["#{path}/"]
+      puts_fail "Last date less than start date" if start_date > end_date
+    else
+      @start_date = Backup::Timestamp.parse_timestamp date[0]
+      @end_date = Backup::Timestamp.parse_timestamp date[0], true
+    end
+  else
+    @start_date = nil
+    @end_date = Time.now.utc
   end
 
-  if opts.to?
-    @to = File.expand_path opts[:to]
-    try_create_dir @to
-  else
-    @to = "/"
+  if opts.jar?
+    opts[:jar].split(" ").each do |jar|
+      versions = @backup.jar_versions(jar)
+
+      unless versions.empty?
+        puts "Versions of backup '#{jar}':"
+
+        versions.each do |version|
+          puts " => #{version.dark_green}: #{Backup::Timestamp.to_str(version)}"
+        end
+      else
+        puts "Versions doesn't exists for jar: #{jar}"
+      end
+    end
+
+    exit
   end
 
-  #TODO: Confirm flag
-  #TODO: Empty destination directory
+  #TODO: Support rescue option as hash
+  if opts.rescue?
+    paths = opts[:rescue].split(" ")
+    jars_list = @backup.jars
 
-  @index = {}
+    include_path = lambda {|path| jars_list.keys.include? path}
 
-  jars_hashes.each do |hash|
-    versions = @backup.jar_versions(hash)
-    # puts "Versions: #{versions}" #FIXME
+    jars_hashes = paths.map do |path|
+      path = File.expand_path path
 
-    last_version = Backup::Timestamp.last_from(versions, @end_date, @start_date)
+      unless include_path[path] or include_path["#{path}/"]
+        puts_fail "Jar \"#{path}\" not exists." 
+      end
 
-    unless last_version.nil?
-      @index[hash] = last_version
+      jars_list[path] || jars_list["#{path}/"]
+    end
+
+    if opts.to?
+      @to = File.expand_path opts[:to]
+      try_create_dir @to
     else
-      error_path = "#{Backup::Jar.hash_to_path(@backup.root_path, hash)}"
-      start_date = Backup::Timestamp.to_s(@start_date)
-      end_date = Backup::Timestamp.to_s(@end_date)
+      @to = "/"
+    end
+
+    #TODO: Confirm flag
+    #TODO: Empty destination directory
+
+    @index = {}
+
+    jars_hashes.each do |hash|
+      versions = @backup.jar_versions(hash)
+      # puts "Versions: #{versions}" #FIXME
+
+      last_version = Backup::Timestamp.last_from(versions, @end_date, @start_date)
 
-      unless @end_date == @start_date
-        puts_fail "Nothing found for #{error_path}, between date: #{start_date} - #{end_date}"
+      unless last_version.nil?
+        @index[hash] = last_version
       else
-        puts_fail "Nothing found for #{error_path}, for date: #{end_date}"
+        error_path = "#{Backup::Jar.hash_to_path(@backup.file_item, @backup.root_path, hash)}"
+        start_date = Backup::Timestamp.to_s(@start_date)
+        end_date = Backup::Timestamp.to_s(@end_date)
+
+        unless @start_date.nil?
+          puts_fail "Nothing found for #{error_path}, between date: #{start_date} - #{end_date}"
+        else
+          puts_fail "Nothing found for #{error_path}, for date: #{end_date}"
+        end
       end
     end
-  end
 
-  @index.each do |hash, timestamp|
-    # puts "#{hash}: #{timestamp}" #FIXME
-    @backup.restore_jar_to(hash, timestamp, @to)
+    @index.each do |hash, timestamp|
+      @backup.restore_jar_to(hash, timestamp, @to)
+    end
+
+    puts "Done!".green
+    exit
   end
 
-  puts "Done!".green
-  exit
-end
+  if !!@config.paths or opts.add?
+    if File.exists? "/var/tmp/encbs.swap"
+      meta = YAML::load open("/var/tmp/encbs.swap").read
+      jar_path, timestamp = meta[:jar_path], meta[:timestamp]
+      dirs = @backup.file_item.dir File.expand_path("../", jar_path)
+      
+      if dirs.include? File.basename(jar_path)
+        meta.delete :timestamp
+        meta.delete :jar_path
+
+        @backup.file_item.create_file_once(
+          "#{jar_path}/#{timestamp}.yml",
+          meta.to_yaml
+        )
+        
+        FileUtils.rm "/var/tmp/encbs.swap"
+      end
+    end
+    
+    if opts.add?
+      paths = opts[:add].split(" ")
+    else
+      paths = @config.paths.split(" ")
+    end
 
-if @config.paths || opts.add?
-  paths = @config.paths.split(" ") || opts[:add].split(" ")
+    paths = paths.map do |path|
+      path = File.expand_path path
+      puts_fail "Path \"#{path}\" not exists." unless File.exists? path
 
-  paths = paths.map do |path|
-    path = File.expand_path path
-    puts_fail "Path \"#{path}\" not exists." unless File.exists? path
+      path
+    end
 
-    path
-  end
+    paths.each do |path|
+      @backup.create! path, @config.increment || opts.increment?
+    end
 
-  paths.each do |path|
-    @backup.create! path, @config.increment || opts.increment?
+    puts "Done!".green
   end
-
-  puts "Done!".green
-end
+ensure
+  remove_lock
+end

data/encbs.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{encbs}
-  s.version = "0.1.3"
+  s.version = "0.2.0.alpha"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
   s.authors = ["Timothy Klim"]
-  s.date = %q{2011-05-14}
+  s.date = %q{2011-05-23}
   s.default_executable = %q{encbs}
   s.description = %q{Simple backup system for pushing into cloud}
   s.email = %q{klimtimothy@gmail.com}
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
     "VERSION",
     "bin/encbs",
     "encbs.gemspec",
+    "lib/archive.rb",
     "lib/backup.rb",
     "lib/backup/file_item.rb",
     "lib/backup/file_item/base.rb",
@@ -58,6 +59,8 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<fog>, [">= 0"])
       s.add_runtime_dependency(%q<slop>, [">= 0"])
       s.add_runtime_dependency(%q<ruby-progressbar>, [">= 0"])
+      s.add_runtime_dependency(%q<lzoruby>, [">= 0"])
+      s.add_development_dependency(%q<rake>, ["= 0.8.4"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.0"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
@@ -65,6 +68,8 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<fog>, [">= 0"])
       s.add_dependency(%q<slop>, [">= 0"])
       s.add_dependency(%q<ruby-progressbar>, [">= 0"])
+      s.add_dependency(%q<lzoruby>, [">= 0"])
+      s.add_dependency(%q<rake>, ["= 0.8.4"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.0"])
       s.add_dependency(%q<rcov>, [">= 0"])
@@ -73,6 +78,8 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<fog>, [">= 0"])
     s.add_dependency(%q<slop>, [">= 0"])
     s.add_dependency(%q<ruby-progressbar>, [">= 0"])
+    s.add_dependency(%q<lzoruby>, [">= 0"])
+    s.add_dependency(%q<rake>, ["= 0.8.4"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.0"])
     s.add_dependency(%q<rcov>, [">= 0"])

data/lib/archive.rb

@@ -0,0 +1,32 @@
+require 'zlib'
+require 'lzoruby'
+
+module GZIP
+  def self.compress string, level
+    z = Zlib::Deflate.new level
+    dst = z.deflate string, Zlib::FINISH
+    z.close
+    dst
+  end
+  
+  def self.decompress string
+    zstream = Zlib::Inflate.new
+    buf = zstream.inflate string
+    zstream.finish
+    zstream.close
+    buf
+  end
+end
+
+class Archive
+  attr_reader :type
+
+  def initialize type
+    fail "Error" unless [:lzo, :gzip].include? type.downcase
+    instance_eval %{@type = #{type.to_s.upcase}}
+  end
+  
+  def method_missing name, *args
+    StringIO.new @type.send(name, *args)
+  end
+end

data/lib/backup/file_item/base.rb

@@ -9,7 +9,7 @@ module Backup
         end
       end
 
-      def stat(file, timestamp = nil)
+      def stat(file)
         files = {}
 
         stat = File.new(file).stat
@@ -18,10 +18,8 @@ module Backup
           :gid => stat.gid,
           :mode => stat.mode
         }
-        files[file][:timestamp] = timestamp if timestamp
-
         unless Dir.exists?(file)
-          files[file][:checksum] = Digest::MD5.hexdigest(File.open(file).read)
+          files[file][:checksum] = Digest::MD5.hexdigest File.open(file, 'rb').read
         end
 
         files

data/lib/backup/file_item/cloud.rb

@@ -3,7 +3,7 @@ require 'backup/file_item/base'
 module Backup
 	module FileItem
     class Cloud < Backup::FileItem::Base
-      attr_reader :key, :secret, :backet, :provider
+      attr_reader :key, :secret, :backet, :provider, :timeout
 
       def initialize(args = {})
         puts_fail "Empty hash in Cloud initialize method" if args.empty?
@@ -12,12 +12,16 @@ module Backup
           puts_fail "'#{arg.to_s.green}' should not be empty" if args[arg].nil?
           instance_eval %{@#{arg} = args[:#{arg}]}
         end
+        @timeout = 60
 
-       try_to_connect_with_cloud 
+        try_to_connect_with_cloud 
+      end
+      
+      def timeout=(time)
+        @timeout = time
       end
 
       def create_directory_once(*directories)
-        # Nothing happen
       end
 
       def create_file_once(file, data)
@@ -44,7 +48,7 @@ module Backup
         files = @directory.files.all(
           :prefix => path,
           :max_keys => 30_000
-        ).map &:key
+        ).map(&:key)
         
         files.map do |item|
           match = item.match(/^#{path}\/([^\/]+#{mask}).*$/)
@@ -64,6 +68,7 @@ module Backup
         begin
           yield
         rescue Exception => e
+          sleep @timeout
           try_to_connect_with_cloud 
 
           yield

data/lib/backup/file_item/local.rb

@@ -1,8 +1,14 @@
 require 'backup/file_item/base'
 
 module Backup
-	module FileItem
+  module FileItem
     class Local < Backup::FileItem::Base
+      attr_reader :timeout
+      
+      def initialize
+        @timeout = 0
+      end
+
       def create_directory_once(*directories)
         directories.each do |path|
           FileUtils.mkdir_p(path) unless Dir.exists?(path)
@@ -10,16 +16,24 @@ module Backup
       end
 
       def create_file_once(file, data)
-        date = date.read if date.is_a? File
-        File.open(file, "w").puts(data) unless File.exists?(file)
+        data = data.read if data.is_a? File or data.is_a? StringIO
+        File.open(file, "wb").puts(data) unless File.exists?(file)
       end
 
       def read_file(file)
-        open(file).read if File.exists? file
+        File.open(file, 'rb').read if File.exists? file
+      end
+      
+      def timeout=(time)
       end
 
       def dir(path, mask = "*")
-        Dir["#{path}/#{mask}"]
+        r_mask = mask.gsub('.', '\.').gsub('*', '[^\/]')
+
+        Dir["#{path}/#{mask}"].map do |item|
+          match = item.match(/^#{path}\/([^\/]+#{r_mask}).*$/)
+          match[1] if match
+        end.compact.uniq
       end
     end
   end

data/lib/backup/jar.rb

@@ -12,82 +12,112 @@ module Backup
       Digest::MD5.hexdigest(@local_path)
     end
 
-    def save(increment = false)
-      unless increment
-        @local_files = hash_local_files
-      else
-        @local_files = {}
-        current_files = hash_local_files
+    def save increment = false, compression = nil
+      @meta_index = {}
+      @local_files = hash_local_files
 
+      if increment
         last_timestamp = Jar.jar_versions(@file_item, @root_path, jar_hash, true).last
 
         if last_timestamp.nil?
           puts_fail "First you must create a full backup for #{@local_path.dark_green}"
         end
 
-        last_index = Jar.fetch_index_for(@file_item, @root_path, jar_hash, last_timestamp)
-
-        current_files.keys.each do |file|
-          @local_files[file] = current_files[file]
+        @last_index = Jar.fetch_index_for(@file_item, @root_path, jar_hash, last_timestamp)
 
-          #TODO: Cut to a new method {
-          current = current_files[file].dup
+        @local_files.keys.each do |file|
+          current = @local_files[file].dup
           current.delete(:timestamp)
-
-          unless last_index[file].nil?
-            backup = last_index[file].dup
+        
+          unless @last_index[file].nil?
+            backup = @last_index[file].dup
             backup.delete(:timestamp)
-
+        
             if (current == backup) or
                (!current[:checksum].nil? and current[:checksum] == backup[:checksum])
-
-              @local_files[file][:timestamp] = last_index[file][:timestamp]
+        
+              @meta_index[file] = @local_files[file]
+              @meta_index[file][:timestamp] = @last_index[file][:timestamp]
             end
           end
-          # }
         end
       end
 
+      unless @key.nil?
+        @meta_index.merge!({
+          :checksum => Base64.encode64(@key.encrypt(@timestamp))
+        })
+      end
+
+      unless compression.nil?
+        @meta_index.merge!({
+          :compression => compression.type.to_s
+        })
+      end
+
       @file_item.create_directory_once meta_jars_path, meta_jar_path, jar_data_path
       @file_item.create_file_once(
-      	"#{meta_jars_path}/#{jar_hash}",
+        "#{meta_jars_path}/#{jar_hash}",
         @file_item.semantic_path(@local_path)
       )
-      @file_item.create_file_once(
-      	"#{meta_jar_path}/#{@timestamp}.yml",
-        @local_files.to_yaml
-			)
 
-      @local_files.select! {|k, v| v[:timestamp] == @timestamp}
       if @file_item.is_a? Backup::FileItem::Cloud
         pbar = ProgressBar.new(
-        	"Uploading",
+          "Uploading",
           @local_files.keys.count
         )
       else
         pbar = ProgressBar.new(
-        	"Copying",
+          "Copying",
           @local_files.keys.count
         )
       end
-
+      
       pbar.bar_mark = '*'
 
-      @local_files.keys.each do |file|
-        unless Dir.exists?(file)
-          data = if @key
-            @key.encrypt_to_stream(File.open(file).read)
-          else
-            File.open(file)
+      #@local_files = @meta_index.select {|k, v| v[:timestamp] == @timestamp if v.is_a? Hash}
+
+      begin
+        @local_files.keys.each do |file|
+          if @meta_index[file].nil?
+            unless Dir.exists?(file)
+              data = StringIO.new File.open(file, 'rb').read
+              checksum = Digest::MD5.hexdigest(data.read)
+
+              data.seek 0
+              data = compression.compress(data.read, 3) unless compression.nil?
+
+              data = @key.encrypt_to_stream(data) if @key
+              
+              @file_item.create_file_once(
+                "#{jar_data_path}/#{@file_item.file_hash file}",
+                data
+              )
+          
+              pbar.inc
+            end
+        
+            @meta_index[file] = @local_files[file]
+            @meta_index[file][:checksum] = checksum
+            @meta_index[file][:timestamp] = @timestamp
           end
-
-          @file_item.create_file_once(
-          	"#{jar_data_path}/#{@file_item.file_hash file}",
-            data
-          )
-
-          pbar.inc
         end
+      rescue Exception => e
+        @meta_index.merge!({
+          :jar_path => meta_jar_path,
+          :timestamp => @timestamp
+        })
+        File.open("/var/tmp/encbs.swap", "w") do |f|
+          f.puts @meta_index.to_yaml
+        end
+        
+        puts
+        puts_fail "Index file has been saved that to allow upload into cloud in next run."
+      else
+        @file_item.create_file_once(
+          "#{meta_jar_path}/#{@timestamp}.yml",
+          @meta_index.to_yaml
+        )
       end
 
       pbar.finish
@@ -96,8 +126,6 @@ module Backup
     def hash_local_files
       files = {}
 
-      puts_verbose "Create index for #{@local_path.dark_green}"
-
       if Dir.exists? @local_path
         matches = Dir.glob(File.join(@local_path, "/**/*"), File::FNM_DOTMATCH)
 
@@ -108,10 +136,10 @@ module Backup
         matches << @local_path
 
         matches.each do |match|
-          files.merge!(@file_item.stat(match, @timestamp))
+          files.merge!(@file_item.stat match)
         end
       else
-        files = @file_item.stat(@local_path, @timestamp)
+        files = @file_item.stat @local_path
       end
 
       files

data/lib/backup.rb

@@ -1,11 +1,14 @@
+require 'stringio'
+
 require 'backup/file_item'
 require 'backup/timestamp'
 require 'backup/jar'
+require 'archive'
 require 'crypto'
 
 module Backup
   class Instance
-    attr_reader :root_path, :timestamp, :hostname, :file_item
+    attr_reader :root_path, :timestamp, :hostname, :file_item, :compression
 
     def initialize(root_path, cloud = false, *args)
       if cloud
@@ -15,22 +18,27 @@ module Backup
       end
 
       @hostname = Socket.gethostname
+      @_root_path = @root_path
       @root_path = "#{root_path}/#{@hostname}"
       @timestamp = Backup::Timestamp.create
     end
 
+    def compression= type
+      @compression = Archive.new type.to_sym
+    end
+
     def hostname=(host)
       @hostname = host
-      @root_path = "#{root_path}/#{@hostname}"
+      @root_path = "#{@_root_path}/#{@hostname}"
     end
 
-    def key=(path)
-      @key = Crypto::Key.from_file(path)
+    def rsa_key(path, size)
+      @key = Crypto::Key.from_file(path, size)
     end
 
     def create!(local_path, increment = false)
       jar = Jar.new(@file_item, @root_path, local_path, @key)
-      jar.save(increment)
+      jar.save(increment, @compression)
     end
 
     def jars
@@ -44,8 +52,24 @@ module Backup
     def restore_jar_to(hash, timestamp, to)
       files = Jar.fetch_index_for(@file_item, @root_path, hash, timestamp)
 
+      unless files[:checksum].nil?
+        if @key.nil? or
+           @key.decrypt(Base64.decode64(files[:checksum])) != timestamp
+
+          puts_fail "Checksum does not match."
+        end
+        files.delete :checksum
+      end
+
+      unless files[:compression].nil?
+        compression = Archive.new files[:compression].to_sym
+        files.delete :compression
+      else
+        compression = nil
+      end
+
       pbar = ProgressBar.new(
-      	"Restoring",
+        "Restoring",
         files.keys.length
       )
       pbar.bar_mark = '*'
@@ -67,22 +91,27 @@ module Backup
             restore_file,
             file_ok[:uid],
             file_ok[:gid],
-        		current_file[:uid],
+            current_file[:uid],
             current_file[:gid]
           )
         else
           try_create_dir(File.dirname restore_file)
 
           begin
-            File.open(restore_file, "w") do |f|
+            File.open(restore_file, "wb") do |f|
               f.chmod current_file[:mode]
               f.chown current_file[:uid], current_file[:gid]
 
               remote_path = "#{@root_path}/#{hash}/#{current_file[:timestamp]}"
               remote_path += "/#{@file_item.file_hash file}"
 
-              data = @file_item.read_file remote_path
-							data = @key.decrypt_from_stream data if @key
+              data = @file_item.read_file(remote_path)
+              data = @key.decrypt_from_stream data if @key
+
+              unless compression.nil?
+                data = compression.decompress(data.chomp).read
+              end
+
               f.puts data
             end
 
@@ -90,7 +119,7 @@ module Backup
           
             check_mode(restore_file, file_ok[:mode], current_file[:mode])
             check_rights(
-            	restore_file,
+              restore_file,
               file_ok[:uid],
               file_ok[:gid],
               current_file[:uid],

data/lib/crypto.rb

@@ -11,13 +11,14 @@ module Crypto
   end
   
   class Key
-    def initialize(data)
+    def initialize(data, size)
       @public = (data =~ /^-----BEGIN (RSA|DSA) PRIVATE KEY-----$/).nil?
       @key = OpenSSL::PKey::RSA.new(data)
+      @size = (size == 4096 ? 512 : 256)
     end
   
-    def self.from_file(filename)    
-      self.new File.read( filename )
+    def self.from_file(filename, size = 4096)
+      self.new(File.read(filename), size)
     end
 
     def encrypt_to_stream(data)
@@ -28,15 +29,16 @@ module Crypto
         encrypt_data << encrypt(buf)
       end
   
-      encrypt_data.seek(0)
+      encrypt_data.seek 0
       encrypt_data
     end
 
     def decrypt_from_stream(data)
-      encrypt_data = StringIO.new data
+      encrypt_data = StringIO.new(data.chomp)
+      encrypt_data.seek 0
       decrypt_data = ""
   
-      while buf = encrypt_data.read(256) do
+      while buf = encrypt_data.read(@size) do
         decrypt_data += decrypt(buf)
       end
 
@@ -45,10 +47,14 @@ module Crypto
 
     def encrypt(text)
       @key.send("#{key_type}_encrypt", text)
+    rescue Exception => e
+      puts_fail "RSA encrypt error: #{e.message}"
     end
     
     def decrypt(text)
       @key.send("#{key_type}_decrypt", text)
+     rescue Exception => e
+       puts_fail "RSA decrypt error: #{e.message}"
     end
   
     def private?

data/lib/encbsconfig.rb

@@ -1,21 +1,21 @@
 class EncbsConfig
-  attr_reader :paths, :bucket, :colorize, :hostname, :increment, :key,
-              :secret, :token, :verbose
+  attr_reader :paths, :bucket, :colorize, :compression, :hostname, :increment,
+              :key, :secret, :size, :token, :timeout, :verbose
 
   def initialize
     @paths = ""
   end
 
   def load(path)
-    [:bucket, :colorize, :hostname, :increment, :key, :secret,
-      :token, :verbose].each {|attr| eval "@#{attr} = nil"}
+    [:bucket, :colorize, :compression, :hostname, :increment, :key, :secret,
+      :size, :token, :timeout, :verbose].each {|attr| eval "@#{attr} = nil"}
 
     @paths = ""
 
-    eval "#{open(path).read}"
+    instance_eval "#{open(path).read}"
   end
 
-  def use_hostname(attr)
+  def use_hostname attr
     @hostname = attr
   end
 
@@ -27,27 +27,39 @@ class EncbsConfig
     @colorize = true
   end
 
-  def public_key(attr)
+  def public_key attr
     @token = attr
   end
 
   def increment!
     @increment = true
   end
+  
+  def use_compression attr
+    @compression = attr
+  end
 
-  def aws_key(attr)
+  def aws_key attr
     @key = attr
   end
 
-  def aws_secret(attr)
+  def aws_secret attr
     @secret = attr
   end
 
-  def aws_bucket(attr)
+  def aws_bucket attr
     @bucket = attr
   end
 
+  def key_size attr
+    @size = attr
+  end
+
   def verbose!
     @verbose = true
   end
+  
+  def set_timeout attr
+    @timeout = attr
+  end
 end

data/lib/helpers.rb

@@ -1,5 +1,5 @@
 def puts_fail(msg)
-  STDERR.puts "#{"Error: ".red}#{msg}"
+  STDERR.puts "#{"Error! ".red}#{msg}"
 
   exit msg.length
 end
@@ -12,15 +12,6 @@ def print_verbose(msg)
   print msg if $PRINT_VERBOSE
 end
 
-def safe_require(&block)
-  yield
-rescue Exception => e
-  puts_fail %Q{This script use these gems: fog, slop.
-    Make sure that you have them all.
-    If you don't have, you may install them: $ gem install fog slop ruby-progressbar
-  }
-end
-
 def try_create_dir(dir)
   begin
     FileUtils.mkdir_p dir unless Dir.exists? dir
@@ -41,6 +32,20 @@ def check_rights(file, first_uid, first_gid, second_uid, second_gid)
   end
 end
 
+def create_lock
+  open("/tmp/encbs.lock", "w") do |f|
+    f.puts Process.pid
+  end
+end
+
+def remove_lock
+  FileUtils.rm "/tmp/encbs.lock" if File.exists? "/tmp/encbs.lock"
+end
+
+def lock_exists?
+  File.exists? "/tmp/encbs.lock"
+end
+
 class String
   def red
     colorize(self, "\e[1m\e[31m")

data/test/helper.rb

@@ -9,6 +9,9 @@ require 'socket'
 require 'helpers'
 require 'progressbar'
 require 'test/unit'
+require 'base64'
+require 'lzoruby'
+require 'zlib'
 
 require 'backup'
 

data/test/test_backup_file_item.rb

@@ -12,30 +12,22 @@ class BackupFileItemTest < Test::Unit::TestCase
   end
 
   def test_file_stat
-    file = @file_item.stat(
-      __FILE__,
-      Backup::Timestamp.create
-    )
+    file = @file_item.stat __FILE__
     key = file.keys.first
 
     assert_not_nil file[key][:uid]
     assert_not_nil file[key][:gid]
     assert_not_nil file[key][:mode]
     assert_not_nil file[key][:checksum]
-    assert_not_nil file[key][:timestamp]
   end
 
   def test_directory_stat
-    file = @file_item.stat(
-      File.dirname(__FILE__),
-      Backup::Timestamp.create
-    )
+    file = @file_item.stat File.dirname(__FILE__)
     key = file.keys.first
 
     assert_not_nil file[key][:uid]
     assert_not_nil file[key][:gid]
     assert_not_nil file[key][:mode]
     assert_nil file[key][:checksum]
-    assert_not_nil file[key][:timestamp]
   end
 end

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: encbs
 version: !ruby/object:Gem::Version 
-  prerelease: 
-  version: 0.1.3
+  prerelease: 6
+  version: 0.2.0.alpha
 platform: ruby
 authors: 
 - Timothy Klim
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-14 00:00:00 +04:00
+date: 2011-05-23 00:00:00 +04:00
 default_executable: encbs
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -47,8 +47,30 @@ dependencies:
   prerelease: false
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: bundler
+  name: lzoruby
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rake
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 0.8.4
+  type: :development
+  prerelease: false
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -56,10 +78,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *id004
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: jeweler
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -67,10 +89,10 @@ dependencies:
         version: 1.6.0
   type: :development
   prerelease: false
-  version_requirements: *id005
+  version_requirements: *id007
 - !ruby/object:Gem::Dependency 
   name: rcov
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -78,7 +100,7 @@ dependencies:
         version: "0"
   type: :development
   prerelease: false
-  version_requirements: *id006
+  version_requirements: *id008
 description: Simple backup system for pushing into cloud
 email: klimtimothy@gmail.com
 executables: 
@@ -96,6 +118,7 @@ files:
 - VERSION
 - bin/encbs
 - encbs.gemspec
+- lib/archive.rb
 - lib/backup.rb
 - lib/backup/file_item.rb
 - lib/backup/file_item/base.rb
@@ -128,16 +151,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -3909970263344363647
+      hash: -2056087500533480585
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version 
-      version: "0"
+      version: 1.3.1
 requirements: []
 
 rubyforge_project: