incremental_backup 0.0.1 → 0.1.0

data/.gitignore

@@ -15,3 +15,7 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+
+live_example/log
+live_example/log.0
+live_example/log.1

data/Guardfile

@@ -0,0 +1,11 @@
+guard 'rspec' do
+
+  # Run spec if spec changes
+  watch(%r{^spec/.+_spec\.rb$})
+
+  # Run all specs if anything changes
+  watch(%r{^lib/(.+)\.rb$}) { 'spec' }
+  watch(%w[Gemfile.lock]) { 'spec' }
+  watch('spec/spec_helper.rb')  { 'spec' }
+end
+

data/README.md

@@ -1,23 +1,31 @@
-# IncrementalBackup (NOT YET WORKING...)
+# incremental\_backup
 
-Create incremental backups via ssh and rsync
+Creates incremental backups via ssh and rsync.
 
-## Installation
+## Description
 
-Add this line to your application's Gemfile:
+incremental\_backup is a tool to easily create incremental backups to a local directory or to a remote server using ssh and rsync.
 
-    gem 'incremental_backup'
+The backup is by default performed every hour of a determined set of files and folders. The backup is easily restorable from every hour in the last day, every day in the last week, every week in the last month, every month in the past three months and every year forever. These default settings can be changed to suit your needs.
 
-And then execute:
+## Dependencies
 
-    $ bundle
+* Ruby (tested on 1.9.3)
+* ssh
+* rsync
 
-Or install it yourself as:
+## Installation
 
     $ gem install incremental_backup
 
 ## Usage
 
+The recommended usage is to create a small ruby application with a gemfile. Take a look at [the example](https://github.com/lasseebert/incremental_backup/tree/master/live_example).
+
+## Cron
+It's recommended to setup a cron job to handle scheduling of the backup.
+
+The cron job can be triggered as often as you would like to, the script still only at most perform one backup per hour. This is useful if your computer is usually not online for more than an hour.
 
 ## Contributing
 

data/Rakefile

@@ -1 +1,4 @@
 require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')

data/incremental_backup.gemspec

@@ -10,10 +10,18 @@ Gem::Specification.new do |gem|
   gem.email         = ["lasseebert@gmail.com"]
   gem.description   = %q{(NOT YET WORKING...) incremental_backup can make incremental backups by hour/day/week/month remotely through ssh and rsync}
   gem.summary       = %q{(NOT YET WORKING...) incremental_backup can make incremental backups by hour/day/week/month remotely through ssh and rsync}
-  gem.homepage      = ""
+  gem.homepage      = 'https://github.com/lasseebert/incremental_backup'
 
   gem.files         = `git ls-files`.split($/)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
+
+  gem.add_development_dependency 'rspec', '~> 2.12'
+  gem.add_development_dependency 'rspec-mocks', '~> 2.12'
+  gem.add_development_dependency 'guard-rspec', '~> 2.4'
+  gem.add_development_dependency 'rb-inotify', '~> 0.8.8'
+  gem.add_development_dependency 'rake', '~> 10.0.3'
+
+  gem.add_dependency 'active_attr', '~> 0.7'
+  gem.add_dependency 'net-ssh', '~> 2.6'
 end

data/lib/incremental_backup/lock.rb

@@ -0,0 +1,56 @@
+require 'fileutils'
+
+module IncrementalBackup
+  class Lock
+
+    LOCK_FILENAME = 'lock_{task_id}'
+
+    attr_accessor :failed
+
+    # Should not be called from other places than Lock.create
+    def initialize task
+      @task = task
+
+      if File.exists? path
+        self.failed = true
+      else
+        FileUtils.touch path
+      end
+    end
+
+    # Obtains lock, run block, release lock
+    def self.create task, &block
+      task.logger.info 'Obtaining lock...'
+
+      lock = Lock.new task
+
+      if lock.failed
+        task.logger.info 'Lock can not be obtained. Exiting!'
+        return
+      end
+
+      task.logger.info 'Obtained lock'
+
+      begin
+        yield
+      ensure
+        task.logger.info 'Releasing lock...'
+        lock.release
+        task.logger.info 'Released lock'
+      end
+
+      true
+    end
+
+    # Release lock
+    def release
+      FileUtils.rm path
+    end
+
+    private
+
+    def path
+      @path ||= File.join @task.settings.settings_path, LOCK_FILENAME.sub(/\{task_id\}/, @task.settings.task_id)
+    end
+  end
+end

data/lib/incremental_backup/rsync.rb

@@ -0,0 +1,62 @@
+require 'pty'
+
+module IncrementalBackup
+  class Rsync
+    def self.execute(logger, local_path, remote_path, options)
+      # TODO: Only use long options
+      rsync_options = {
+        "-azprvP" => nil,
+        "--delete" => nil,
+        "--delete-excluded" => nil,
+        "--modify-window" => '2',
+        "--force" => nil,
+        "--ignore-errors" => nil,
+        "--stats" => nil
+      }
+      rsync_options["--exclude-from"] = options[:exclude_file] if options[:exclude_file]
+      rsync_options["--link-dest"] = options[:link_dest] if options[:link_dest]
+
+      rsync_options = rsync_options.map{|key, value| "#{key}#{value ? "=#{value}" : ''}" }.join(' ')
+
+      rsync_command = "rsync #{rsync_options} -e ssh #{local_path} #{remote_path}"
+      if options[:max_download_speed] || options[:max_upload_speed]
+        trickle = "trickle"
+        trickle += " -d #{options[:max_download_speed]}" if options[:max_download_speed]
+        trickle += " -u #{options[:max_upload_speed]}" if options[:max_upload_speed]
+        rsync_command = "#{trickle} #{rsync_command}"
+      end
+
+      execute_shell logger, rsync_command
+
+    end
+
+    private
+
+    def self.execute_shell(logger, command)
+      begin
+        last_line = nil
+        PTY.spawn(command) do |stdout, stdin, pid|
+          begin
+            stdout.each do |line| 
+              logger.info line.gsub(/\n/, '')
+              last_line = line
+            end
+          rescue Errno::EIO => e
+            # Can't tell the difference between process being killed
+            # or process ended because it was done
+            unless last_line =~ /total size is/
+              logger.error "Errno::EIO"
+              logger.error e
+              raise e
+            end
+          end
+        end
+      rescue PTY::ChildExited => e
+        logger.error "PTY::ChildExited"
+        logger.error e
+        raise e
+      end
+    end
+
+  end
+end

data/lib/incremental_backup/task.rb

@@ -0,0 +1,166 @@
+require 'logger'
+require 'net/ssh'
+
+module IncrementalBackup
+  class Task
+
+    LOG_FILENAME = 'log'
+    NUM_LOG_FILES = 10
+
+    attr_accessor :settings
+
+    def initialize(&block)
+      self.settings = TaskSettings.new
+      yield settings
+    end
+
+    # Perform the backup
+    def run
+
+      # Validate - this will throw an exception if settings are not valid
+      validate_settings
+
+      # Run everything inside a lock, ensuring that only one instance of this
+      # task is running.
+      Lock.create(self) do
+
+        # Find the schedule to run
+        schedule = find_schedule
+        unless schedule
+          logger.info "No backup needed - exiting"
+          return
+        end
+
+        logger.info "Starting #{schedule} backup to #{settings.remote_server}"
+
+        # Paths and other options
+        timestamp = Time.now.strftime('backup_%Y-%m-%d-T%H-%M-%S')
+        current_path = File.join(settings.remote_path, 'current')
+        progress_path = File.join(settings.remote_path, 'incomplete')
+        complete_path = File.join(schedule_path(schedule), timestamp)
+        login = "#{settings.remote_user}@#{settings.remote_server}"
+        rsync_path = "#{login}:#{progress_path}"
+
+        # Make schedule folder
+        execute_ssh "mkdir --verbose --parents #{schedule_path schedule}"
+
+        # Rsync
+        Rsync.execute(logger, settings.local_path, rsync_path, {
+          exclude_file: settings.exclude_file,
+          link_dest: current_path,
+          max_upload_speed: settings.max_upload_speed,
+          max_download_speed: settings.max_download_speed
+        })
+
+        # shuffle backups around
+        logger.info "Do the backup shuffle"
+        execute_ssh [
+          "mv --verbose            #{progress_path} #{complete_path}",
+          "rm --verbose --force    #{current_path}",
+          "ln --verbose --symbolic #{complete_path} #{current_path}",
+        ]
+
+        delete_old_backups schedule
+
+        logger.info "#{schedule} backup done"
+      end
+
+    rescue Exception => exception
+      logger.error exception.message
+      logger.error exception.backtrace
+    end
+
+    def logger
+      @logger ||= Logger.new(File.join(settings.settings_path, LOG_FILENAME), NUM_LOG_FILES)
+    end
+
+    private
+
+    def schedule_path(schedule)
+      File.join(settings.remote_path, schedule.to_s)
+    end
+
+    def delete_old_backups(schedule)
+      backups = list_backup_dir schedule
+      backups_to_keep = settings.send("#{schedule}_backups")
+      backups.sort!
+      backups_to_delete = backups - backups.last(backups_to_keep)
+      if backups_to_delete.any?
+        if backups_to_delete.length == 1
+          logger.info "Deleting old backup #{backups_to_delete.first}"
+        else
+          logger.info "Deleting #{backups_to_delete.length} old backups"
+        end
+        execute_ssh(backups_to_delete.map { |path| "rm --force --recursive #{path}" })
+      end
+    end
+
+    def list_backup_dir(schedule)
+      execute_ssh("mkdir -p #{schedule_path schedule}")
+      execute_ssh("find #{schedule_path schedule} -maxdepth 1 -mindepth 1").split("\n")
+    end
+
+    def validate_settings
+      unless settings.valid?
+        logger.error "Invalid settings:"
+        settings.errors.full_messages.each do |message|
+          logger.error message
+        end
+        throw "Invalid settings. Check the log file"
+      end
+      logger.info "Settings validated"
+    end
+
+    # Runs one ore more commands remotely via ssh
+    def execute_ssh(commands)
+      commands = [commands] unless commands.is_a? Array
+      result = ""
+      Net::SSH.start settings.remote_server, settings.remote_user do |ssh|
+        commands.each do |command|
+          was_error = false
+          logger.info "ssh: #{command}"
+          ssh.exec! command do |channel, stream, data|
+            case stream
+            when :stdout
+              logger.info data
+              result += "#{data}\n" unless data.empty?
+            when :stderr
+              logger.error data
+              was_error = true
+            end
+          end
+          throw "Exception during ssh, look in log file" if was_error
+        end
+      end
+      result
+    end
+
+    # Find out which schedule to run
+    def find_schedule
+      minutes = {
+        # If a cron job is run hourly it can be off by a couple of seconds
+        # from the last run. Set hourly to 58 minutes
+        hourly: 58, 
+        daily: 24*60,
+        weekly: 7*24*60,
+        monthly: 30*24*60,
+        yearly: 365*24*60
+      }
+
+      now = DateTime.now
+      [:yearly, :monthly, :weekly, :daily, :hourly].each do |schedule|
+        list = list_backup_dir schedule
+        date = list.map { |path| parse_backup_dir_name path, now.offset }.max
+        return schedule if !date || (now - date) * 24 * 60 >= minutes[schedule]
+      end
+
+      nil
+    end
+
+    def parse_backup_dir_name(dir, offset)
+      if dir =~ /(\d{4})-(\d{2})-(\d{2})-T(\d{2})-(\d{2})-(\d{2})$/
+        DateTime.new($1.to_i, $2.to_i, $3.to_i, $4.to_i, $5.to_i, $6.to_i, offset)
+      end
+    end
+  end
+end

data/lib/incremental_backup/task_settings.rb

@@ -0,0 +1,45 @@
+require 'active_attr'
+module IncrementalBackup
+  class TaskSettings
+    include ActiveAttr::Model
+
+    # Max backups to keep
+    attribute :hourly_backups, default: 24
+    attribute :daily_backups, default: 7
+    attribute :weekly_backups, default: 5
+    attribute :monthly_backups, default: 12
+    attribute :yearly_backups, default: 100
+
+    # Unique task id, must be a valid file name
+    attribute :task_id
+
+    # Path to store log file, lock file and more
+    attribute :settings_path
+
+    # Path to the exclude file, containing all files and dirs not to include
+    attribute :exclude_file
+
+    # Paths
+    attribute :local_path
+    attribute :remote_path
+
+    # Login
+    attribute :remote_server
+    attribute :remote_user
+
+    # Throttle
+    attribute :max_upload_speed
+    attribute :max_download_speed
+
+    # Validation
+    validates :task_id, presence: true
+    validates :settings_path, presence: true
+    validates :remote_server, presence: true
+    validates :local_path, presence: true
+    validates :remote_path, presence: true
+    validates :remote_user, presence: true
+
+    
+
+  end
+end

data/lib/incremental_backup/version.rb

@@ -1,3 +1,3 @@
 module IncrementalBackup
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/lib/incremental_backup.rb

@@ -1,5 +1,5 @@
-require "incremental_backup/version"
-
-module IncrementalBackup
-  # Your code goes here...
-end
+require_relative 'incremental_backup/version'
+require_relative 'incremental_backup/task'
+require_relative 'incremental_backup/task_settings'
+require_relative 'incremental_backup/lock'
+require_relative 'incremental_backup/rsync'

data/live_example/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gem 'incremental_backup', '~> 0.0'
+

data/live_example/example_task.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'incremental_backup'
+
+task = IncrementalBackup::Task.new do |settings|
+
+  # Defines the id of the task. This is used to ensure that the backup cannot
+  # be running simultaneously in two processes. The name must be a valid file
+  # name
+  settings.task_id = 'my_backup_task'
+
+  # Defines how many backups to keep
+  settings.hourly_backups= 24
+  settings.daily_backups = 7
+  settings.weekly_backups = 5
+  settings.monthly_backups = 12
+  settings.yearly_backups = 100
+
+  # This is where all helper files are saved. These include a log file, a lock
+  # file and a file remembering the dates of the the last backups. This would
+  # typically be a hidden directory in the home folder, e.g. ~/.incremental_backup
+  settings.settings_path = File.dirname(__FILE__)
+
+  # Login options
+  settings.remote_server = 'hostname or ip'
+  settings.remote_user = 'username on server'
+
+  # Paths
+  settings.local_path = '~'
+  settings.remote_path = '~/backup'
+
+  # Exclude all the files and folder listed in this file. This file is in default rsync syntax
+  settings.exclude_file = File.join File.dirname(__FILE__), 'exclude.file'
+end
+
+task.run

data/live_example/exclude.file

@@ -0,0 +1,11 @@
+.cache/
+.rbenv/
+.rvm/
+Downloads/
+Dropbox/
+
+tmp
+cache
+development.log
+development.sqlite3
+test.log

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rspec'
+require 'incremental_backup'
+
+RSpec.configure do |config|
+  config.color_enabled = true
+  config.formatter     = 'documentation'
+end

data/spec/unit/task_settings_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+
+describe IncrementalBackup::TaskSettings do
+  it 'should not be valid without settings_path' do
+    settings = IncrementalBackup::TaskSettings.new
+    settings.should_not be_valid
+  end
+end

data/spec/unit/task_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+describe IncrementalBackup::Task do
+  it 'should be configured' do
+    task = IncrementalBackup::Task.new do |config|
+      config.hourly_backups = 1
+      config.daily_backups = 2
+      config.weekly_backups = 3
+      config.monthly_backups = 4
+    end
+    task.settings.hourly_backups.should == 1
+    task.settings.daily_backups.should == 2
+    task.settings.weekly_backups.should == 3
+    task.settings.monthly_backups.should == 4
+  end
+
+  it 'throws error with invalid settings do' do
+
+    task = IncrementalBackup::Task.new do
+    end
+
+    logger_double = double('logger')
+    logger_double.should_receive(:error).at_least(:once)
+    task.stub(:logger).and_return logger_double
+
+    task.run
+  end
+
+  it 'runs' do
+    task = IncrementalBackup::Task.new do |settings|
+      settings.task_id = "test_test"
+      settings.settings_path = "some/path"
+      settings.remote_server = "myserver"
+      settings.local_path = "some/path"
+      settings.remote_path = "some/path"
+      settings.remote_user = "me"
+    end
+
+    logger_double = double('logger')
+    logger_errors = []
+    logger_double.stub(:error) { |message| logger_errors << message }
+    logger_double.stub(:info)
+    task.stub(:logger).and_return logger_double
+
+    task.stub(:execute_ssh)
+    task.stub(:list_backup_dir).and_return([
+      "/hourly/backup_2013-02-14-T10-39-27",
+      "/hourly/backup_2013-02-14-T10-21-43",
+      "/hourly/backup_2013-02-14-T11-02-46",
+      "/hourly/backup_2013-02-14-T10-35-04",
+      "/hourly/backup_2013-02-14-T10-28-11",
+      "/hourly/backup_2013-02-14-T10-33-24"
+    ])
+
+    IncrementalBackup::Lock.stub(:create).and_yield
+    IncrementalBackup::Rsync.stub(:execute)
+
+    task.run
+
+    throw logger_errors.join("\n") if logger_errors.any?
+  end
+end

data/spec/unit/version_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe 'version' do
+  it 'should have a version' do
+    IncrementalBackup::VERSION.should_not be_nil
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: incremental_backup
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,120 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-21 00:00:00.000000000 Z
-dependencies: []
+date: 2013-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.12'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.12'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: rb-inotify
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.8
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.0.3
+- !ruby/object:Gem::Dependency
+  name: active_attr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
 description: (NOT YET WORKING...) incremental_backup can make incremental backups
   by hour/day/week/month remotely through ssh and rsync
 email:
@@ -21,13 +133,25 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
+- Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - incremental_backup.gemspec
 - lib/incremental_backup.rb
+- lib/incremental_backup/lock.rb
+- lib/incremental_backup/rsync.rb
+- lib/incremental_backup/task.rb
+- lib/incremental_backup/task_settings.rb
 - lib/incremental_backup/version.rb
-homepage: ''
+- live_example/Gemfile
+- live_example/example_task.rb
+- live_example/exclude.file
+- spec/spec_helper.rb
+- spec/unit/task_settings_spec.rb
+- spec/unit/task_spec.rb
+- spec/unit/version_spec.rb
+homepage: https://github.com/lasseebert/incremental_backup
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -41,7 +165,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3434899829866863485
+      hash: -952483362192499202
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -50,7 +174,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3434899829866863485
+      hash: -952483362192499202
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -58,4 +182,8 @@ signing_key:
 specification_version: 3
 summary: (NOT YET WORKING...) incremental_backup can make incremental backups by hour/day/week/month
   remotely through ssh and rsync
-test_files: []
+test_files:
+- spec/spec_helper.rb
+- spec/unit/task_settings_spec.rb
+- spec/unit/task_spec.rb
+- spec/unit/version_spec.rb