yob 1.0.0

data/README.rdoc

@@ -0,0 +1,27 @@
+= YOB
+
+An online backup system for MySQL and PostgreSQL, using GnuPG for encryption and AWS S3 for storage.
+
+YOB is pluggable, so it's easy to make different database, encryption and storage engines.
+
+== Install
+
+You'll need a few libraries installed.
+
+  aptitude install libgpgme11-dev libsqlite3-dev libxml2-dev libxslt-dev libopenssl-ruby1.8
+
+Install yob from rubygems:
+
+  gem install yob
+
+Then:
+
+  useradd --home-dir /var/lib/yob --create-home yob
+
+If you're using MySQL, create ~yob/.my.cnf with mysql username and password and
+
+  GRANT select,reload,super,replication client ON *.* TO yob@localhost IDENTIFIED BY 'somepassword'
+
+== Licence
+
+Licenced under the MIT licence.  Copyright 2011-2012 YouDo Limited.

data/bin/yob

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+require 'yob'
+
+def run
+  yob = Yob.new(Yob::Configuration.new(ARGV))
+  case ARGV[0]
+  when 'full'
+    yob.backup :full_backup
+  when 'partial'
+    yob.backup :partial_backup
+  else
+    puts "specify 'full' or 'partial' on the command line"
+    exit 1
+  end
+rescue Yob::Configuration::Error => e
+  $stderr.puts e.message
+  exit 1
+end
+
+run

data/lib/yob.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+
+class Yob
+  def initialize(configuration)
+    @configuration = configuration
+    @store = @configuration.store_handler_class.new(@configuration)
+    @database = @configuration.database_handler_class.new(@configuration)
+    @encryption = @configuration.encryption_handler_class.new(@configuration)
+  end
+
+  def backup(type)
+    @database.send(type) do |filename, rd|
+      storage_pipe = @store.storage_input_pipe(filename)
+      if rd
+        @encryption.encrypt(rd, storage_pipe)
+        storage_pipe.close
+        nil
+      else
+        @encryption.encryption_input_pipe(storage_pipe)
+      end
+    end
+    Process.waitall
+  end
+end
+
+require 'yob/configuration'
+require 'yob/database'
+require 'yob/encrypt'
+require 'yob/store'

data/lib/yob/configuration.rb

@@ -0,0 +1,66 @@
+require 'yaml'
+require 'optparse'
+
+class Yob::Configuration
+  Error = Class.new(StandardError)
+
+  def initialize(argv = [])
+    command_line_options = {}
+    configuration_path = ["/etc/yob.yml", File.dirname(__FILE__) << "/yob.yml"].detect {|file| File.exists?(file)}
+
+    OptionParser.new(argv) do |opts|
+      opts.banner = "Usage: yob [options] full|partial"
+
+      opts.on("--debug", "Print debug output") do
+        command_line_options["debug"] = true
+      end
+
+      opts.on("-c", "--config CONFIG_FILE", String, "Specify a path to the configuration file") do |file|
+        configuration_path = file
+      end
+
+      opts.on_tail("-h", "--help", "Show this message") do
+        puts opts
+        exit
+      end
+    end.parse!
+
+    raise Error, "could not locate a yob.yml file in any of the normal locations" unless configuration_path
+
+    configuration = YAML.load(IO.read(configuration_path))["configuration"]
+    @hash = configuration.merge(command_line_options)
+  end
+
+  def [](key)
+    @hash[key.to_s]
+  end
+
+  def fetch(key, default)
+    @hash[key.to_s] || default
+  end
+
+  def method_missing(key, *args)
+    key = key.to_s
+    raise Error, "Required configuration parameter '#{key}' not specified in configuration file" unless @hash.member?(key)
+    @hash[key]
+  end
+
+  def database_handler_class
+    case database_handler.downcase
+    when "postgresql" then Yob::Database::Postgresql
+    when "mysql"      then Yob::Database::Mysql
+    else                   raise "Unrecognised database_handler"
+    end
+  end
+
+  def store_handler_class
+    case storage_handler.downcase
+    when "aws"  then Yob::Store::AWS
+    else             raise "Unrecognised storage_handler"
+    end
+  end
+
+  def encryption_handler_class
+    Yob::Encrypt::GnuPG
+  end
+end

data/lib/yob/database.rb

@@ -0,0 +1,128 @@
+#!/usr/bin/env ruby
+
+module Yob::Database
+  class Base
+    attr_reader :configuration
+
+    def initialize(configuration)
+      @configuration = configuration
+    end
+
+    protected
+    def random_filename_string
+      (0..15).inject(".") {|s, i| s << (97 + rand(26)).chr} if configuration["randomise_filename"]
+    end
+  end
+
+  class Mysql < Base
+    def initialize(configuration)
+      super
+
+      unless File.directory?(configuration.mysql_log_directory)
+        raise Yob::Configuration::Error, "mysql_log_directory does not exist"
+      end
+
+      unless File.exists?(configuration.mysqldump_executable)
+        raise Yob::Configuration::Error, "mysqldump_executable does not exist"
+      end
+    end
+
+    def full_backup
+      writer = yield Time.now.strftime("%Y%m%d-%H%M%S#{random_filename_string}.sql.gpg"), nil
+      begin
+        puts "[Database::Mysql] dumping all databases to SQL..."
+        system("#{configuration.mysqldump_executable} --all-databases --default-character-set=utf8 --skip-opt --create-options --add-drop-database --extended-insert --flush-logs --master-data --quick --single-transaction >&#{writer.fileno}")
+        puts "[Database::Mysql] dump completed"
+      ensure
+        writer.close
+      end
+    end
+
+    def partial_backup
+      require 'sqlite3'
+
+      @db = SQLite3::Database.new(configuration.fetch("file_database", "#{File.dirname(__FILE__)}/yob.db"))
+      @db.execute("CREATE TABLE IF NOT EXISTS files (id INTEGER PRIMARY KEY AUTOINCREMENT, filename varchar(255) unique not null, file_size integer not null, file_time datetime not null)")
+
+      files = Dir["#{configuration.mysql_log_directory}/mysql-bin.*"]
+      files.each do |filename|
+        next if filename[-5..-1] == 'index'
+
+        stats = File.stat(filename)
+        file_time = stats.mtime.strftime("%Y-%m-%d %H:%M:%S")
+
+        row = @db.get_first_row("SELECT id, file_size, file_time FROM files WHERE filename = ?", filename)
+        if row && row[1].to_i == stats.size && row[2] == file_time
+          puts "[Database::Mysql] skipping #{filename}" if @configuration["debug"]
+        else
+          File.open(filename, "r") do |logfile|
+            yield "#{File.basename(filename)}#{random_filename_string}.gpg", logfile
+          end
+
+          if row
+            @db.execute("UPDATE files SET file_size = ?, file_time = ? WHERE id = ?", stats.size, file_time, row[0])
+          else
+            stmt = @db.prepare("INSERT INTO files (filename, file_size, file_time) VALUES (?, ?, ?)")
+            stmt.execute(filename, stats.size, file_time)
+          end
+        end
+      end
+    end
+  end
+
+  class Postgresql < Base
+    def initialize(configuration)
+      super
+      require 'archive/tar/minitar'
+    end
+
+    def full_backup
+      require 'pg'
+
+      connect_and_execute "SELECT pg_start_backup('yob')"
+
+      begin
+        writer = yield Time.now.strftime("%Y%m%d-%H%M%S#{random_filename_string}.tar.gpg"), nil
+        tar = Archive::Tar::Minitar::Output.new(writer)
+        begin
+          Find.find(configuration.postgresql_data_directory) do |entry|
+            Archive::Tar::Minitar.pack_file(entry, tar) unless entry.include?("/pg_xlog/")
+          end
+        ensure
+          tar.close
+        end
+        writer.close
+      ensure
+        connect_and_execute "SELECT pg_stop_backup()"
+      end
+    end
+
+    def partial_backup
+      raise "yob partial must be called with two more arguments - the full path and the destination filename - for a postgresql partial backup" unless ARGV.length == 3
+      full_path = ARGV[1]
+      destination_filename = ARGV[2]
+
+      File.open(full_path, "r") do |file|
+        yield "#{destination_filename}#{random_filename_string}", file
+      end
+    end
+
+    protected
+    def connect_and_execute(sql)
+      hostname = @configuration.fetch("postgresql_hostname", "localhost")
+      port = @configuration.fetch("postgresql_port", 5432)
+
+      connection = PGconn.connect(
+        hostname, # nil if UNIX socket
+        port,     # nil if UNIX socket
+        '',       # options
+        '',       # unused by library
+        @configuration.fetch("postgresql_default_database", "postgres"),
+        @configuration["postgresql_username"],
+        @configuration["postgresql_password"])
+
+      connection.exec(sql)
+      connection.close
+    end
+  end
+end

data/lib/yob/encrypt.rb

@@ -0,0 +1,36 @@
+module Yob::Encrypt
+  class GnuPG
+    attr_reader :pid
+
+    def initialize(configuration)
+      require 'gpgme'
+      @keys = configuration.encryption_key_names
+    end
+
+    def encryption_input_pipe(output_pipe)
+      rd, wr = IO.pipe
+      @pid = fork do
+        $0 = "yob: GnuPG encryption"
+        wr.close
+        encrypt(rd, output_pipe)
+      end
+      rd.close
+      output_pipe.close # the fork takes ownership of this descriptor
+      wr
+    end
+
+    def encrypt(rd, wr)
+      puts "[Encrypt::GnuPG] encrypting input"
+
+      plain_data = GPGME::Data.new(rd)
+      cipher_data = GPGME::Data.new(wr)
+      keys = GPGME::Key.find(:public, @keys)
+
+      GPGME::Ctx.new do |ctx|
+        ctx.encrypt(keys, plain_data, cipher_data, GPGME::ENCRYPT_ALWAYS_TRUST)
+      end
+
+      puts "[Encrypt::GnuPG] encrypting complete"
+    end
+  end
+end

data/lib/yob/store.rb

@@ -0,0 +1,79 @@
+module Yob::Store
+  class AWS
+    BLOCK_SIZE = 1048576 * 5 # must be at least 5 MB
+
+    attr_reader :pid
+
+    def initialize(configuration)
+      require 'aws-sdk'
+
+      @configuration = configuration
+
+      ::AWS.config(:access_key_id     => configuration.aws_access_key_id,
+                   :secret_access_key => configuration.aws_secret_access_key)
+    end
+
+    def storage_input_pipe(filename)
+      rd, wr = IO.pipe
+      @pid = fork do
+        $0 = "yob: AWS S3 storage"
+        wr.close
+        store(filename, rd)
+        rd.close
+      end
+      rd.close
+      wr
+    end
+
+    def store(filename, file_handle)
+      puts "[Store::AWS] uploading #{filename}"
+      object = s3_bucket.objects["#{@configuration["aws_filename_prefix"]}#{filename}"]
+
+      data = file_handle.read(BLOCK_SIZE)
+      if data.nil? || data.length == 0
+        print "[Store::AWS] no file data received, upload aborted"
+        return
+      end
+
+      # If the entire file is less than BLOCK_SIZE, send it in one hit.
+      # Otherwise use AWS's multipart upload feature.  Note that each part must be at least 5MB.
+      if data.length < BLOCK_SIZE
+        object.write(data)
+      else
+        upload = object.multipart_uploads.create
+        begin
+          print "[Store::AWS] multipart upload started\n" if @configuration["debug"]
+          bytes = 0
+          while data && data.length > 0
+            upload.add_part(data)
+            bytes += data.length
+            print "[Store::AWS] #{bytes} bytes sent\n" if @configuration["debug"]
+            data = file_handle.read(BLOCK_SIZE)
+          end
+          upload.close
+        rescue
+          upload.abort
+          raise
+        end
+      end
+
+      if grant_to = @configuration["aws_grant_access_to"]
+        puts "[Store::AWS] granting access to #{filename}"
+        object.acl = access_control_list(grant_to)
+      end
+
+      puts "[Store::AWS] uploaded"
+    end
+
+    protected
+    def s3_bucket
+      ::AWS::S3.new.buckets[@configuration.aws_bucket]
+    end
+
+    def access_control_list(email)
+      acl = ::AWS::S3::AccessControlList.new
+      acl.grant(:full_control).to(:amazon_customer_email => email)
+      acl
+    end
+  end
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: yob
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Roger Nesbitt
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitar
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gpgme
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: YouDo Online Backup
+email: roger@youdo.co.nz
+executables:
+- yob
+extensions: []
+extra_rdoc_files: []
+files:
+- README.rdoc
+- lib/yob.rb
+- bin/yob
+- lib/yob/configuration.rb
+- lib/yob/database.rb
+- lib/yob/encrypt.rb
+- lib/yob/store.rb
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: YouDo Online Backup
+test_files: []