bibliotech 0.1.0

data/lib/bibliotech/command_generator.rb

@@ -0,0 +1,86 @@
+require 'caliph'
+
+require 'bibliotech/builders/gzip'
+require 'bibliotech/builders/postgres'
+require 'bibliotech/builders/mysql'
+
+module BiblioTech
+  class CommandGenerator
+
+    include Caliph::CommandLineDSL
+
+    attr_accessor :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def export(options = nil)
+      options = config.merge(options || {})
+      command = cmd
+      command = Builders::Export.for(options).go(command)
+      Builders::FileOutput.for(options).go(command)
+    end
+
+    def import(options = nil)
+      options = config.merge(options || {})
+      command = cmd()
+      command = Builders::Import.for(options).go(command)
+      Builders::FileInput.for(options).go(command)
+    end
+
+    def fetch(remote, filename, options = nil)
+      options = config.merge(options || {})
+      cmd("scp") do |cmd|
+        options.optionally{ cmd.options << "-i #{options.id_file(remote)}" }
+        cmd.options << options.remote_file(remote, filename)
+        cmd.options << options.local_file(filename)
+      end
+    end
+
+    def push(remote, filename, options = nil)
+      options = config.merge(options || {})
+      cmd("scp") do |cmd|
+        cmd.options << options.local_file(filename)
+        cmd.options << options.remote_file(remote, filename)
+      end
+    end
+
+    def remote_cli(remote, *command_options)
+      options = {}
+      if command_options.last.is_a? Hash
+        options = command_options.pop
+      end
+      options = config.merge(options)
+      command_on_remote = cmd("cd") do |cmd|
+        cmd.options << options.root_dir_on(remote)
+      end & cmd("bundle", "exec", "bibliotech", *command_options)
+      cmd("ssh") do |cmd|
+        cmd.options << "-n" #because we're not going to be doing any input
+        options.optionally{ cmd.options << "-i #{options.id_file(remote)}" }
+        options.optionally{ cmd.options << "-l #{options.remote_user(remote)}" }
+
+        cmd.options << options.remote_host(remote)
+
+        options.optionally{ cmd.options << "-p #{options.remote_port(remote)}" } #ok
+
+
+        options.optionally do
+          options.ssh_options(remote).each do |opt|
+            cmd.options << "-o #{opt}"
+          end
+        end
+      end - escaped_command(command_on_remote)
+    end
+
+    def wipe()
+      raise NotImplementedError
+    end
+    def delete()
+      raise NotImplementedError
+    end
+    def create()
+      raise NotImplementedError
+    end
+  end
+end

data/lib/bibliotech/command_runner.rb

@@ -0,0 +1,36 @@
+module BiblioTech
+  class CommandRunner
+
+    attr_reader :generator
+    attr_accessor :shell
+
+    def initialize(config)
+      @config = config
+      @generator = CommandGenerator.for(config.db_config)
+      @shell = Caliph.new
+    end
+
+    def export(filepath)
+      run decorate_for_compression(generator, filepath).export(filepath)
+    end
+
+    def import(filepath)
+      run decorate_for_compression(generator, filepath).import(filepath)
+    end
+
+    #def wipe()
+      #tables = system(CommandGenerator.new.fetch_tables(@config))
+      #filter_tables_for_wipeable(tables)
+      #system(CommandGenerator.new.wipe_tables(@config,tables))
+    #end
+    def run(command)
+      @shell.run(command)
+    end
+
+    private
+    def decorate_for_compression(generator, filepath)
+      Compression.for(filepath, generator)
+    end
+
+  end
+end

data/lib/bibliotech/compression/bzip2.rb

@@ -0,0 +1,6 @@
+module BiblioTech
+  class Compression::Bzip2 < Compression
+    register /\.bz2\Z/, self
+    register /\.bzip2\Z/, self
+  end
+end

data/lib/bibliotech/compression/gzip.rb

@@ -0,0 +1,6 @@
+module BiblioTech
+  class Compression::Gzip < Compression
+    register /\.gz\Z/, self
+    register /\.gzip\Z/, self
+  end
+end

data/lib/bibliotech/compression/sevenzip.rb

@@ -0,0 +1,5 @@
+module BiblioTech
+  class Compression::SevenZip < Compression
+    register /\.7z\Z/, self
+  end
+end

data/lib/bibliotech/compression.rb

@@ -0,0 +1,35 @@
+module BiblioTech
+  class Compression
+
+    class << self
+      def register(adapter_pattern, klass)
+        Compression.registry[adapter_pattern] = klass
+      end
+
+      def registry
+        @adapter_registry ||={}
+      end
+
+      def supported_adapters
+        @adapter_registry.keys
+      end
+
+      def for(filepath, generator)
+        _, klass = @adapter_registry.find{ |pattern, klass|
+          filepath =~ pattern
+        }
+        return generator if klass.nil?
+        klass.new(generator)
+      end
+    end
+
+    def initialize(generator)
+      @generator = generator
+    end
+
+  end
+end
+
+require 'bibliotech/compression/gzip'
+require 'bibliotech/compression/sevenzip'
+require 'bibliotech/compression/bzip2'

data/lib/bibliotech/config.rb

@@ -0,0 +1,269 @@
+module BiblioTech
+  class Config
+    class MissingConfig < KeyError; end
+
+    CONFIG_STEPS = {
+      :database_config_file => [ "database_config_file" ] ,
+      :database_config_env  => [ "database_config_env"  ] ,
+      :host                 => [ "host" ]                 ,
+      :port                 => [ "port" ]                 ,
+      :user                 => [ "user" ]                 ,
+      :rsa_files            => [ "rsa_files" ]            ,
+      :ssh_options          => [ "ssh_options" ]          ,
+      :file                 => [ "backups"                , "file"      ] ,
+      :filename             => [ "backups"                , "filename"  ] ,
+      :backup_path          => [ "backups"                , "dir"       ] ,
+      :root_path            => [ "path" ]                 ,
+      :fetch_dir            => [ "fetched_dir" ]          ,
+      :compressor           => [ "backups"                , "compress"  ] ,
+      :prune_schedule       => [ "backups"                , "keep"      ] ,
+      :backup_name          => [ "backups"                , "prefix"    ] ,
+      :backup_frequency     => [ "backups"                , "frequency" ] ,
+      :db_adapter           => [ "database_config"        , "adapter"   ] ,
+      :db_host              => [ "database_config"        , "host"      ] ,
+      :db_port              => [ "database_config"        , "port"      ] ,
+      :db_database          => [ "database_config"        , "database"  ] ,
+      :db_username          => [ "database_config"        , "username"  ] ,
+      :db_password          => [ "database_config"        , "password"  ] ,
+    }
+
+    def initialize(valise)
+      @valise = valise
+    end
+
+    attr_reader :valise
+    attr_writer :hash
+
+    def hash
+      @hash ||= stringify_keys(valise.contents("config.yaml"))
+    end
+
+    def stringify_keys(hash) # sym -> string
+      hash.keys.each do |key|
+        if key.is_a?(Symbol)
+          hash[key.to_s] = hash.delete(key)
+        end
+        if hash[key.to_s].is_a?(Hash)
+          hash[key.to_s] = stringify_keys(hash[key.to_s])
+        end
+      end
+      hash
+    end
+
+    def merge(other_hash)
+      self.class.new(valise).tap do |newbie|
+        newbie.hash = hash.merge(stringify_keys(other_hash))
+      end
+    end
+
+    def steps_for(key)
+      CONFIG_STEPS.fetch(key)
+    end
+
+    def optional(&block)
+      yield
+    rescue MissingConfig
+    end
+    alias optionally optional
+
+    def extract(*steps_chain)
+      steps_chain.each do |steps|
+        begin
+          return steps.inject(hash) do |hash, step|
+            raise MissingConfig if hash.nil?
+            hash.fetch(step)
+          end
+        rescue KeyError
+        end
+      end
+      raise MissingConfig, "No value configured at any of: #{steps_chain.map{|steps| steps.join(">")}}"
+    end
+
+    def local
+      extract(["local"])
+    end
+
+    def remote
+      extract(["remote"])
+    end
+
+    def local_get(key)
+      steps = steps_for(key)
+      steps_chain =
+        begin
+          [steps, [local] + steps]
+        rescue MissingConfig
+          [steps]
+        end
+      extract(*steps_chain)
+    end
+
+    def remote_get(remote_name, key)
+      steps = [remote_name] + steps_for(key)
+      extract(steps, ["remotes"] + steps)
+    end
+
+    def ssh_options(for_remote)
+      steps = steps_for(:ssh_options) + [for_remote]
+      steps_chain =
+        begin
+          [steps, [local] + steps]
+        rescue MissingConfig
+          [steps]
+        end
+      extract(steps_chain)
+    end
+
+    def id_file(for_remote)
+      steps = steps_for(:rsa_files) + [for_remote]
+      steps_chain =
+        begin
+          [steps, [local] + steps]
+        rescue MissingConfig
+          [steps]
+        end
+      extract(steps_chain)
+    end
+
+    def local_path
+      local_get(:fetch_dir)
+    rescue MissingConfig
+      local_get(:root_path)
+    end
+
+    def local_file(filename)
+      File::join(local_path, filename)
+    end
+
+    def root_dir_on(remote)
+      remote_get(remote, :root_path)
+    end
+
+    def remote_host(remote)
+      remote_get(remote, :host)
+    end
+
+    def remote_port(remote)
+      remote_get(remote, :port)
+    end
+
+    def remote_user(remote)
+      remote_get(remote, :user)
+    end
+
+    def remote_path(remote)
+      path = "#{remote_host(remote)}:#{root_dir_on(remote)}"
+      begin
+        "#{remote_user(remote)}@#{path}"
+      rescue MissingConfig
+        path
+      end
+    end
+
+    def remote_file(remote, filename)
+      File::join(remote_path(remote), filename)
+    end
+
+    SCHEDULE_SHORTHANDS = {
+      "hourly"      => 60,
+      "hourlies"    => 60,
+      "daily"       => 60 * 24,
+      "dailies"     => 60 * 24,
+      "weekly"      => 60 * 24 * 7,
+      "weeklies"    => 60 * 24 * 7,
+      "monthly"     => 60 * 24 * 30,
+      "monthlies"   => 60 * 24 * 30,
+      "quarterly"   => 60 * 24 * 120,
+      "quarterlies" => 60 * 24 * 120,
+      "yearly"      => 60 * 24 * 365,
+      "yearlies"    => 60 * 24 * 365,
+    }
+    def regularize_frequency(frequency)
+      Integer( SCHEDULE_SHORTHANDS.fetch(frequency){ frequency } )
+    rescue ArgumentError
+      raise "#{frequency.inspect} is neither a number of minutes or a shorthand. Try:\n  #{SCHEDULE_SHORTHANDS.keys.join(" ")}"
+    end
+
+    def backup_name
+      local_get(:backup_name)
+    end
+
+    def backup_frequency
+      @backup_frequency ||= regularize_frequency(local_get(:backup_frequency))
+    end
+
+    def each_prune_schedule
+      local_get(:prune_schedule).each do |frequency, limit|
+        real_frequency = regularize_frequency(frequency)
+        unless real_frequency % backup_frequency == 0
+          raise "Pruning frequency #{real_frequency}:#{frequency} is not a multiple of backup frequency: #{backup_frequency}:#{local_get(:backup_frequency)}"
+        end
+        yield(real_frequency, limit)
+      end
+    end
+
+    def database_config
+      hash["database_config"] ||= valise.contents(local_get(:database_config_file))[local_get(:database_config_env)]
+    end
+
+    #@group File management
+    def backup_file
+      local_get(:file)
+    rescue MissingConfig
+      ::File.join(backup_path, filename)
+    end
+
+    def filename
+      local_get(:filename)
+    end
+
+    def backup_path
+      local_get(:backup_path)
+    end
+
+    def expander
+      if remote.nil?
+        local_get(:expander)
+      else
+        remote_get(remote, :expander)
+      end
+    end
+
+    def compressor
+      local_get(:compressor)
+    end
+    #@endgroup
+
+    #@group Database
+    def adapter
+      database_config
+      local_get(:db_adapter)
+    end
+
+    def host
+      database_config
+      local_get(:db_host)
+    end
+
+    def port
+      database_config
+      local_get(:db_port)
+    end
+
+    def username
+      database_config
+      local_get(:db_username)
+    end
+
+    def database
+      database_config
+      local_get(:db_database)
+    end
+
+    def password
+      database_config
+      local_get(:db_password)
+    end
+    #@endgroup
+  end
+end

data/lib/bibliotech/rake_lib.rb

@@ -0,0 +1,82 @@
+require 'mattock'
+require 'bibliotech/application'
+
+module BiblioTech
+  class Tasklib < ::Mattock::Tasklib
+    setting(:app)
+    setting(:config_path)
+    setting(:local, nil)
+    setting(:remote, nil)
+
+    def default_configuration
+      super
+      self.app = App.new
+
+      self.config_path = app.config_path
+      from_hash(app.config.hash)
+      @default_state = to_hash
+      @default_state.delete(:app)
+      @default_state.delete(:config_path)
+    end
+
+    def resolve_configuration
+      configured_state = to_hash
+      configured_state.delete(:app)
+      configured_state.delete(:config_path)
+      case [config_path == app.config_path, configured_state == to_hash.delete(:config_path)]
+      when [false, false]
+      when [true, true]
+        raise "Cannot both change to config path and any other setting (sorry) - put configs in a file"
+      when [true, false]
+        app.config.hash.merge!(configured_state)
+      when [false, true]
+        app.config_path = config_path
+        app.reset
+      end
+      super
+    end
+
+    default_namespace :bibliotech
+
+    def define
+      in_namespace do
+        namespace :backups do
+          task :restore, [:name] do |task, args|
+            fail ":name is required" if args[:name].nil?
+            options = { :backups => { :filename => args[:name] } }
+            if %r[/] =~ args[:name]
+              options = { :backups => { :file => args[:name] } }
+            end
+            app.import(options)
+          end
+
+          task :create, [:prefix] do |task, args|
+            fail ":prefix is required" if args[:prefix].nil?
+            app.create_backup( :backups => { :prefix => args[:prefix] } )
+          end
+
+          task :clean, [:prefix] do |task, args|
+            fail ":prefix is required" if args[:prefix].nil?
+            app.prune( :backups => { :prefix => args[:prefix] } )
+          end
+
+          task :perform, [:prefix] => [:create, :clean]
+        end
+
+        namespace :remote_sync do
+          task :down do
+            filename = app.remote_cli(remote, "latest")
+            app.get(remote, filename)
+            app.import(:backups => { :filename => filename})
+          end
+
+          task :up do
+            filename = app.latest
+            app.send(remote, filename)
+            app.remote_cli(remote, "load", filename)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bibliotech.rb

@@ -0,0 +1,7 @@
+module BiblioTech
+end
+
+require 'bibliotech/config'
+require 'bibliotech/command_generator'
+require 'bibliotech/command_runner'
+require 'bibliotech/compression'

data/spec/bibliotech/backup_pruner_spec.rb

@@ -0,0 +1,58 @@
+require 'bibliotech/application'
+require 'bibliotech/backups/pruner'
+require 'file-sandbox'
+module BiblioTech
+  describe Backups::Pruner do
+    include FileSandbox
+
+    before :each do
+      sandbox.new :directory => "db_backups"
+      sandbox.new :file => "db_backups/backup-2014-08-12_00:00.sql.7z"
+    end
+
+    let :app do
+      App.new
+    end
+
+    it "should something latest" do
+      expect(app.latest("local" => "production")).to eql "db_backups/backup-2014-08-12_00:00.sql.7z"
+    end
+
+  end
+
+  describe Backups::PruneList do
+    subject :pruner do
+      Backups::PruneList.new("/some/path/for/files", "testing")
+    end
+
+    it "should warn when other files are present" do
+      expect(pruner).to receive(:warn)
+      pruner.build_record("some.random.file")
+    end
+
+    it "should fail when correct prefix doesn't match timestamp" do
+      expect do
+        pruner.build_record("testing-WACKYTIMESTAMP.sql.gz")
+      end.to raise_error
+    end
+
+    describe "creating filenames" do
+      it "should match filenames it creates" do
+        time = Time.new(2014, 7, 30, 3, 14, 37, 0)
+        record = pruner.build_record(Backups::PruneList.filename_for("testing", time))
+        expect(record.timestamp).to be_within(60).of(time)
+      end
+    end
+
+    describe "producing a record" do
+      subject :record do
+        pruner.build_record("testing-2014-07-30_03:14.sql.gz")
+      end
+
+      it { is_expected.to be_a(Backups::FileRecord) }
+      it "should have a good time" do
+        expect(record.timestamp).to eql Time.new(2014, 7, 30, 3, 14, 0, 0)
+      end
+    end
+  end
+end

data/spec/bibliotech/backup_scheduler_spec.rb

@@ -0,0 +1,108 @@
+require 'bibliotech/backups/scheduler'
+
+module BiblioTech::Backups
+  describe Scheduler do
+    let(:test_jitter){ 0 }
+
+    let :unfiltered_files do
+      (0..interval).step(frequency).map do |seconds| #every 15 seconds for 8 hours
+        seconds = seconds - test_jitter/2 + rand(test_jitter)
+        FileRecord.new("", Time.now - seconds)
+      end
+    end
+
+    let :filtered_files do
+      scheduler.mark(unfiltered_files)
+    end
+
+    let :kept_files do
+      filtered_files.select do |record|
+        record.keep?
+      end
+    end
+
+    describe "without a limit" do
+      let :scheduler do
+        Scheduler.new(60, nil)
+      end
+
+      context "when there's more than enough backups" do
+        let(:interval){ 60*60*12 - 1}
+        let(:frequency) { 15 }
+        let(:test_jitter){ 60 }
+
+        it "should mark 8 files kept" do
+          expect(kept_files.count).to eql 13
+        end
+      end
+    end
+
+    describe "with a limit" do
+      let :scheduler do
+        Scheduler.new(60, 8)
+      end
+
+      context "when there's just enough backups" do
+        let(:interval){ 60*60*8 - 1 }
+        let(:frequency){ 60*8 }
+        let(:test_jitter){ 60 }
+
+        it "should mark 8 files kept" do
+          expect(kept_files.count).to eql 8
+        end
+
+        context "even if we're pruning much later" do
+          let :filtered_files do
+            unfiltered_files.each do |record|
+              record.timestamp += 60*60*24
+            end
+
+            scheduler.mark(unfiltered_files)
+          end
+
+          it "should mark 8 files kept" do
+            expect(kept_files.count).to eql 8
+          end
+        end
+      end
+
+      context "when there's more than enough backups" do
+        let(:interval){ 60*60*12 }
+        let(:frequency) { 15 }
+        let(:test_jitter){ 60 }
+
+        it "should mark 8 files kept" do
+          expect(kept_files.count).to eql 8
+        end
+      end
+
+      context "when there are too few backups" do
+        let(:interval){ 60*60*4 - 1 }
+        let(:frequency){ 60*8 }
+        let(:test_jitter){ 60 }
+
+        it "should mark 8 files kept" do
+          expect(kept_files.count).to eql 5
+        end
+      end
+
+      context "when files already marked to keep" do
+        let :filtered_files do
+          unfiltered_files.each do |record|
+            record.keep = true
+          end
+
+          scheduler.mark(unfiltered_files)
+        end
+
+        let(:interval){ 60*60*12 }
+        let(:frequency) { 15 }
+        let(:test_jitter){ 60 }
+
+        it "should not unmark any" do
+          expect(kept_files.length).to eql(unfiltered_files.length)
+        end
+      end
+    end
+  end
+end