bibliotech 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fa336e5473f11901ab7dcddd786ad077ca6cc9d4
+  data.tar.gz: 7823f13b23335c838867c1c6d20dee7707924c39
+SHA512:
+  metadata.gz: fb3d5921a720ab1dc4e2b682d8d6be78155a4f6c39a70554b1c83a164696624fbf7fc3188b865c2a8904e20cacb8a1240a1e407867f13bc6eba7da7a675e013f
+  data.tar.gz: a16f586a9e18301134a6a454a7cd3749b64994753f4b94a8076ca862bbcf82479f2dd83e508fa08c3a0d3264293fccb9d3d11a0e6b5d7ee1c98f3d7796b3f9fb

data/bin/bibliotech

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'bibliotech/cli'
+
+BiblioTech::CLI.start(ARGV)

data/doc/example_config_file.yml

@@ -0,0 +1,58 @@
+# bibliotech.yml on production server
+# used by commands like
+#   rake bibliotech:backups:create
+#   rake bibliotech:backups:cycle
+backups:
+  dir: db_backups
+  compress:  gzip   # [ none, gzip, bzip2, 7zip ]
+  keep:
+    hourlies:   48
+    dailies:    14
+    weeklies:   8
+    monthlies:  all
+
+  create: hourly
+  prune:  daily
+  clone:
+    storage: s3  [ file, ftp, scp ]
+    target: ecliptic_db_backups   # URL
+
+database_config_env: production
+database_config_file: 'config/database.yml'   # this is the default
+# -- or --
+database_config:
+  hostname: xxxx
+  username: xxxxx
+  password: xxxxx
+  port: xxxx
+
+#bibliotech.yml on staging server
+#useful for:
+#  > rake bibliotech:restore_from[<remote name, e.g. production>]
+#   # (copies most recent database backup from this path or URL and restores it)
+backups: none
+database_config_env: staging
+database_config_file: 'config/database.yml'   # this is the default
+# -- or --
+database_config:
+  hostname: xxxx
+  username: xxxxx
+  password: xxxxx
+  port: xxxx
+
+remotes:
+  production:
+    host: some.server.com
+    path: "/var/www/eclipticenterprises.com/current"
+    compressed: gzip
+
+
+#bibliotech.yml on dev
+# used by commands like
+#   > rake bibliotech:remote_sync:down[production]   # production is default if ommitted
+#   > rake bibliotech:remote_sync:up
+#   > cap bibliotech:remote_sync:down[production]  # uses Cap config, so URL below is unnecessary in config
+#
+remotes:
+  production: "root@appserver2.lrdesign.com:/var/www/eclipticenterprises.com/current"
+  staging:    "root@appserver2.lrdesign.com:/var/www/staging.eclipticenterprises.com/current"

data/doc/todo.txt

@@ -0,0 +1,19 @@
+* load DB configs from a database.yml
+* export commands: mysql
+* export commands: posgresql
+* export commands: postgis
+* import commands: mysql
+* import commands: postgresql
+* import commands: postgis
+* rake task: backup databases
+* rake task: filter dated DB backups
+* configuration: how many backups to keep of what age
+*
+* rake task: import DB locally by filename
+* rake task: import[&migrate] most recent DB (at default name) from production, locally
+* rake task: import[&migrate] most recent DB (at default name) from production, remotely
+* rake task: download and import remote sql backup
+* rake task: upload and import local backup to remote:
+*
+* cap task: download and import remote sql backup
+* cap task: upload and import local backup to remote:

data/lib/bibliotech/application.rb

@@ -0,0 +1,95 @@
+require 'bibliotech'
+require 'caliph'
+require 'valise'
+require 'bibliotech/backups/pruner'
+
+module BiblioTech
+  class Application
+    attr_accessor :config_path, :config_hash
+    attr_writer :shell
+
+    def initialize
+      @memos = {}
+      @shell = Caliph.new
+      @config_path = %w{/etc/bibliotech /usr/share/bibliotech ~/.bibliotech ./.bibliotech ./config/bibliotech}
+    end
+
+    def valise
+      @memos[:valise] ||=
+        begin
+          dirs = config_path
+          Valise::define do
+            dirs.reverse.each do |dir|
+              rw dir
+            end
+            ro from_here(%w{.. default_configuration}, up_to("lib"))
+            handle "*.yaml", :yaml, :hash_merge
+            handle "*.yml", :yaml, :hash_merge
+          end
+        end
+    end
+
+    def config
+      @memos[:config] ||= Config.new(valise)
+    end
+
+    def commands
+      @memos[:command] ||= CommandGenerator.new(config)
+    end
+
+    def pruner(options)
+      Backups::Pruner.new(config.merge(options))
+    end
+
+    def prune_list(options)
+      pruner(options).list
+    end
+
+    def reset
+      @memos.clear
+    end
+
+    def import(options)
+      @shell.run(commands.import(options))
+    end
+
+    def export(options)
+      @shell.run(commands.export(options))
+    end
+
+    def create_backup(options)
+      time = Time.now.utc
+      pruner = pruner(options)
+      return unless pruner.backup_needed?(time)
+      options["backups"] ||= options[:backups] || {}
+      options["backups"]["filename"] = pruner.filename_for(time)
+      export(options)
+    end
+
+    #pull a dump from a remote
+    def get(options)
+      @shell.run(commands.fetch(options))
+    end
+
+    #push a dump to a remote
+    def send(options)
+      @shell.run(commands.push(options))
+    end
+
+    #clean up the DB dumps
+    def prune(options=nil)
+      pruner(option || {}).go
+    end
+
+    #return the latest dump of the DB
+    def latest(options = nil)
+      pruner(options || {}).most_recent.path
+    end
+
+    def remote_cli(remote, command, options)
+      @shell.run(commands.ssh_cli(remote, command, options))
+    end
+  end
+
+  App = Application
+end

data/lib/bibliotech/backups/file_record.rb

@@ -0,0 +1,16 @@
+module BiblioTech
+  module Backups
+    class FileRecord
+      attr_accessor :path, :timestamp, :keep
+
+      def initialize(path, timestamp)
+        @path, @timestamp = path, timestamp
+        @keep = false
+      end
+
+      def keep?
+        !!@keep
+      end
+    end
+  end
+end

data/lib/bibliotech/backups/prune_list.rb

@@ -0,0 +1,58 @@
+module BiblioTech
+  module Backups
+    class PruneList
+      TIMESTAMP_REGEX = /(?<year>\d{4})-(?<month>\d{2})-(?<day>\d{2})_(?<hour>\d{2}):(?<minute>\d{2})/
+
+      attr_accessor :path, :prefix
+
+      def initialize(path, prefix)
+        @path, @prefix = path, prefix
+      end
+
+      def list
+        files = []
+        Dir.new(path).each do |file|
+          next if %w{. ..}.include?(file)
+          file_record = build_record(file)
+          if file_record.nil?
+          else
+            files << file_record
+          end
+        end
+        files
+      end
+
+      def prefix_timestamp_re
+        prefix_re(TIMESTAMP_REGEX)
+      end
+
+      def prefix_re(also)
+        /\A#{prefix}-#{also}\..*\z/
+      end
+
+      def self.filename_for(prefix, time)
+        time.strftime("#{prefix}-%Y-%m-%d_%H:%M.sql")
+      end
+
+      def build_record(file)
+        if file =~ prefix_re(/.*/)
+          if !(match = prefix_timestamp_re.match(file)).nil?
+            timespec = %w{year month day hour minute}.map do |part|
+              Integer(match[part], 10)
+            end
+            parsed_time = Time::utc(*timespec)
+            return FileRecord.new(File::join(path, file), parsed_time)
+          else
+            raise "File prefixed #{prefix} doesn't match #{prefix_timestamp_re.to_s}: #{File::join(path, file)}"
+          end
+        else
+          if file !~ TIMESTAMP_REGEX
+            warn "Stray file in backups directory: #{File::join(path, file)}"
+            return nil
+          end
+        end
+        return nil
+      end
+    end
+  end
+end

data/lib/bibliotech/backups/pruner.rb

@@ -0,0 +1,71 @@
+require 'bibliotech/backups/prune_list'
+require 'bibliotech/backups/file_record'
+require 'bibliotech/backups/scheduler'
+
+module BiblioTech
+  module Backups
+    class Pruner
+      def initialize(config)
+        @config = config
+      end
+      attr_reader :config
+
+      def path
+        @path ||= config.backup_path
+      end
+
+      def name
+        @name ||= config.backup_name
+      end
+
+      def schedules
+        @schedules ||=
+          [].tap do |array|
+          config.each_prune_schedule do |frequency, limit|
+            array << Scheduler.new(frequency, limit)
+          end
+          end
+      end
+
+      def backup_needed?(time)
+        time - most_recent.timestamp < config.backup_frequency * 60
+      end
+
+      def list
+        @list ||=
+          begin
+            list = PruneList.new(path, name).list
+            schedules.each do |schedule|
+              schedule.mark(list)
+            end
+            list
+          end
+      end
+
+      def most_recent
+        list.max_by do |record|
+          record.timestamp
+        end
+      end
+
+      def filename_for(time)
+        PruneList.filename_for(time)
+      end
+
+      def pruneable
+        list.select do |record|
+          !record.keep?
+        end
+      end
+
+      def go
+        return if schedules.empty?
+        pruneable.each {|record| delete(record.path) }
+      end
+
+      def delete(path)
+        File.unlink(path)
+      end
+    end
+  end
+end

data/lib/bibliotech/backups/scheduler.rb

@@ -0,0 +1,49 @@
+require 'bibliotech/backups/file_record'
+
+module BiblioTech
+  module Backups
+    class Scheduler
+      attr_accessor :frequency, :limit
+
+      def initialize(frequency, limit)
+        @frequency, @limit = frequency, limit
+        @limit = nil if limit == "all"
+      end
+
+      def end_time(file_list)
+        file_list.map{|record| record.timestamp}.max
+      end
+
+      def compute_start_time(file_list)
+        limit_time = Time.at(0)
+        unless limit.nil?
+          limit_time = end_time(file_list) - limit * freq_seconds
+        end
+        [limit_time, file_list.map{|record| record.timestamp}.min - range].max
+      end
+
+      def freq_seconds
+        frequency * 60
+      end
+
+      def range
+        freq_seconds / 2
+      end
+
+      def mark(file_list)
+        time = end_time(file_list)
+        start_time = compute_start_time(file_list)
+        while time > start_time do
+          closest = file_list.min_by do |record|
+            (record.timestamp - time).abs
+          end
+          if (closest.timestamp - time).abs < range
+            closest.keep = true
+          end
+          time -= freq_seconds
+        end
+        return file_list
+      end
+    end
+  end
+end

data/lib/bibliotech/builders/database.rb

@@ -0,0 +1,25 @@
+require 'bibliotech/builders'
+
+module BiblioTech
+  module Builders
+    class Database < Base
+      def self.find_class(config)
+        adapter_registry.fetch(config.adapter) do
+          raise "config.adapter is #{config.adapter.inspect} - supported adapters are #{supported_adapters.join(", ")}"
+        end
+      end
+    end
+
+    class Import < Database
+      def self.registry_host
+        Import
+      end
+    end
+
+    class Export < Database
+      def self.registry_host
+        Export
+      end
+    end
+  end
+end

data/lib/bibliotech/builders/file.rb

@@ -0,0 +1,75 @@
+require 'bibliotech/builders'
+
+module BiblioTech
+  module Builders
+    class File < Base
+      def self.find_class(config)
+        file = config.backup_file
+
+        explicit = find_explicit(config)
+        return explicit unless explicit.nil?
+
+        _, klass = adapter_registry.find{ |pattern, klass|
+          next if pattern.is_a? Symbol
+          file =~ pattern
+        }
+        klass || identity_adapter
+      rescue Config::MissingConfig
+        return NullAdapter
+      end
+
+      def file
+        config.backup_file
+      end
+    end
+
+    class FileInput < File
+      def self.identity_adapter
+        IdentityFileInput
+      end
+
+      def self.find_explicit(config)
+        return adapter_registry.fetch(config.expander) do
+          raise "config.expander is #{config.expander.inspect} - supported expanders are #{supported_adapters.select{|ad| ad.is_a? Symbol}.join(", ")}"
+        end
+      rescue Config::MissingConfig
+        nil
+      end
+
+      def self.registry_host
+        FileInput
+      end
+    end
+
+    class FileOutput < File
+      def self.identity_adapter
+        IdentityFileOutput
+      end
+
+      def self.find_explicit(config)
+        return adapter_registry.fetch(config.compressor) do
+          raise "config.compressor is #{config.compressor.inspect} - supported compressors are #{supported_adapters.select{|ad| ad.is_a? Symbol}.join(", ")}"
+        end
+      rescue KeyError
+      end
+
+      def self.registry_host
+        FileOutput
+      end
+    end
+
+    class IdentityFileInput < FileInput
+      def go(cmd)
+        cmd.redirect_stdin(file)
+        cmd
+      end
+    end
+
+    class IdentityFileOutput < FileOutput
+      def go(cmd)
+        cmd.redirect_stdout(file)
+        cmd
+      end
+    end
+  end
+end

data/lib/bibliotech/builders/gzip.rb

@@ -0,0 +1,51 @@
+require 'bibliotech/builders/file'
+
+module BiblioTech
+  module Builders
+    class GzipExpander < FileInput
+      register(/.*\.gz\z/)
+      register(/.*\.gzip\z/)
+
+      def go(command)
+        command = cmd("gunzip", file) | command
+      end
+    end
+
+    class ExplicitGzipExpander < GzipExpander
+      register :gzip
+
+      def file
+        file = super
+        unless PATTERNS.any?{|pattern| pattern =~ file}
+          return file + ".gz"
+        end
+        file
+      end
+    end
+
+    class GzipCompressor < FileOutput
+      PATTERNS = [ /.*\.gz\z/, /.*\.gzip\z/ ]
+      PATTERNS.each do |pattern|
+        register pattern
+      end
+
+      def go(cmd)
+        cmd |= %w{gzip}
+        cmd.redirect_stdout(file)
+        cmd
+      end
+    end
+
+    class ExplicitGzipCompressor < GzipCompressor
+      register :gzip
+
+      def file
+        file = super
+        unless PATTERNS.any?{|pattern| pattern =~ file}
+          return file + ".gz"
+        end
+        file
+      end
+    end
+  end
+end

data/lib/bibliotech/builders/mysql.rb

@@ -0,0 +1,35 @@
+require 'bibliotech/builders/database'
+
+module BiblioTech
+  module Builders
+    module MySql
+      class Export < Builders::Export
+        register :mysql
+
+        def go(command)
+          command.from('mysqldump')
+          config.optional{ command.options << "-h #{config.host}" }
+          config.optional{ command.options << "-u #{config.username}" }
+          config.optional{ command.options << "-P #{config.port}" } #ok
+          config.optional{ command.options << "--password='#{config.password}'" }
+          command.options << "#{config.database}"
+          command
+        end
+      end
+
+      class Import < Builders::Import
+        register :mysql
+
+        def go(command)
+          command.from('mysql')
+          config.optional{ command.options << "-h #{config.host}" }
+          config.optional{ command.options << "-u #{config.username}" }
+          config.optional{ command.options << "-P #{config.port}" } #ok
+          config.optional{ command.options << "--password='#{config.password}'" }
+          command.options << "#{config.database}"
+          command
+        end
+      end
+    end
+  end
+end

data/lib/bibliotech/builders/postgres.rb

@@ -0,0 +1,37 @@
+require 'bibliotech/builders/database'
+
+module BiblioTech
+  module Builders
+    module Postgres
+      class Export < Builders::Export
+        register :postgres
+
+        def go(command)
+          command.from('pg_dump', '-Fc')
+          config.optional{ command.options << "-h #{config.host}" }
+          config.optional{ command.env["PGPASSWORD"] = config.password }
+          config.optional{ command.options << "-p #{config.port}" } #ok
+
+          command.options << "-U #{config.username}"
+          command.options << "#{config.database}"
+          command
+        end
+      end
+
+      class Import < Builders::Import
+        register :postgres
+
+        def go(command)
+          command.from('pg_restore')
+          config.optional{ command.options << "-h #{config.host}" }
+          config.optional{ command.env["PGPASSWORD"] = config.password }
+          config.optional{ command.options << "-p #{config.port}" } #ok
+
+          command.options << "-U #{config.username}"
+          command.options << "-d #{config.database}"
+          command
+        end
+      end
+    end
+  end
+end

data/lib/bibliotech/builders.rb

@@ -0,0 +1,43 @@
+module BiblioTech
+  module Builders
+    class Base
+      include Caliph::CommandLineDSL
+      class << self
+        def register(adapter_name)
+          adapter_registry[adapter_name] = self
+        end
+
+        def adapter_registry
+          registry_host.registry
+        end
+
+        def registry
+          @registry ||={}
+        end
+
+        def supported_adapters
+          adapter_registry.keys
+        end
+
+        def for(config)
+          find_class(config).new(config)
+        end
+
+        def null_adapter
+          NullAdapter
+        end
+      end
+
+      def initialize(config)
+        @config = config
+      end
+      attr_reader :config
+    end
+
+    class NullAdapter < Base
+      def go(cmd)
+        cmd
+      end
+    end
+  end
+end

data/lib/bibliotech/cli.rb

@@ -0,0 +1,24 @@
+require 'thor'
+require 'bibliotech/application'
+
+module BiblioTech
+  class CLI < Thor
+    desc "latest", "Outputs the latest DB dump available locally"
+    def latest
+      app = App.new
+      app.latest
+    end
+
+    desc "dump FILENAME", "Create a new database dump into FILE"
+    def dump(file)
+      app = App.new
+      app.export(:backups => { :filename => file })
+    end
+
+    desc "load FILENAME", "Load a database file from FILE"
+    def load(file)
+      app = App.new
+      app.import(:backups => { :filename => file })
+    end
+  end
+end