bibliotech 0.2.13 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a22ff69ff51f9da65c676de7b84084129f2a9499
-  data.tar.gz: 414d7b654e61ab08ac35e26c3ecebb161761c4dd
+  metadata.gz: 07e130f15c7d64d6bd751f7663a913adb30a5551
+  data.tar.gz: efc5d69b6b40c211c0959020bfaf7ab46044ec56
 SHA512:
-  metadata.gz: ee5d38a9f9a2a393e9b44d07f15dba07e8285cad1b5a931ad5de2b3ca955e13b15962aec59cc76c699fd1aa494c10460e5ce1a9d26e4ac3e6085fdc6d4cfe8d7
-  data.tar.gz: 490311e700290021309b7991116fb11bc79db31be92b53c87144a355c7527ad3139ff41994f3c72a872bec96c7060998d1cb47c91668531795fdfc661564512b
+  metadata.gz: 95ac9c114a022bbbda10710e62a4f9191aa242f7f9c4b68aa8fd9576424c65309e168bbc5b0ed0e5727477af76be81017cd71ceeca87e2c25cd1488d57e0a2b3
+  data.tar.gz: 88626b4a157c11e5d4294bfd3aa8a7b730db4f685c5e755a83f87935e27cd42a37b7fc00a44a95be0342489689e68278411b73ccd68d24d17c46d21d0f951a83

data/default_configuration/config.yaml

@@ -7,6 +7,9 @@ backups:
   dir: db_backups
   compress:  gzip   # [ none, gzip, bzip2, 7zip ]
   prefix: backup
+log:
+  target: stderr
+  level: debug
 
 production:
   backups:

data/lib/bibliotech/application.rb

@@ -2,9 +2,12 @@ require 'bibliotech'
 require 'caliph'
 require 'valise'
 require 'bibliotech/backups/pruner'
+require 'bibliotech/logger'
 
 module BiblioTech
   class Application
+    include Logging
+
     attr_accessor :config_path, :config_hash
     attr_writer :shell
 
@@ -30,7 +33,19 @@ module BiblioTech
     end
 
     def config
-      @memos[:config] ||= Config.new(valise)
+      @memos[:config] ||=
+        begin
+          Config.new(valise).tap do |config|
+            setup_logger(config)
+          end
+        end
+    end
+
+    def setup_logger(config)
+      logger = Logger.new(config.log_target)
+      logger.level = config.log_level
+      BiblioTech::Logging.logger = logger
+      log.info("Started logging")
     end
 
     def commands
@@ -59,6 +74,7 @@ module BiblioTech
 
     def create_backup(options)
       time = Time.now.utc
+      log.warn{ "Creating a backup at #{time}" }
       pruner = pruner(options)
       return unless pruner.backup_needed?(time)
       options["backups"] ||= options[:backups] || {}
@@ -68,25 +84,32 @@ module BiblioTech
 
     #pull a dump from a remote
     def get(remote, options)
+      log.warn{ "Getting a dump from #{remote}" }
       @shell.run(commands.fetch(remote, options))
     end
 
     #push a dump to a remote
     def send(remote, options)
+      log.warn{ "Sending a dump to #{remote}" }
       @shell.run(commands.push(remote, options))
     end
 
     #clean up the DB dumps
     def prune(options=nil)
+      log.warn{ "Pruning DB records" }
       pruner(options || {}).go
     end
 
     #return the latest dump of the DB
     def latest(options = nil)
-      pruner(options || {}).most_recent.path
+      log.info{ "Getting most recent DB dump" }
+      pruner(options || {}).most_recent.path.tap do |latest|
+        log.info{ "  #{latest}" }
+      end
     end
 
     def remote_cli(remote, command, *options)
+      log.warn{ "Running #{command} on #{remote}" }
       @shell.run(commands.remote_cli(remote, command, *options))
     end
   end

data/lib/bibliotech/backups/file_record.rb

@@ -1,16 +1,22 @@
 module BiblioTech
   module Backups
     class FileRecord
-      attr_accessor :path, :timestamp, :keep
+      attr_accessor :path, :timestamp, :keep, :scheduled_by
 
       def initialize(path, timestamp)
         @path, @timestamp = path, timestamp
         @keep = false
+        @scheduled_by = []
       end
 
       def keep?
         !!@keep
       end
+
+      def in_schedule(name)
+        @scheduled_by << name
+        @keep = true
+      end
     end
   end
 end

data/lib/bibliotech/backups/prune_list.rb

@@ -19,7 +19,9 @@ module BiblioTech
             files << file_record
           end
         end
-        files
+        files.sort_by do |record|
+          - record.timestamp.to_i
+        end
       end
 
       def prefix_timestamp_re

data/lib/bibliotech/backups/pruner.rb

@@ -1,10 +1,11 @@
 require 'bibliotech/backups/prune_list'
 require 'bibliotech/backups/file_record'
-require 'bibliotech/backups/scheduler'
+require 'bibliotech/logger'
 
 module BiblioTech
   module Backups
     class Pruner
+      include Logging
       def initialize(config)
         @config = config
       end
@@ -19,18 +20,17 @@ module BiblioTech
       end
 
       def schedules
-        @schedules ||=
-          [].tap do |array|
-          config.each_prune_schedule do |frequency, limit|
-            array << Scheduler.new(frequency, limit)
-          end
-          end
+        @schedules ||= config.prune_schedules
+      end
+
+      def frequency
+        @frequency ||= config.backup_frequency
       end
 
       def backup_needed?(time)
         most_recent = most_recent()
         return true if most_recent.nil?
-        (time - most_recent.timestamp) > (config.backup_frequency * 60)
+        (time - most_recent.timestamp) > (frequency * 60)
       end
 
       def list
@@ -50,11 +50,19 @@ module BiblioTech
       end
 
       def filename_for(time)
-        PruneList.filename_for(config.backup_name, time)
+        PruneList.filename_for(name, time)
       end
 
       def pruneable
         mark_list
+        if list.empty?
+          log.warn{ "No backup files in #{path} / #{name} !" }
+        end
+        list.each do |record|
+          log.info{
+            "#{record.path} #{record.timestamp} #{record.keep ? "kept: #{record.scheduled_by.inspect}" : "discarding"}"
+          }
+        end
         list.select do |record|
           !record.keep?
         end

data/lib/bibliotech/backups/scheduler.rb

@@ -3,9 +3,10 @@ require 'bibliotech/backups/file_record'
 module BiblioTech
   module Backups
     class Scheduler
-      attr_accessor :frequency, :limit
+      attr_accessor :frequency, :limit, :name
 
-      def initialize(frequency, limit)
+      def initialize(name, frequency, limit)
+        @name = name
         @frequency, @limit = frequency, limit
         @limit = nil if limit == "all"
       end
@@ -51,7 +52,7 @@ module BiblioTech
           closest = file_list.first
 
           if (time - closest.timestamp) < freq_seconds
-            closest.keep = true
+            closest.in_schedule(name)
           end
           time -= freq_seconds
         end

data/lib/bibliotech/command_generator.rb

@@ -3,11 +3,13 @@ require 'caliph'
 require 'bibliotech/builders/gzip'
 require 'bibliotech/builders/postgres'
 require 'bibliotech/builders/mysql'
+require 'bibliotech/logger'
 
 module BiblioTech
   class CommandGenerator
 
     include Caliph::CommandLineDSL
+    include Logging
 
     attr_accessor :config
 
@@ -19,14 +21,18 @@ module BiblioTech
       options = config.merge(options || {})
       command = cmd
       command = Builders::Export.for(options).go(command)
-      Builders::FileOutput.for(options).go(command)
+      Builders::FileOutput.for(options).go(command).tap do |cmd|
+        log.info{ cmd.command }
+      end
     end
 
     def import(options = nil)
       options = config.merge(options || {})
       command = cmd()
       command = Builders::Import.for(options).go(command)
-      Builders::FileInput.for(options).go(command)
+      Builders::FileInput.for(options).go(command).tap do |cmd|
+        log.info{ cmd.command }
+      end
     end
 
     def fetch(remote, filename, options = nil)
@@ -39,6 +45,8 @@ module BiblioTech
         options.optionally{ cmd.options << "-i #{options.id_file(remote)}" }
         cmd.options << options.remote_file(remote, filename)
         cmd.options << local_path
+      end.tap do |cmd|
+        log.info{ cmd.command }
       end
     end
 
@@ -47,6 +55,8 @@ module BiblioTech
       cmd("scp") do |cmd|
         cmd.options << options.local_file(filename)
         cmd.options << options.remote_file(remote, filename)
+      end.tap do |cmd|
+        log.info{ cmd.command }
       end
     end
 
@@ -74,7 +84,9 @@ module BiblioTech
             cmd.options << "-o #{opt}"
           end
         end
-      end - escaped_command(command_on_remote)
+      end - escaped_command(command_on_remote).tap do |cmd|
+        log.info{ cmd.command }
+      end
     end
 
     def wipe()

data/lib/bibliotech/config.rb

@@ -1,3 +1,5 @@
+require 'bibliotech/backups/scheduler'
+
 module BiblioTech
   class Config
     class MissingConfig < KeyError; end
@@ -12,6 +14,8 @@ module BiblioTech
       :rsa_files            => [ "rsa_files" ]            ,
       :ssh_options          => [ "ssh_options" ]          ,
       :fetch_dir            => [ "fetched_dir" ]          ,
+      :log_target           => [ "log"                    , "target"    ],
+      :log_level            => [ "log"                    , "level"     ],
       :file                 => [ "backups"                , "file"      ] ,
       :filename             => [ "backups"                , "filename"  ] ,
       :backup_path          => [ "backups"                , "dir"       ] ,
@@ -113,6 +117,32 @@ module BiblioTech
       extract(steps, ["remotes"] + steps)
     end
 
+    def log_target
+      target_path = local_get(:log_target)
+      case target_path
+      when "STDERR", "stderr"
+        return $stderr
+      when "STDOUT", "stdout"
+        return $stdout
+      else
+        require 'fileutils'
+        FileUtils.mkdir_p(File.dirname(target_path))
+        return File.open(target_path, "a")
+      end
+    rescue
+      warn "Trouble opening configured log file - logging to stderr"
+      warn $!.inspect
+      return $STDERR
+    end
+
+    def log_level
+      level = "debug"
+      optionally do
+        level = local_get(:log_level)
+      end
+      return BiblioTech::Logging.log_level(level)
+    end
+
     def ssh_options(for_remote)
       steps = steps_for(:ssh_options) + [for_remote]
       steps_chain =
@@ -202,7 +232,7 @@ module BiblioTech
       @backup_frequency ||= regularize_frequency(local_get(:backup_frequency))
     end
 
-    def each_prune_schedule
+    def prune_schedules
       local_get(:prune_schedule).map do |frequency, limit|
         real_frequency = regularize_frequency(frequency)
         unless real_frequency % backup_frequency == 0
@@ -215,11 +245,11 @@ module BiblioTech
           else
             Integer(limit)
           end
-        [real_frequency, limit]
-      end.sort_by do |frequency, limit|
+        [frequency, real_frequency, limit]
+      end.sort_by do |freq_name, frequency, limit|
         frequency
-      end.each do |frequency, limit|
-        yield(frequency, limit)
+      end.map do |freq_name, frequency, limit|
+        Backups::Scheduler.new(freq_name, frequency, limit)
       end
     end
 

data/lib/bibliotech/logger.rb

@@ -0,0 +1,36 @@
+require 'logger'
+
+module BiblioTech
+  module Logging
+    def self.logger
+      return @logger
+    end
+
+    def self.logger=(value)
+      @logger = value
+    end
+
+    def log
+      return BiblioTech::Logging.logger
+    end
+    module_function :log
+
+    def self.log_level(string)
+      case string
+      when /fatal/i
+        Logger::FATAL
+      when /error/i
+        Logger::ERROR
+      when /warn/i
+        Logger::WARN
+      when /info/i
+        Logger::INFO
+      when /debug/i
+        Logger::DEBUG
+      else
+        Logger::DEBUG
+      end
+    end
+  end
+
+end

data/spec/bibliotech/backup_pruner_spec.rb

@@ -1,5 +1,6 @@
 require 'bibliotech/application'
 require 'bibliotech/backups/pruner'
+require 'bibliotech/backups/scheduler'
 require 'file-sandbox'
 module BiblioTech
   describe Backups::Pruner do
@@ -7,6 +8,7 @@ module BiblioTech
 
     before :each do
       sandbox.new :directory => "db_backups"
+      sandbox.new :file => '.bibliotech/config.yaml', :with_contents => "log:\n  target: ../tmp/test.log"
     end
 
     let :app do
@@ -17,13 +19,17 @@ module BiblioTech
       {:daily => 100}
     end
 
+    let :config do
+      double(Config).tap do |config|
+        allow(config).to receive(:backup_path){ "db_backups" }
+        allow(config).to receive(:backup_name){ "testing" }
+        allow(config).to receive(:backup_frequency){ 60 * 24 }
+        allow(config).to receive(:schedules){ [ Backup::Scheduler.new("daily", 60 * 24, 100) ] }
+      end
+    end
+
     let :pruner do
-      app.pruner({:backups => {
-        :frequency => "daily",
-        :prefix => "testing",
-        :keep => schedule,
-        :dir => "db_backups"
-      }})
+      Backups::Pruner.new(config)
     end
 
     it "should generate a filename for current time" do
@@ -65,6 +71,115 @@ module BiblioTech
         expect(pruner.backup_needed?(Time.now.utc)).to be_truthy
       end
     end
+
+    context "marking for pruning" do
+      before :each do
+        allow(config).to receive(:prune_schedules){
+          [
+            Backups::Scheduler.new("hourlies", 60, 48),
+            Backups::Scheduler.new("dailies", 24 * 60, 14),
+            Backups::Scheduler.new("weeklies", 7 * 24 * 60, 8),
+            Backups::Scheduler.new("monthlies", 30 * 24 * 60, nil)
+          ]
+        }
+
+        Logging.log.debug{ "Start test" }
+      end
+
+      it "should have schedules" do
+        expect(pruner.schedules.length).to eq(4)
+      end
+
+      it "should keep single backup" do
+        sandbox.new :file => "db_backups/#{pruner.filename_for(Time.now.utc)}"
+
+        expect(pruner.pruneable).to be_empty
+
+        expect(pruner.list.length).to eq(1)
+        pruner.list.each do |record|
+          expect(record.keep?).to eq(true)
+        end
+      end
+
+      it "should keep 48 hours of backup" do
+        now = Time.now.utc
+        (0..47).each do |interval|
+          sandbox.new :file => "db_backups/#{pruner.filename_for(Time.now.utc - interval * 60 * 60)}"
+        end
+
+        expect(pruner.pruneable).to be_empty
+
+        expect(pruner.list.length).to eq(48)
+        pruner.list.each do |record|
+          expect(record.keep?).to eq(true)
+        end
+      end
+
+      it "should prune old backups" do
+        now = Time.now.utc
+        (0..470).each do |interval|
+          sandbox.new :file => "db_backups/#{pruner.filename_for(now - interval * 60 * 60)}"
+        end
+
+        expect(pruner.pruneable.length).to eq(471 - 48 - (14 - 2) - 1) # 2 dailies hourly etc.
+
+        expect(pruner.list.length).to eq(471)
+      end
+
+
+      context "repruning" do
+        shared_examples_for "well mannered pruner" do
+          it "should not re-prune old backups" do
+            (0..47).each do |interval|
+              sandbox.new :file => "db_backups/#{pruner.filename_for(now - interval * 60 * 60)}"
+            end
+            (0..11).each do |interval|
+              sandbox.new :file => "db_backups/#{pruner.filename_for(now - 48 * 60 * 60 - interval * 24 * 60 * 60)}"
+            end
+            sandbox.new :file => "db_backups/#{pruner.filename_for(now - 48 * 60 * 60 - 12 * 24 * 60 * 60 - 7 * 24 * 60 * 60)}"
+
+            expect(pruner.pruneable).to be_empty
+
+            expect(pruner.list.length).to eq(48 + 12 + 1)
+            pruner.list.each do |record|
+              expect(record.keep?).to eq(true)
+            end
+          end
+        end
+
+        context "right now" do
+          it_behaves_like "well mannered pruner" do
+            let :now do
+              Time.now.utc
+            end
+          end
+        end
+
+        context "30 minutes ago" do
+          it_behaves_like "well mannered pruner" do
+            let :now do
+              Time.now.utc - 30 * 60
+            end
+          end
+        end
+
+        context "60 minutes ago" do
+          it_behaves_like "well mannered pruner" do
+            let :now do
+              Time.now.utc - 60 * 60
+            end
+          end
+        end
+
+        context "90 minutes ago" do
+          it_behaves_like "well mannered pruner" do
+            let :now do
+              Time.now.utc - 90 * 60
+            end
+          end
+        end
+      end
+    end
   end
 
   describe Backups::PruneList do

data/spec/bibliotech/backup_scheduler_spec.rb

@@ -23,7 +23,7 @@ module BiblioTech::Backups
 
     describe "without a limit" do
       let :scheduler do
-        Scheduler.new(60, nil)
+        Scheduler.new("hourly", 60, nil)
       end
 
       context "when there's more than enough backups" do
@@ -39,7 +39,7 @@ module BiblioTech::Backups
 
     describe "with a limit" do
       let :scheduler do
-        Scheduler.new(60, 8)
+        Scheduler.new("hourly", 60, 8)
       end
 
       context "when there's just enough backups" do

data/spec/bibliotech/config_spec.rb

@@ -89,10 +89,8 @@ module BiblioTech
       end
 
       let :schedule_array do
-        [].tap do |array|
-          config.each_prune_schedule do |freq, lim|
-            array << [freq, lim]
-          end
+        config.prune_schedules.map do |sched|
+          [sched.frequency, sched.limit]
         end
       end
 

data/spec/spec_helper.rb

@@ -1,2 +1,6 @@
 require 'rspec'
 require File.join(File.dirname(__FILE__), '..', 'lib', 'bibliotech')
+
+File.open('tmp/test.log', "w") do |logfile|
+  logfile.truncate(0)
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bibliotech
 version: !ruby/object:Gem::Version
-  version: 0.2.13
+  version: 0.3.0
 platform: ruby
 authors:
 - Evan Dorn
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-04 00:00:00.000000000 Z
+date: 2014-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: caliph
@@ -87,6 +87,7 @@ files:
 - lib/bibliotech/builders/postgres.rb
 - lib/bibliotech/builders/mysql.rb
 - lib/bibliotech/config.rb
+- lib/bibliotech/logger.rb
 - lib/bibliotech/command_runner.rb
 - lib/bibliotech/command_generator.rb
 - lib/bibliotech/rake_lib.rb
@@ -128,7 +129,7 @@ rdoc_options:
 - --main
 - doc/README
 - --title
-- bibliotech-0.2.13 Documentation
+- bibliotech-0.3.0 Documentation
 require_paths:
 - lib/
 required_ruby_version: !ruby/object:Gem::Requirement