bibliotech 0.2.12 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49e433e80626f0ee2eba944de215c187cb975ace
-  data.tar.gz: 88b3fea79e358263eb44f0c595df403664cad512
+  metadata.gz: a22ff69ff51f9da65c676de7b84084129f2a9499
+  data.tar.gz: 414d7b654e61ab08ac35e26c3ecebb161761c4dd
 SHA512:
-  metadata.gz: bf161c4d8a01523f787d325decbe783aec812b8c8c8d71d198074e386aecabc566c7835f54c22084fe35f3ece849f7f108fe0282a1cae35a9033565693ae6f04
-  data.tar.gz: b92108171ef774da7c5679e72c48e00692e0a56d40020c28a38aeeafa4931c3fb7b103deb9a5b367a5aef6e7f913aab3a3106118c551b56c709808f2a28953b7
+  metadata.gz: ee5d38a9f9a2a393e9b44d07f15dba07e8285cad1b5a931ad5de2b3ca955e13b15962aec59cc76c699fd1aa494c10460e5ce1a9d26e4ac3e6085fdc6d4cfe8d7
+  data.tar.gz: 490311e700290021309b7991116fb11bc79db31be92b53c87144a355c7527ad3139ff41994f3c72a872bec96c7060998d1cb47c91668531795fdfc661564512b

data/lib/bibliotech/application.rb

@@ -67,13 +67,13 @@ module BiblioTech
     end
 
     #pull a dump from a remote
-    def get(options)
-      @shell.run(commands.fetch(options))
+    def get(remote, options)
+      @shell.run(commands.fetch(remote, options))
     end
 
     #push a dump to a remote
-    def send(options)
-      @shell.run(commands.push(options))
+    def send(remote, options)
+      @shell.run(commands.push(remote, options))
     end
 
     #clean up the DB dumps
@@ -86,8 +86,8 @@ module BiblioTech
       pruner(options || {}).most_recent.path
     end
 
-    def remote_cli(remote, command, options)
-      @shell.run(commands.ssh_cli(remote, command, options))
+    def remote_cli(remote, command, *options)
+      @shell.run(commands.remote_cli(remote, command, *options))
     end
   end
 

data/lib/bibliotech/backups/scheduler.rb

@@ -10,41 +10,52 @@ module BiblioTech
         @limit = nil if limit == "all"
       end
 
-      def end_time(file_list)
-        return Time.at(0) if file_list.empty?
-        file_list.map{|record| record.timestamp}.max
+      def freq_seconds
+        frequency * 60
+      end
+
+      def range
+        freq_seconds / 2
       end
 
-      def compute_start_time(file_list)
+      # The earliest possible time to keep a file in.
+      def compute_earliest_time(file_list)
         limit_time = Time.at(0)
         return limit_time if file_list.empty?
         unless limit.nil?
-          limit_time = end_time(file_list) - limit * freq_seconds
+          limit_time = latest_time(file_list) - limit * freq_seconds
         end
-        [limit_time, file_list.map{|record| record.timestamp}.min - range].max
+        [limit_time, file_list.last.timestamp - range].max
       end
 
-      def freq_seconds
-        frequency * 60
+      # The latest possible time to keep a file in.
+      def latest_time(file_list)
+        return Time.at(0) if file_list.empty?
+        file_list.first.timestamp
       end
 
-      def range
-        freq_seconds / 2
-      end
+      # Working from the latest time backwards, mark the closest file to the
+      # appropriate frequencies as keepable
+      def mark(original_file_list)
+        file_list = original_file_list.sort_by{|record| -record.timestamp.to_i} #sort from newest to oldest
 
-      def mark(file_list)
-        time = end_time(file_list)
-        start_time = compute_start_time(file_list)
-        while time > start_time do
-          closest = file_list.min_by do |record|
-            (record.timestamp - time).abs
+        time = latest_time(file_list)
+        earliest_time = compute_earliest_time(file_list)
+        while time > earliest_time do
+          file_list.delete_if do |record|
+            record.timestamp > time
           end
-          if (closest.timestamp - time).abs < range
+
+          break if file_list.empty?
+
+          closest = file_list.first
+
+          if (time - closest.timestamp) < freq_seconds
             closest.keep = true
           end
           time -= freq_seconds
         end
-        return file_list
+        return original_file_list
       end
     end
   end

data/lib/bibliotech/builders/gzip.rb

@@ -3,11 +3,13 @@ require 'bibliotech/builders/file'
 module BiblioTech
   module Builders
     class GzipExpander < FileInput
-      register(/.*\.gz\z/)
-      register(/.*\.gzip\z/)
+      PATTERNS = [ /.*\.gz\z/, /.*\.gzip\z/ ]
+      PATTERNS.each do |pattern|
+        register pattern
+      end
 
       def go(command)
-        command = cmd("gunzip", file) | command
+        command = cmd("gunzip", "-c", file) | command
       end
     end
 

data/lib/bibliotech/command_generator.rb

@@ -31,10 +31,14 @@ module BiblioTech
 
     def fetch(remote, filename, options = nil)
       options = config.merge(options || {})
-      cmd("scp") do |cmd|
+      local_path = options.local_file(filename)
+      cmd("mkdir") do |cmd|
+        cmd.options << "--parents"
+        cmd.options << File::dirname(local_path)
+      end & cmd("scp") do |cmd|
         options.optionally{ cmd.options << "-i #{options.id_file(remote)}" }
         cmd.options << options.remote_file(remote, filename)
-        cmd.options << options.local_file(filename)
+        cmd.options << local_path
       end
     end
 

data/lib/bibliotech/config.rb

@@ -5,16 +5,16 @@ module BiblioTech
     CONFIG_STEPS = {
       :database_config_file => [ "database_config_file" ] ,
       :database_config_env  => [ "database_config_env"  ] ,
+      :root_path            => [ "path" ]                 ,
       :host                 => [ "host" ]                 ,
       :port                 => [ "port" ]                 ,
       :user                 => [ "user" ]                 ,
       :rsa_files            => [ "rsa_files" ]            ,
       :ssh_options          => [ "ssh_options" ]          ,
+      :fetch_dir            => [ "fetched_dir" ]          ,
       :file                 => [ "backups"                , "file"      ] ,
       :filename             => [ "backups"                , "filename"  ] ,
       :backup_path          => [ "backups"                , "dir"       ] ,
-      :root_path            => [ "path" ]                 ,
-      :fetch_dir            => [ "fetched_dir" ]          ,
       :compressor           => [ "backups"                , "compress"  ] ,
       :prune_schedule       => [ "backups"                , "keep"      ] ,
       :backup_name          => [ "backups"                , "prefix"    ] ,
@@ -208,6 +208,13 @@ module BiblioTech
         unless real_frequency % backup_frequency == 0
           raise "Pruning frequency #{real_frequency}:#{frequency} is not a multiple of backup frequency: #{backup_frequency}:#{local_get(:backup_frequency)}"
         end
+        limit =
+          case limit
+          when "all"
+            nil
+          else
+            Integer(limit)
+          end
         [real_frequency, limit]
       end.sort_by do |frequency, limit|
         frequency
@@ -244,9 +251,9 @@ module BiblioTech
 
     def expander
       if remote.nil?
-        local_get(:expander)
+        local_get(:compressor)
       else
-        remote_get(remote, :expander)
+        remote_get(remote, :compressor)
       end
     end
 

data/lib/bibliotech/rake_lib.rb

@@ -74,16 +74,19 @@ module BiblioTech
         namespace :remote_sync do
           desc "Pull the latest DB dump from the remote server into our local DB"
           task :down do
-            filename = app.remote_cli(remote, "latest")
-            app.get(remote, filename)
-            app.import(:backups => { :filename => filename})
+            result = app.remote_cli(remote, "latest")
+            result.must_succeed!
+            filename = result.stdout.chomp
+
+            app.get(remote, filename).must_succeed!
+            app.import(:backups => { :file => app.config.local_file(filename)}).must_succeed!
           end
 
           desc "Push the latest local DB dump to the remote server's DB"
           task :up do
             filename = app.latest
-            app.send(remote, filename)
-            app.remote_cli(remote, "load", filename)
+            app.send(remote, filename).must_succeed!
+            app.remote_cli(remote, "load", filename).must_succeed!
           end
         end
       end

data/spec/bibliotech/backup_pruner_spec.rb

@@ -13,11 +13,15 @@ module BiblioTech
       App.new
     end
 
+    let :schedule do
+      {:daily => 100}
+    end
+
     let :pruner do
       app.pruner({:backups => {
         :frequency => "daily",
         :prefix => "testing",
-        :keep => {:daily => 100},
+        :keep => schedule,
         :dir => "db_backups"
       }})
     end

data/spec/bibliotech/backup_scheduler_spec.rb

@@ -5,7 +5,7 @@ module BiblioTech::Backups
     let(:test_jitter){ 0 }
 
     let :unfiltered_files do
-      (0..interval).step(frequency).map do |seconds| #every 15 seconds for 8 hours
+      (0..interval).step(frequency).map do |seconds| #e.g. every 15 seconds for 8 hours
         seconds = seconds - test_jitter/2 + rand(test_jitter)
         FileRecord.new("", Time.now - seconds)
       end
@@ -27,12 +27,12 @@ module BiblioTech::Backups
       end
 
       context "when there's more than enough backups" do
-        let(:interval){ 60*60*12 - 1}
+        let(:interval){ 60*60*12 - test_jitter}
         let(:frequency) { 15 }
         let(:test_jitter){ 60 }
 
         it "should mark 8 files kept" do
-          expect(kept_files.count).to eql 13
+          expect(kept_files.count).to eql 12
         end
       end
     end
@@ -43,7 +43,7 @@ module BiblioTech::Backups
       end
 
       context "when there's just enough backups" do
-        let(:interval){ 60*60*8 - 1 }
+        let(:interval){ 60*60*8 - test_jitter }
         let(:frequency){ 60*8 }
         let(:test_jitter){ 60 }
 
@@ -85,12 +85,12 @@ module BiblioTech::Backups
       end
 
       context "when there are too few backups" do
-        let(:interval){ 60*60*4 - 1 }
+        let(:interval){ 60*60*4 - test_jitter }
         let(:frequency){ 60*8 }
         let(:test_jitter){ 60 }
 
-        it "should mark 5 files kept" do
-          expect(kept_files.count).to eql 5
+        it "should mark 4 files kept" do
+          expect(kept_files.count).to eql 4
         end
       end
 

data/spec/bibliotech/command_generator/mysql_spec.rb

@@ -18,7 +18,7 @@ module BiblioTech
     let( :filename )  { "export.sql"  }
     let( :path     )  { "/some/path"  }
 
-    let :base_config_hash do
+      let :base_config_hash do
       { "database_config" => {
         "adapter" => :mysql,
         "database" => db_name,
@@ -158,7 +158,7 @@ module BiblioTech
 
             it { expect(command).to be_a(Caliph::PipelineChain) }
             it { expect(first_cmd.executable).to eq('gunzip') }
-            it { expect(first_cmd.options).to eq(["#{path}/#{filename}.gz"]) }
+            it { expect(first_cmd.options).to eq(["-c", "#{path}/#{filename}.gz"]) }
           end
         end
       end

data/spec/bibliotech/command_generator/postgres_spec.rb

@@ -171,7 +171,7 @@ module BiblioTech
 
             it { expect(command).to be_a(Caliph::PipelineChain) }
             it { expect(first_cmd.executable).to eq('gunzip') }
-            it { expect(first_cmd.options).to eq(["#{path}/#{filename}.gz"]) }
+            it { expect(first_cmd.options).to eq(["-c", "#{path}/#{filename}.gz"]) }
           end
         end
       end

data/spec/bibliotech/command_generator_spec.rb

@@ -70,7 +70,7 @@ module BiblioTech
       end
 
       it "should produce a fetch command" do
-        expect(generator.fetch("staging", "latest.sql.gz").command).to match(/\Ascp.*@.*latest\.sql\.gz.*latest\.sql\.gz\z/)
+        expect(generator.fetch("staging", "latest.sql.gz").command).to match(/scp.*@.*latest\.sql\.gz.*latest\.sql\.gz\z/)
       end
 
       it "should produce a push command" do

data/spec/bibliotech/config_spec.rb

@@ -144,12 +144,13 @@ module BiblioTech
             "keep" => {
             "hourlies" => 24,
             "daily" => 7,
-            "weeklies" => 4
+            "weeklies" => 4,
+            "monthly" => "all"
           }}}
         end
 
         it "should produce correct schedule" do
-          expect(schedule_array).to contain_exactly([60, 24], [60*24, 7], [60*24*7, 4])
+          expect(schedule_array).to contain_exactly([60, 24], [60*24, 7], [60*24*7, 4], [60*24*30, nil])
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bibliotech
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.13
 platform: ruby
 authors:
 - Evan Dorn
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-21 00:00:00.000000000 Z
+date: 2014-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: caliph
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: valise
   requirement: !ruby/object:Gem::Requirement
@@ -128,7 +128,7 @@ rdoc_options:
 - --main
 - doc/README
 - --title
-- bibliotech-0.2.12 Documentation
+- bibliotech-0.2.13 Documentation
 require_paths:
 - lib/
 required_ruby_version: !ruby/object:Gem::Requirement