postmodern 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8584fc36609820bec538da1d614c2fa8e51e135a
-  data.tar.gz: 6e4b0248349daa0c274b06bdfd39120ff3655bb4
+  metadata.gz: 330a276a44bee39eeedd67c516efb9302db306a0
+  data.tar.gz: b32e4ff5c4a05d922725cfbf387e660c85918b34
 SHA512:
-  metadata.gz: d73e87d8fc6f856fe8af0ab931af813f0d0aeb857bd9d4ba161139f7d3768059d6093467fcf5c0bd4bccd5a09ee924620424de58b7734480792245af4b2597ff
-  data.tar.gz: 878c15c555776ec7fb49ce94e2ea00c7925425852b5bda364c6056439994e94eb4975562418018bcc84542b934645fa307a792e52c26e5a6f54ff3638497be0c
+  metadata.gz: ae60db81ea441c8358df814c734be3ef5b9998bdcaa96f0a2936f1a07fdff24f1dff6ca56d4d23937815358f329ae381ac8f03a673c419afb8e3f586f411d752
+  data.tar.gz: b9009ef87cd0abfc7b37b65ca960e506c2be47e885333424924ac4604f4b61522ed2882ba248d5568381dd1127bbe8059bb5851e75a1455cc182e296825bbca9

data/README.md

@@ -36,6 +36,7 @@ Usage: postmodern (vacuum|freeze) <options>
     -W, --password PASS
 
     -t, --timeout TIMEOUT            Halt after timeout minutes -- default 120
+    -P, --pause PAUSE                Pause (minutes) after each table vacuum -- default 10
     -d, --database DB                Database to vacuum. Required.
 
     -B, --tablesize BYTES            minimum table size to vacuum -- default 1000000
@@ -65,6 +66,11 @@ database activity. They vacuum or vacuum freeze each table that requires
 it (based on command line options). Before each operation, the scripts check
 to make sure they have not gone longer than `--timeout` seconds.
 
+`--pause` is useful to allow I/O-bound replicas to catch up on replication
+before starting new vacuum events. When vacuuming large tables with many
+dead tuples, a lot of changes need to be sent to replicas. When using replicas
+with spinning disks, this can saturate the I/O of the disk array.
+
 ### WAL archives
 
 The wal archiving scripts packaged in this gem are intended to serve as

data/lib/postmodern/vacuum/vacuum.rb

@@ -42,6 +42,10 @@ module Postmodern
             self.options[:timeout] = opt
           end
 
+          opts.on('-P', '--pause PAUSE', 'Pause (minutes) after each table vacuum -- default 10') do |opt|
+            self.options[:pause] = opt
+          end
+
           opts.on('-d', '--database DB', 'Database to vacuum. Required.') do |opt|
             self.options[:database] = opt
           end
@@ -101,13 +105,15 @@ module Postmodern
       end
 
       def vacuum
-        tables_to_vacuum.each do |table|
+        tables_to_vacuum_size = tables_to_vacuum.size
+        tables_to_vacuum.each_with_index do |table,index|
           Postmodern.logger.info "Vacuuming #{table}"
           adapter.execute(vacuum_statement(table)) unless dryrun?
           if timedout?
             Postmodern.logger.warn "Vacuuming timed out"
             break
           end
+          pause unless index == tables_to_vacuum_size - 1
         end
         Postmodern.logger.info "Vacuuming finished"
       end
@@ -120,6 +126,12 @@ module Postmodern
         Time.now >= start_time + (options[:timeout].to_i * 60)
       end
 
+      def pause
+        pause_time = options[:pause].to_i * 60
+        Postmodern.logger.info "Pausing before next vacuum for #{pause_time} minutes."
+        sleep(pause_time) unless dryrun?
+      end
+
       def dryrun?
         !!options[:dryrun]
       end

data/lib/postmodern/version.rb

@@ -1,3 +1,3 @@
 module Postmodern
-  VERSION = "0.2.3"
+  VERSION = "0.3.0"
 end

data/spec/features/freeze_spec.rb

@@ -10,6 +10,7 @@ Usage: postmodern (vacuum|freeze) <options>
     -W, --password PASS
 
     -t, --timeout TIMEOUT            Halt after timeout minutes -- default 120
+    -P, --pause PAUSE                Pause (minutes) after each table vacuum -- default 10
     -d, --database DB                Database to vacuum. Required.
 
     -B, --tablesize BYTES            minimum table size to vacuum -- default 1000000

data/spec/features/vacuum_spec.rb

@@ -10,6 +10,7 @@ Usage: postmodern (vacuum|freeze) <options>
     -W, --password PASS
 
     -t, --timeout TIMEOUT            Halt after timeout minutes -- default 120
+    -P, --pause PAUSE                Pause (minutes) after each table vacuum -- default 10
     -d, --database DB                Database to vacuum. Required.
 
     -B, --tablesize BYTES            minimum table size to vacuum -- default 1000000

data/spec/postmodern/vacuum/freeze_spec.rb

@@ -47,6 +47,7 @@ LIMIT 1000;
     before do
       allow(Postmodern::DB::Adapter).to receive(:new).and_return(adapter)
       allow(command).to receive(:tables_to_vacuum).and_return(tables_to_vacuum)
+      allow(command).to receive(:pause)
     end
 
     it "vacuums each table" do

data/spec/postmodern/vacuum/vacuum_spec.rb

@@ -50,13 +50,14 @@ describe Postmodern::Vacuum::Vacuum do
     let(:args) { %w(--database mydb) }
 
     before do
-      allow(command).to receive(:adapter).and_return(adapter)
+      allow(command).to receive(:configure_vacuum_cost)
+      allow(command).to receive(:vacuum)
     end
 
     it 'executes vacuum operations in order' do
-      expect(command).to receive(:configure_vacuum_cost).once
-      expect(command).to receive(:vacuum).once
       command.run
+      expect(command).to have_received(:configure_vacuum_cost).once
+      expect(command).to have_received(:vacuum).once
     end
   end
 
@@ -109,8 +110,8 @@ describe Postmodern::Vacuum::Vacuum do
 
     it 'finds list of tables to vacuum' do
       result = [
-        { 'full_table_name' => 'table1'},
-        { 'full_table_name' => 'table2'},
+        {'full_table_name' => 'table1'},
+        {'full_table_name' => 'table2'},
       ]
       allow(command).to receive(:adapter).and_return(adapter)
       expect(adapter).to receive(:execute).with(%Q{WITH deadrow_tables AS (
@@ -137,6 +138,7 @@ ORDER BY dead_pct DESC, table_bytes DESC;
     before do
       allow(Postmodern::DB::Adapter).to receive(:new).and_return(adapter)
       allow(command).to receive(:tables_to_vacuum).and_return(tables_to_vacuum)
+      allow(command).to receive(:pause)
     end
 
     let(:tables_to_vacuum) { %w(table1 table2 table3) }
@@ -163,6 +165,11 @@ ORDER BY dead_pct DESC, table_bytes DESC;
       expect(adapter).not_to have_received(:execute).with("VACUUM ANALYZE %s" % tables_to_vacuum[1])
       expect(adapter).not_to have_received(:execute).with("VACUUM ANALYZE %s" % tables_to_vacuum[2])
     end
+
+    it 'pauses between each vacuum' do
+      command.vacuum
+      expect(command).to have_received(:pause).exactly(2).times
+    end
   end
 
   describe '#dryrun?' do
@@ -183,6 +190,29 @@ ORDER BY dead_pct DESC, table_bytes DESC;
     end
   end
 
+  describe '#pause' do
+    let(:time) { Time.now }
+    let(:args) { %w(--d db -P 30) }
+
+    before do
+      allow(command).to receive(:sleep)
+    end
+
+    it "Should sleep for specified time" do
+      command.pause
+      expect(command).to have_received(:sleep).with(30 * 60).once
+    end
+
+    context 'during dry-run' do
+      let(:args) { %w(--d db -P 30 --dry-run) }
+
+      it "Should not sleep during dry-run" do
+        command.pause
+        expect(command).not_to have_received(:sleep)
+      end
+    end
+  end
+
   describe '#timedout?' do
     let(:time) { Time.now }
     let(:args) { %w(--d db -t 15) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postmodern
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Eric Saxby
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-01 00:00:00.000000000 Z
+date: 2014-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg