pg_shrink 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4d4bd5c0b682c29ae81072befc0f5c962319809a
-  data.tar.gz: d9d4aa0ceae5e3c816dfd8cea704a0f5b3f2a404
+  metadata.gz: 1f5d381b33119abf32ee3e51c3d9efe5997194ff
+  data.tar.gz: 255bcaed0c21c079370b483ac5941a5c7a320c1f
 SHA512:
-  metadata.gz: 9183c4d2b6226d8e0c7b610066333de8f7fa1d731663e5f67241722601336f7749d285b42ef3c84b044c0039b0ba671e0a1085ea622af1d9e086999d1ec92db0
-  data.tar.gz: 0e55a008ff71f575854f2acc1add7dd7a7f7459385eac3fdf63346d94f469ea0c46e86c50b0b627a01249b8d2595c8c3ff375af1d9579d9e4b26acf8e7995b98
+  metadata.gz: 8c6a6fc900c387196c7dfb973337123867b010bb6e7e9ea85e543e69ff23c0d69a1cfab472133b406d4ee421f8a9caf2e650671ca7cc9fee45b35016dffb8783
+  data.tar.gz: 477d58878e5e00c8bf2f73d2818e4e672d5172eff9533c44d97025fdf1013637a380bda218d0e970575c846606471845538efdd8d62734b343738d8a22115ea2

data/Shrinkfile.example

@@ -1,12 +1,16 @@
 filter_table :users do |f|
 
-  # filter_by takes a block and yields the fields of each record (as a hash)
-  # the block should return true to keep the record, false if not.  For
-  # ease of use and extensibility, we allow multiple filter_by blocks
-  # rather than forcing all logic into one block.
-  f.filter_by do |u|
-    u[:id] % 1000 == 0
-  end
+  # filter_by takes a sql condition for the records you would like to keep.
+  # This condition can be as a hash or string
+  f.filter_by("id % 1000 == 0")
+  
+  # NOTE:  It can also take a block that expects the fields of each record (as
+  # a hash) the block should return true to keep the record, false if not.  
+  # However, this approach is substantially slower than the conditions hash
+  # based approach.
+  #f.filter_by do |u|
+  #  u[:id] % 1000 == 0
+  #end
 
   # lock takes a block and yields the fields of each record (as a hash of
   # fieldname => value) If the block returns true this record is immune to all

data/bin/pg_shrink

@@ -37,6 +37,10 @@ Please make sure you have a Shrinkfile or specify one using -c
       options[:force] = true
     end
 
+    verbose_desc = 'run in verbose mode'
+    opts.on('-v', '--verbose', verbose_desc) do
+      options[:log] = true
+    end
 
     opts.on('-h', '--help', 'Show this message and exit') do |h|
       puts opts

data/lib/pg_shrink/database/postgres.rb

@@ -27,6 +27,15 @@ module PgShrink
      end
     end
 
+    def database_name
+      if @opts[:postgres_url]
+        @opts[:postgres_url] =~ /.*\/([^\/]+)$/
+        return $1
+      else
+        @opts[:database]
+      end
+    end
+
     def batch_size
       @opts[:batch_size]
     end
@@ -46,13 +55,18 @@ module PgShrink
       end
       max_id = self.connection["select max(#{primary_key}) from #{table_name}"].
                     first[:max]
-      i = 1;
+      i = 0;
       while i < max_id  do
         sql = "select * from #{table_name} where " +
-                 "#{primary_key} >= #{i} and #{primary_key} < #{i + batch_size}"
-        batch = self.connection[sql].all
+                 "#{primary_key} > #{i} limit #{batch_size}"
+        batch = self.connection[sql].all.compact
+
         yield(batch)
-        i = i + batch_size
+        if batch.any?
+          i = batch.last[primary_key]
+        else
+          break
+        end
       end
     end
 
@@ -87,8 +101,67 @@ module PgShrink
       self.connection.from(table_name).where(opts).all
     end
 
-    def delete_records(table_name, condition_to_delete)
-      self.connection.from(table_name).where(condition_to_delete).delete
+    def delete_records(table_name, conditions, exclude_conditions = [])
+      query = connection.from(table_name)
+      Array.wrap(conditions).compact.each do |cond|
+        query = query.where(cond)
+      end
+      Array.wrap(exclude_conditions).compact.each do |exclude_cond|
+        query = query.exclude(exclude_cond)
+      end
+      query.delete
+    end
+
+    def propagate_delete(opts)
+      # what we conceptually want to do is delete the left outer join where id is null.
+      # That's not working in postgres, so we instead use where not exists.  Docs
+      # indicate using where not exists and select 1 in this case.
+      # See:
+      # http://www.postgresql.org/docs/current/interactive/functions-subquery.html#FUNCTIONS-SUBQUERY-EXISTS
+      query = "DELETE FROM #{opts[:child_table]} WHERE NOT EXISTS (" +
+                "SELECT 1 from #{opts[:parent_table]} where " +
+                "#{opts[:child_table]}.#{opts[:child_key]} = " +
+                "#{opts[:parent_table]}.#{opts[:parent_key]}" +
+              ")"
+
+
+      # Outside of the join statements, we want to maintain the ease of hash-based
+      # conditions.  Do this by using a query builder but then swapping in delete SQL
+      # in the end.
+      query_builder = connection.from(opts[:child_table])
+      Array.wrap(opts[:conditions]).compact.each do |cond|
+        query_builder = query_builder.where(cond)
+      end
+      Array.wrap(opts[:exclude]).compact.each do |exclude_cond|
+        query_builder = query_builder.exclude(exclude_cond)
+      end
+      sql = query_builder.sql.gsub("WHERE", "AND").
+                              gsub("SELECT * FROM \"#{opts[:child_table]}\"",
+                                   query)
+
+      connection[sql].delete
+    end
+
+    def vacuum_and_reindex!(table_name)
+      self.log("Beginning vacuum on #{table_name}")
+      connection["vacuum full #{table_name}"].first
+      self.log("Beginning reindex on #{table_name}")
+      connection["reindex table #{table_name}"].first
+      self.log("done reindexing #{table_name}")
+    end
+    
+    def vacuum_and_reindex_all!
+      self.log("Beginning full database vacuum")
+      connection["vacuum full"].first
+      self.log("beginning full database reindex")
+      connection["reindex database #{database_name}"].first
+      self.log("done reindexing full database")
+    end
+
+    def shrink!
+      filter!
+      vacuum_and_reindex_all!
+      sanitize!
     end
   end
 end

data/lib/pg_shrink/database.rb

@@ -41,8 +41,21 @@ module PgShrink
       raise "implement in subclass"
     end
 
-    # The delete_records method takes a table name and a condition to delete on.
-    def delete_records(table_name, condition)
+    # The delete_records method takes a table name  a condition to delete on,
+    # and a condition to prevent deletion on.  This can be used to combine
+    # a targeted deletion with exclusions or to delete an entire table but
+    # for some exclusions by passing no conditions but some exclude conditions.
+    def delete_records(table_name, conditions, exclude_conditions = [])
+      raise "implement in subclass"
+    end
+
+    # vacuum and reindex is pg specific... do nothing in other cases
+    def vacuum_and_reindex!(table_name)
+    end
+
+    # This is kind of a leaky abstraction b/c I'm not sure how this would work
+    # outside of sql
+    def propagate_delete(opts)
       raise "implement in subclass"
     end
 
@@ -58,5 +71,15 @@ module PgShrink
       filter!
       sanitize!
     end
+
+    def initialize(opts = {})
+      @opts = opts
+    end
+
+    def log(message)
+      if @opts[:log]
+        puts "#{Time.now}: #{message}"
+      end
+    end
   end
 end

data/lib/pg_shrink/sub_table_filter.rb

@@ -1,6 +1,30 @@
 module PgShrink
   class SubTableFilter < SubTableOperator
 
+    def propagate_table!
+      primary_key = @opts[:primary_key]
+      foreign_key = @opts[:foreign_key]
+      additional_conditions = {}
+      if @opts[:type_key] && @opts[:type]
+        additional_conditions[@opts[:type_key]] = @opts[:type]
+      end
+      self.database.log("Beginning subtable propagation from " +
+               "#{self.parent.table_name} to #{self.table.table_name}")
+      self.database.propagate_delete(:parent_table => self.parent.table_name,
+                                     :child_table => self.table.table_name,
+                                     :parent_key => primary_key,
+                                     :child_key => foreign_key,
+                                     :conditions => additional_conditions,
+                                     :exclude => self.table.lock_opts)
+
+      self.database.log("Done with subtable propagation from " +
+               "#{self.parent.table_name} to #{self.table.table_name}")
+      if self.table.subtable_filters.any?
+        self.database.vacuum_and_reindex!(self.table.table_name)
+        self.table.subtable_filters.each(&:propagate_table!)
+      end
+    end
+
     def propagate!(old_parent_data, new_parent_data)
       return if (old_parent_data.empty? && new_parent_data.empty?)
       old_batch_keys = old_parent_data.map {|record| record[@opts[:primary_key]]}

data/lib/pg_shrink/sub_table_operator.rb

@@ -38,7 +38,6 @@ module PgShrink
       raise "Implement in subclass"
     end
 
-
   end
 end
 

data/lib/pg_shrink/table.rb

@@ -3,7 +3,7 @@ module PgShrink
     attr_accessor :table_name
     attr_accessor :database
     attr_accessor :opts
-    attr_reader :filters, :sanitizers, :subtable_filters, :subtable_sanitizers
+    attr_reader :filters, :sanitizers, :subtable_filters, :subtable_sanitizers, :lock_opts
     # TODO:  Figure out, do we need to be able to support tables with no
     # keys?  If so, how should we handle that?
     def initialize(database, table_name, opts = {})
@@ -31,12 +31,27 @@ module PgShrink
     end
 
     def lock(opts = {}, &block)
-      @lock = block
+      @lock_opts = opts
+      if block_given?
+        puts "WARNING:  Block-based lock on #{self.table_name} will make things SLOW"
+        @lock_block = block
+      end
+    end
+
+    def has_lock?
+      (@lock_opts && @lock_opts.any?) || @lock_block
+    end
+
+    def lock_condition_ok?
+      !@lock_block
     end
 
     def locked?(record)
-      if @lock
-        @lock.call(record)
+      if @lock_block
+        @lock_block.call(record)
+      elsif @lock_opts && @lock_opts.any?
+        raise "Unimplemented:  Condition-based locks with block-based " +
+              "filter on table #{self.table_name}"
       end
     end
 
@@ -101,6 +116,17 @@ module PgShrink
       end
     end
 
+    def condition_filter(filter)
+      self.database.log("Beginning filter on #{table_name}")
+      self.database.delete_records(self.table_name, {}, [filter.opts, lock_opts].compact)
+      self.database.log("Done filtering on #{table_name}")
+      # If there aren't any subtables, there isn't much benefit to vacuuming in
+      # the middle, and we'll wait until we're done with all filters
+      if self.subtable_filters.any?
+        self.database.vacuum_and_reindex!(self.table_name)
+      end
+    end
+
     def filter_batch(batch, &filter_block)
       new_set = batch.select do |record|
         locked?(record) || filter_block.call(record.dup)
@@ -111,11 +137,7 @@ module PgShrink
 
     def sanitize_batch(batch, &sanitize_block)
       new_set = batch.map do |record|
-        if locked?(record)
-          record.dup
-        else
-          sanitize_block.call(record.dup)
-        end
+        sanitize_block.call(record.dup)
       end
       update_records(batch, new_set)
       sanitize_subtables(batch, new_set)
@@ -126,9 +148,14 @@ module PgShrink
         remove!
       else
         self.filters.each do |filter|
-          self.records_in_batches do |batch|
-            self.filter_batch(batch) do |record|
-              filter.apply(record)
+          if filter.conditions? && self.lock_condition_ok?
+            self.condition_filter(filter)
+            self.subtable_filters.each(&:propagate_table!)
+          else
+            self.records_in_batches do |batch|
+              self.filter_batch(batch) do |record|
+                filter.apply(record)
+              end
             end
           end
         end
@@ -146,7 +173,7 @@ module PgShrink
     end
 
     def can_just_remove?
-      self.subtable_filters.empty? && self.subtable_sanitizers.empty? && !@lock
+      self.subtable_filters.empty? && self.subtable_sanitizers.empty? && !has_lock?
     end
 
     # Mark @remove and add filter so that if we're in the simple case we can

data/lib/pg_shrink/table_filter.rb

@@ -1,14 +1,36 @@
 module PgShrink
   class TableFilter
-    attr_accessor :table
-    def initialize(table, opts, &block)
+    attr_accessor :table, :opts
+    def initialize(table, opts = nil, &block)
       self.table = table
-      @opts = opts # Currently not used, but who knows
-      @block = block
+      @opts = opts
+      @block = block if block_given?
+    end
+
+    def conditions?
+      # use !empty instead of any? because we accept string conditions
+      !@block
     end
 
     def apply(hash)
-      @block.call(hash)
+      if @block
+        @block.call(hash)
+      # if we have a straightforwards conditions hash can just do in place comparisons
+      elsif @opts.is_a?(Hash)
+        @opts.each do |k, v|
+          if [Array, Range].include?(v.class)
+            return false unless v.include?(hash[k])
+          elsif [String, Integer, Float].include?(v.class)
+            return false unless hash[k] == v
+          else
+            raise "Unsupported condition type for mixing with block locks: #{v.class}"
+          end
+        end
+        return true
+      #TODO:  Figure out if this case matters and we want to support it.
+      elsif @opts.is_a?(String)
+        raise "Unsupported:  Mixing string conditions with block locks"
+      end
     end
   end
 end

data/lib/pg_shrink/version.rb

@@ -1,3 +1,3 @@
 module PgShrink
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/pg_shrink.rb

@@ -19,7 +19,8 @@ module PgShrink
       url: nil,
       config: 'Shrinkfile',
       force: false,
-      batch_size: 10000
+      batch_size: 10000,
+      log: false,
     }
   end
 
@@ -60,7 +61,8 @@ module PgShrink
     end
 
     database = Database::Postgres.new(:postgres_url => options[:url],
-                                      :batch_size => batch_size)
+                                      :batch_size => batch_size,
+                                      :log => options[:log])
 
     database.instance_eval(File.read(options[:config]), options[:config], 1)
 

data/spec/pg_shrink_spec.rb

@@ -53,6 +53,135 @@ describe PgShrink do
             end
           end
         end
+        describe "with a conditions string" do
+          before(:each) do
+            database.filter_table(:users) do |f|
+              f.filter_by("name like '%test 1%'")
+            end
+          end
+
+          it "Should not call records in batches" do
+            expect(database).not_to receive(:records_in_batches)
+            database.shrink!
+          end
+
+          it "Should call delete_records once" do
+            expect(database).to receive(:delete_records).once
+            database.shrink!
+          end
+
+          it "Should result in the appropriate records being deleted" do
+            database.shrink!
+            remaining_users = database.connection.from(:users).all
+            # 1 and 10-19
+            expect(remaining_users.size).to eq(11)
+          end
+        end
+
+        describe "when filtering just with a conditions hash" do
+          before(:each) do
+            database.filter_table(:users) do |f|
+              f.filter_by({
+                :name => ["test 1", "test 2", "test 3", "test 4", "test 5",
+                          "test 6", "test 7", "test 8", "test 9", "test 10"]
+              })
+            end
+          end
+
+          it "Should not call records in batches" do
+            expect(database).not_to receive(:records_in_batches)
+            database.shrink!
+          end
+
+          it "Should call delete_records once" do
+            expect(database).to receive(:delete_records).once
+            database.shrink!
+          end
+
+          it "Should result in the appropriate records being deleted" do
+            database.shrink!
+            remaining_users = database.connection.from(:users).all
+            expect(remaining_users.size).to eq(10)
+          end
+
+          describe "and a condition hash lock" do
+            before(:each) do
+              database.filter_table(:users) do |f|
+                f.lock(name: "test 11")
+              end
+            end
+            it "Should not call records in batches" do
+              expect(database).not_to receive(:records_in_batches)
+              database.shrink!
+            end
+
+            it "Should call delete_records once" do
+              expect(database).to receive(:delete_records).once
+              database.shrink!
+            end
+            it "Still results in the appropriate records being deleted" do
+              database.shrink!
+              remaining_users = database.connection.from(:users).all
+              expect(remaining_users.size).to eq(11)
+            end
+          end
+
+          describe "and a condition string lock" do
+            before(:each) do
+              database.filter_table(:users) do |f|
+                f.lock("name = 'test 11'")
+              end
+            end
+            it "Should not call records in batches" do
+              expect(database).not_to receive(:records_in_batches)
+              database.shrink!
+            end
+
+            it "Should call delete_records once" do
+              expect(database).to receive(:delete_records).once
+              database.shrink!
+            end
+            it "Still results in the appropriate records being deleted" do
+              database.shrink!
+              remaining_users = database.connection.from(:users).all
+              expect(remaining_users.size).to eq(11)
+            end
+          end
+
+          describe "and a block-based lock" do
+            before(:each) do
+              database.filter_table(:users) do |f|
+                f.lock do |u|
+                  u[:name] == "test 11"
+                end
+              end
+            end
+            it "falls back to results in batches" do
+              expect(database).to receive(:records_in_batches)
+              database.shrink!
+            end
+            it "Still results in the appropriate records being deleted" do
+              database.shrink!
+              remaining_users = database.connection.from(:users).all
+              expect(remaining_users.size).to eq(11)
+            end
+
+          end
+
+          describe "with a subtable_filter" do
+            before(:each) do
+              database.filter_table(:users) do |f|
+                f.filter_subtable(:user_preferences, :foreign_key => :user_id)
+              end
+            end
+
+            it "should remove the appropriate subtable records" do
+              database.shrink!
+              remaining_prefs = database.connection.from(:user_preferences).all
+              expect(remaining_prefs.size).to eq(30)
+            end
+          end
+        end
 
         it "Should not run delete if there is nothing filtered" do
           database.filter_table(:users) do |f|
@@ -312,6 +441,90 @@ describe PgShrink do
         end
       end
 
+      describe "with condition filters" do
+        describe "simple cascade" do
+          before(:each) do
+            database.filter_table(:users) do |f|
+              f.filter_by(:name => "test 1")
+              f.filter_subtable(:preferences, :foreign_key => :context_id,
+                                :type_key => :context_type, :type => 'User')
+            end
+            database.filter!
+          end
+          it "will filter prefs with context_type 'User'" do
+            #
+            remaining_user = database.connection.from(:users).first
+            remaining_preferences = database.connection.from(:preferences).
+              where(:context_type => 'User').all
+            expect(remaining_preferences.size).to eq(3)
+            expect(remaining_preferences.map {|u| u[:context_id]}.uniq).
+              to eq([remaining_user[:id]])
+          end
+
+          it "will not filter preferences without context_type user" do
+            remaining_preferences = database.connection.from(:preferences).
+              where(:context_type => 'OtherClass').all
+            expect(remaining_preferences.size).to eq(20)
+          end
+        end
+        describe "an extra layer of polymorphic subtables" do
+          before(:all) do
+            connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
+                         connection
+            PgSpecHelper.create_table(connection, :preference_dependents,
+                                      {'context_id' => 'integer',
+                                       'context_type' => 'character varying(256)',
+                                       'value' => 'character varying(256)'})
+          end
+
+          before(:each) do
+            PgSpecHelper.clear_table(database.connection, :preference_dependents)
+            prefs = database.connection.from(:preferences).all
+            prefs.each do |pref|
+              database.connection.run(
+                "insert into preference_dependents " +
+                "(context_id, context_type, value) " +
+                "values (#{pref[:id]}, 'Preference', 'depvalue#{pref[:id]}')")
+
+              database.connection.run(
+                "insert into preference_dependents " +
+                "(context_id, context_type, value) " +
+                "values (#{pref[:id]}, 'SomeOtherClass', 'fakevalue#{pref[:id]}')")
+
+            end
+
+            database.filter_table(:users) do |f|
+              f.filter_by(:name => "test 1")
+              f.filter_subtable(:preferences, :foreign_key => :context_id,
+                                :type_key => :context_type, :type => 'User')
+            end
+
+            database.filter_table(:preferences) do |f|
+              f.filter_subtable(:preference_dependents,
+                                :foreign_key => :context_id,
+                                :type_key => :context_type,
+                                :type => 'Preference')
+            end
+            database.filter!
+          end
+          it "will filter preference dependents associated with preferences" do
+            remaining_preferences = database.connection.from(:preferences).all
+            remaining_dependents = database.connection.
+              from(:preference_dependents).
+              where(:context_type => 'Preference').all
+
+            expect(remaining_dependents.size).to eq(remaining_preferences.size)
+          end
+
+          it "will not filter preference dependents with different type" do
+            other_dependents = database.connection.
+              from(:preference_dependents).
+              where(:context_type => 'SomeOtherClass').all
+            expect(other_dependents.size).to eq(80)
+          end
+        end
+      end
+
       describe "simple two table filtering" do
         before(:each) do
           database.filter_table(:users) do |f|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_shrink
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Kevin Ball
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-02 00:00:00.000000000 Z
+date: 2014-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg