pseudo_cleaner 0.0.35 → 0.0.36

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YzliYTc5NmFlMGQyM2E2OTZiNmNkZmI1NWQzOGViMWZmZjRiNWFiZA==
+    MjIyM2E5NGJmZGQ4YmRhMzBkOTMzMDEwYmRmY2RjNmZhMmY1YzljNw==
   data.tar.gz: !binary |-
-    Nzk4NTNhODc2ZmQ0NGU5YWIyZjVkNmI4YWYyMjBiZmFlYWE3OWQ1ZQ==
+    YmM3ZjJmZDFhMmRlMjNmMDU0ZTVjNDM3NzA3YjYxMjRjYTgyMTEwZg==
 SHA512:
   metadata.gz: !binary |-
-    MGZkODUwODZiOGFjMzhjYTY0MjI4M2FmYmFhOWVhNDM5MjM1YjRmODE3N2Y5
-    N2M1ZjNhNDk3YWZjYzBjYmQ5NTU0MGJhMzQ0ODYyNzI5ZDQ0NmJkZjE2YTc5
-    NmM0MTU1YzI3YzI2MWU5ZjVhZTljOGQ5NzczZDA0ZGZmNDA3ODU=
+    ODAzYzRjZGJjNjIzMTUxOGZlNTAyNjUxYWNmODY1ZGZkNTRhMGUwOWU4Y2Y3
+    ZjliZWJlZWQzY2JhMDQ4ODdmYjc1YTUwZGJjYTY5NDAwYjU1YmRmMGE5Njlm
+    N2JmNzVhNGJhYzc2N2EzZmY0MjBiNTQyMzY3MTYxYWRjNWZlYWQ=
   data.tar.gz: !binary |-
-    NWNmMTQxNzFkZWZhOGI2MGNjZjU4YmFmMmU0NzJhOGI4ZTBiNmFkNDBlMTZk
-    Y2NjNjU4NTk5NjI2OWIyY2EyMGIxY2UwZGNlOTYwYzhjZjJlZTY1OWRlNGIy
-    OTNiYTEzMDc4YzUzMzI4YjY1NzQzODgxMTM1MzYyN2U2MTg2MDI=
+    MWJkNjdmZjI1MzNlYjQ3MmJjN2Y4MmRiNDYwMGNiZTYxODBiYTM5ZmIxZWFk
+    YmY5NTA3OWI5NTFjYWQ0N2ViMDZjNzFjZWE5MzFlNjcxNjBmNjgyMGViMTU1
+    ZGYwZjAzZmU5MWE4ZGU3ZGY3OWI5MGQ1NTI1ZmU0MTNhMWJlYTk=

data/lib/pseudo_cleaner/master_cleaner.rb

@@ -4,7 +4,7 @@ module PseudoCleaner
   class MasterCleaner
     @@suite_cleaner          = nil
     @@cleaner_classes        = nil
-    @@redis_classes          = nil
+    # @@redis_classes          = nil
     @@cleaner_classes_sorted = false
     @@report_table           = nil
     @@report_error           = false
@@ -72,10 +72,10 @@ module PseudoCleaner
         end
       end
 
-      def clean_redis(redis)
-        @@redis_classes ||= []
-        @@redis_classes << redis
-      end
+      # def clean_redis(redis)
+      #   @@redis_classes ||= []
+      #   @@redis_classes << redis
+      # end
 
       def database_cleaner
         if Object.const_defined?("ActiveRecord", false) && ActiveRecord.const_defined?("Base", false)
@@ -252,7 +252,7 @@ module PseudoCleaner
 
           PseudoCleaner::MasterCleaner.create_table_cleaners
           PseudoCleaner::MasterCleaner.create_custom_cleaners
-          PseudoCleaner::MasterCleaner.create_redis_cleaners
+          # PseudoCleaner::MasterCleaner.create_redis_cleaners
         end
 
         @@cleaner_classes
@@ -309,13 +309,13 @@ module PseudoCleaner
         end
       end
 
-      def create_redis_cleaners
-        if @@redis_classes
-          @@redis_classes.each do |redis|
-            PseudoCleaner::MasterCleaner.cleaner_classes << [redis, nil, PseudoCleaner::RedisCleaner]
-          end
-        end
-      end
+      # def create_redis_cleaners
+      #   if @@redis_classes
+      #     @@redis_classes.each do |redis|
+      #       PseudoCleaner::MasterCleaner.cleaner_classes << [redis, nil, PseudoCleaner::RedisMonitorCleaner]
+      #     end
+      #   end
+      # end
 
       def find_file_class(seeder_file, seeder_root)
         check_class      = Object

data/lib/pseudo_cleaner/redis_cleaner.rb

@@ -37,13 +37,20 @@ module PseudoCleaner
   # check if somebody else needed to do some work.  Testing with Rails, I find I have to actually sleep, so I do a
   # very short time like 0.01.
   class RedisCleaner
-    # SUITE_KEY = "PseudoDelete::RedisCleaner:initial_redis_state"
-
     FLUSH_COMMANDS =
         [
             "flushall",
             "flushdb"
         ]
+    SET_COMMANDS   =
+        [
+            "sadd",
+            "zadd",
+            "srem",
+            "zrem",
+            "zremrangebyrank",
+            "zremrangebyscore"
+        ]
     WRITE_COMMANDS =
         [
             "append",
@@ -90,7 +97,6 @@ module PseudoCleaner
             "rpoplpush",
             "rpush",
             "rpushx",
-            "sadd",
             "sdiffstore",
             "set",
             "setbit",
@@ -101,14 +107,9 @@ module PseudoCleaner
             "smove",
             "sort",
             "spop",
-            "srem",
             "sunionstore",
-            "zadd",
             "zincrby",
             "zinterstore",
-            "zrem",
-            "zremrangebyrank",
-            "zremrangebyscore",
         ]
     READ_COMMANDS  =
         [
@@ -160,179 +161,168 @@ module PseudoCleaner
             "zscore",
         ]
 
-    attr_reader :monitor_thread
     attr_reader :initial_keys
     attr_accessor :options
 
-    class RedisMessage
-      attr_reader :message
-      attr_reader :time_stamp
-      attr_reader :db
-      attr_reader :host
-      attr_reader :port
-      attr_reader :command
-      attr_reader :cur_pos
-
-      def initialize(message_string)
-        @message = message_string
+    def initialize(start_method, end_method, table, options)
+      @initial_keys       = SortedSet.new
+      @monitor_thread     = nil
+      @redis_name         = nil
+      @suite_altered_keys = SortedSet.new
+      @updated_keys       = SortedSet.new
+      @multi_commands     = []
+      @in_multi           = false
+      @in_redis_cleanup   = false
+      @suspend_tracking   = false
 
-        parse_message
+      unless PseudoCleaner::MasterCleaner::VALID_START_METHODS.include?(start_method)
+        raise "You must specify a valid start function from: #{PseudoCleaner::MasterCleaner::VALID_START_METHODS}."
       end
+      unless PseudoCleaner::MasterCleaner::VALID_END_METHODS.include?(end_method)
+        raise "You must specify a valid end function from: #{PseudoCleaner::MasterCleaner::VALID_END_METHODS}."
+      end
+
+      @options = options
+
+      @options[:table_start_method] ||= start_method
+      @options[:table_end_method]   ||= end_method
+      @options[:output_diagnostics] ||= PseudoCleaner::Configuration.current_instance.output_diagnostics ||
+          PseudoCleaner::Configuration.current_instance.post_transaction_analysis
 
-      def parse_message
-        if @message =~ /[0-9]+\.[0-9]+ \[[0-9]+ [^:]+:[^\]]+\] \"[^\"]+\"/
-          end_pos     = @message.index(" ")
-          @time_stamp = @message[0..end_pos - 1]
+      @redis = table
+    end
 
-          @cur_pos = end_pos + 2 # " ["
-          end_pos  = @message.index(" ", @cur_pos)
-          @db      = @message[@cur_pos..end_pos - 1].to_i
+    # Ruby defines a now deprecated type method so we need to override it here
+    # since it will never hit method_missing
+    def type(key)
+      redis.type(key)
+    end
 
-          @cur_pos = end_pos + 1
-          end_pos  = @message.index(":", @cur_pos)
-          @host    = @message[@cur_pos..end_pos - 1]
+    alias_method :self_respond_to?, :respond_to?
 
-          @cur_pos = end_pos + 1
-          end_pos  = @message.index("]", @cur_pos)
-          @port    = @message[@cur_pos..end_pos - 1].to_i
+    # emulate Ruby 1.9+ and keep respond_to_missing? logic together.
+    def respond_to?(command, include_private=false)
+      super or respond_to_missing?(command, include_private)
+    end
 
-          @cur_pos = end_pos + 2 # "] "
-          @command = next_value.downcase
-        else
-          @command = @message
-        end
-      end
+    def updated_keys
+      @updated_keys ||= SortedSet.new
+    end
 
-      def next_value
-        in_quote = (@message[@cur_pos] == '"')
-        if in_quote
-          @cur_pos += 1
-          end_pos  = @cur_pos
-          while (end_pos && end_pos < @message.length)
-            end_pos = @message.index("\"", end_pos)
+    def method_missing(command, *args, &block)
+      normalized_command = command.to_s.downcase
 
-            num_backslashes = 0
-            back_pos        = end_pos
+      if redis.respond_to?(normalized_command)
+        if normalized_command == "pipelined" ||
+            (normalized_command == "multi" && block)
+          @in_multi = true
+          normalized_command = "exec"
+        end
 
-            while @message[back_pos - 1] == "\\"
-              num_backslashes += 1
-              back_pos        -= 1
-            end
+        response = redis.send(command, *args, &block)
 
-            break if (num_backslashes % 2) == 0
-            end_pos += 1
-          end
+        if @in_multi && !(["exec", "discard"].include?(normalized_command))
+          @multi_commands << [normalized_command, *args]
         else
-          end_pos = @message.index(" ", @cur_pos)
+          process_command(response, normalized_command, *args)
         end
-        the_value = @message[@cur_pos..end_pos - 1]
-        end_pos   += 1 if in_quote
 
-        @cur_pos = end_pos + 1
-
-        the_value.gsub("\\\\", "\\").gsub("\\\"", "\"")
+        response
+      else
+        super
       end
+    end
 
-      def keys
-        unless defined?(@message_keys)
-          @message_keys = []
-
-          if Redis::Namespace::COMMANDS.include? command
-            handling = Redis::Namespace::COMMANDS[command.to_s.downcase]
+    def process_command(response, *args)
+      unless @in_redis_cleanup || @suspend_tracking
+        if "multi" == args[0]
+          @in_multi       = true
+          @multi_commands = []
+        elsif ["exec", "pipelined"].include?(args[0])
+          begin
+            raise "exec response does not match sent commands" unless response.length == @multi_commands.length
+
+            response.each_with_index do |command_response, index|
+              process_command(command_response, *(@multi_commands[index]))
+            end
+          ensure
+            @in_multi       = false
+            @multi_commands = []
+          end
+        elsif "discard" == args[0]
+          @in_multi       = false
+          @multi_commands = []
+        elsif WRITE_COMMANDS.include?(args[0])
+          updated_keys.merge(extract_keys(*args))
+        elsif SET_COMMANDS.include?(args[0])
+          if [true, false].include?(response) ? response : (response > 0)
+            updated_keys.merge(extract_keys(*args))
+          end
+        end
+      end
+    end
 
-            (before, after) = handling
+    def respond_to_missing?(command, include_all=false)
+      return true if WRITE_COMMANDS.include?(command.to_s.downcase)
 
-            case before
-              when :first
-                @message_keys << next_value
+      # blind passthrough is deprecated and will be removed in 2.0
+      if redis.respond_to?(command, include_all)
+        return true
+      end
 
-              when :all
-                while @cur_pos < @message.length
-                  @message_keys << next_value
-                end
+      defined?(super) && super
+    end
 
-              when :exclude_first
-                next_value
-                while @cur_pos < @message.length
-                  @message_keys << next_value
-                end
+    def extract_keys(command, *args)
+      handling     = Redis::Namespace::COMMANDS[command.to_s.downcase]
+      message_keys = []
 
-              when :exclude_last
-                while @cur_pos < @message.length
-                  @message_keys << next_value
-                end
-                @message_keys.delete_at(@message_keys.length - 1)
-
-              when :exclude_options
-                options = ["weights", "aggregate", "sum", "min", "max"]
-                while @cur_pos < @message.length
-                  @message_keys << next_value
-                  if options.include?(@message_keys[-1].downcase)
-                    @message_keys.delete_at(@message_keys.length - 1)
-                    break
-                  end
-                end
+      (before, after) = handling
 
-              when :alternate
-                while @cur_pos < @message.length
-                  @message_keys << next_value
-                  next_value
-                end
+      case before
+        when :first
+          message_keys << args[0] if args[0]
 
-              when :sort
-                next_value
+        when :all
+          args.each do |arg|
+            message_keys << arg
+          end
 
-                while @cur_pos < @message.length
-                  a_value = next_value
-                  if a_value.downcase == "store"
-                    @message_keys[0] = next_value
-                  end
-                end
+        when :exclude_first
+          args.each do |arg|
+            message_keys << arg
+          end
+          message_keys.shift
 
-              # when :eval_style
-              #
-              # when :scan_style
-            end
+        when :exclude_last
+          args.each do |arg|
+            message_keys << arg
           end
-        end
+          message_keys.pop unless message_keys.length == 1
 
-        @message_keys
-      end
+        when :exclude_options
+          args.each do |arg|
+            message_keys << arg unless arg.is_a?(Hash)
+          end
 
-      def to_s
-        {
-            time_stamp: time_stamp,
-            db:         db,
-            host:       host,
-            port:       port,
-            command:    command,
-            message:    message,
-            cur_pos:    cur_pos
-        }.to_s
-      end
-    end
+        when :alternate
+          args.each_with_index do |arg, i|
+            message_keys << arg if i.even?
+          end
 
-    def initialize(start_method, end_method, table, options)
-      @initial_keys       = SortedSet.new
-      @monitor_thread     = nil
-      @redis_name         = nil
-      @suite_altered_keys = SortedSet.new
+        when :sort
+          if args[-1].is_a?(Hash)
+            if args[1][:store]
+              message_keys << args[1][:store]
+            end
+          end
 
-      unless PseudoCleaner::MasterCleaner::VALID_START_METHODS.include?(start_method)
-        raise "You must specify a valid start function from: #{PseudoCleaner::MasterCleaner::VALID_START_METHODS}."
-      end
-      unless PseudoCleaner::MasterCleaner::VALID_END_METHODS.include?(end_method)
-        raise "You must specify a valid end function from: #{PseudoCleaner::MasterCleaner::VALID_END_METHODS}."
+        # when :eval_style
+        #
+        # when :scan_style
       end
 
-      @options = options
-
-      @options[:table_start_method] ||= start_method
-      @options[:table_end_method]   ||= end_method
-      @options[:output_diagnostics] ||= PseudoCleaner::Configuration.current_instance.output_diagnostics ||
-          PseudoCleaner::Configuration.current_instance.post_transaction_analysis
-
-      @redis = table
+      message_keys
     end
 
     def <=>(right_object)
@@ -359,15 +349,19 @@ module PseudoCleaner
     def suite_start test_strategy
       @test_strategy ||= test_strategy
 
-      # if redis.type(PseudoCleaner::RedisCleaner::SUITE_KEY) == "set"
-      #   @initial_keys = SortedSet.new(redis.smembers(PseudoCleaner::RedisCleaner::SUITE_KEY))
-      #   report_end_of_suite_state "before suite start"
-      # end
-      # redis.del PseudoCleaner::RedisCleaner::SUITE_KEY
-
       start_monitor
     end
 
+    def suspend_tracking(&block)
+      begin
+        @suspend_tracking = true
+
+        block.yield
+      ensure
+        @suspend_tracking = false
+      end
+    end
+
     def test_start test_strategy
       @test_strategy ||= test_strategy
 
@@ -407,21 +401,12 @@ module PseudoCleaner
 
     def suite_end test_strategy
       report_end_of_suite_state "suite end"
-
-      if monitor_thread
-        monitor_thread.kill
-        @monitor_thread = nil
-      end
     end
 
     def reset_suite
       report_end_of_suite_state "reset suite"
 
-      if monitor_thread
-        monitor_thread.kill
-        @monitor_thread = nil
-        start_monitor
-      end
+      start_monitor
     end
 
     def ignore_regexes
@@ -485,14 +470,6 @@ module PseudoCleaner
       end
     end
 
-    def synchronize_key
-      @synchronize_key ||= "redis_cleaner::synchronization_key_#{rand(1..1_000_000_000_000_000_000)}_#{rand(1..1_000_000_000_000_000_000)}"
-    end
-
-    def synchronize_end_key
-      @synchronize_end_key ||= "redis_cleaner::synchronization_end_key_#{rand(1..1_000_000_000_000_000_000)}_#{rand(1..1_000_000_000_000_000_000)}"
-    end
-
     def report_end_of_suite_state report_reason
       current_keys = SortedSet.new(redis.keys)
 
@@ -514,30 +491,24 @@ module PseudoCleaner
     end
 
     def synchronize_test_values(&block)
-      updated_values = nil
+      updated_values = updated_keys.dup
 
-      if monitor_thread
-        redis.setex(synchronize_key, 1, true)
-        updated_values = queue.pop
-      end
-
-      block.yield updated_values
+      @in_redis_cleanup = true
 
-      redis.setex(synchronize_end_key, 1, true)
-    end
+      begin
+        block.yield updated_values
+      ensure
+        @in_redis_cleanup = false
+      end
 
-    def queue
-      @queue ||= Queue.new
+      @updated_keys = SortedSet.new
     end
 
     def start_monitor
       cleaner_class = self
 
       @initial_keys = SortedSet.new(redis.keys)
-      # @initial_keys.add(PseudoCleaner::RedisCleaner::SUITE_KEY)
-      # @initial_keys.each do |key_value|
-      #   redis.sadd(PseudoCleaner::RedisCleaner::SUITE_KEY, key_value)
-      # end
+
       if @options[:output_diagnostics]
         if PseudoCleaner::MasterCleaner.report_table
           Cornucopia::Util::ReportTable.new(nested_table:         PseudoCleaner::MasterCleaner.report_table,
@@ -550,60 +521,6 @@ module PseudoCleaner
           PseudoCleaner::Logger.write("    Initial keys count - #{@initial_keys.count}")
         end
       end
-
-      unless @monitor_thread
-        @monitor_thread = Thread.new do
-          in_redis_cleanup = false
-          updated_keys     = SortedSet.new
-
-          monitor_redis    = Redis.new(cleaner_class.redis.client.options)
-          redis_options    = monitor_redis.client.options.with_indifferent_access
-          cleaner_class_db = redis_options[:db]
-
-          monitor_redis.monitor do |message|
-            redis_message = RedisMessage.new message
-
-            if redis_message.db == cleaner_class_db
-              process_command = true
-
-              if redis_message.command == "setex"
-                if redis_message.keys[0] == cleaner_class.synchronize_key
-                  process_command = false
-
-                  in_redis_cleanup = true
-                  return_values    = updated_keys
-                  updated_keys     = SortedSet.new
-                  cleaner_class.queue << return_values
-                elsif redis_message.keys[0] == cleaner_class.synchronize_end_key
-                  in_redis_cleanup                       = false
-                  cleaner_class.monitor_thread[:updated] = nil
-                  process_command                        = false
-                end
-              elsif redis_message.command == "del"
-                if in_redis_cleanup
-                  process_command = false
-                end
-              end
-
-              if process_command
-                # flush...
-                if PseudoCleaner::RedisCleaner::WRITE_COMMANDS.include? redis_message.command
-                  updated_keys.merge(redis_message.keys)
-                elsif PseudoCleaner::RedisCleaner::FLUSH_COMMANDS.include? redis_message.command
-                  # Not sure I can get the keys at this point...
-                  # updated_keys.merge(cleaner_class.redis.keys)
-                end
-              end
-            elsif "flushall" == redis_message.command
-              # Not sure I can get the keys at this point...
-              # updated_keys.merge(cleaner_class.redis.keys)
-            end
-          end
-        end
-
-        sleep(0.01)
-        redis.get(synchronize_key)
-      end
     end
 
     def report_record(key_name)