redis-bitops 0.2

data/lib/redis/bitops/queries/lazy_evaluation.rb

@@ -0,0 +1,47 @@
+require 'forwardable'
+
+class Redis
+  module Bitops
+    module Queries
+    
+      # Support for materializing expressions when one of the supported bitmap methods is called.
+      #
+      # Example
+      #
+      #   a = $redis.sparse_bitmap("a")
+      #   b = $redis.sparse_bitmap("b")
+      #   a[0] = true
+      #   result = a | b
+      #   puts result[0] => true
+      #
+      module LazyEvaluation
+        extend Forwardable
+      
+        def_delegators :dest, :bitcount, :[], :[]=, :<<, :delete!, :root_key
+      
+        def dest
+          if @dest.nil?
+            @dest = temp_bitmap
+            do_evaluate(@dest)
+          end
+          @dest
+        end
+
+        # Optimizes the expression and materializes it into bitmap 'dest'.
+        #
+        def evaluate(dest_bitmap)
+          if @dest
+            @dest.copy_to(dest_bitmap)
+          else
+            do_evaluate(dest_bitmap)
+          end
+        end
+      
+        protected def do_evaluate(dest_bitmap)
+          optimize!
+          materialize(dest_bitmap)
+        end
+      end
+    end
+  end
+end

data/lib/redis/bitops/queries/materialization_helpers.rb

@@ -0,0 +1,53 @@
+class Redis
+  module Bitops
+    module Queries
+    
+      # Helpers for materialization, i.e. running a BITOP command or, possibly, another command
+      # and saving its results to a Redis database in 'intermediate'.
+      #
+      module MaterializationHelpers
+      
+        # Materializes the operand 'o' saving the result in 'redis'.
+        # If the operand can be materialized, it does so storing the result in 'intermediate'
+        # unless the latter is nil. In that case, a temp intermediate bitmap is created to hold 
+        # the result (and the bitmap is added to 'temp_intermediates').
+        #
+        def resolve_operand(o, intermediate, temp_intermediates)
+          if o.respond_to?(:materialize)
+            if intermediate.nil?
+              new_intermediate = temp_bitmap
+              temp_intermediates << new_intermediate
+            end
+            intermediate ||= new_intermediate
+            o.materialize(intermediate)
+            [intermediate, nil]
+          else
+            [o, intermediate]
+          end
+        end
+      
+        # Creates a temp bitmap.
+        #
+        def temp_bitmap
+          bitmap = bitmap_factory.call(unique_key)
+          bitmap
+        end
+      
+        # Generates a random unique key. 
+        #
+        # TODO: The key _should_ be unique and not repeat in the 
+        # database but this isn't guaranteed. Considering the intended usage though
+        # (creation of temporary intermediate bitmaps while materializing
+        # queries), it should be sufficient.
+        #
+        def unique_key
+          "redis:bitops:#{SecureRandom.hex(20)}"
+        end
+      
+        def bitmap_factory
+          raise "Override in the class using the module to return the bitmap factory."
+        end
+      end
+    end
+  end
+end

data/lib/redis/bitops/queries/tree_building_helpers.rb

@@ -0,0 +1,46 @@
+class Redis
+  module Bitops
+    module Queries
+      # Helpers for expression tree building.
+      #
+      # Add new logical operators here as/if they become supported by Redis' BITOP command.
+      #
+      module TreeBuildingHelpers
+      
+        # Logical AND operator.
+        #
+        def & (rhs)
+          BinaryOperator.new(:and, self, rhs)
+        end
+      
+        # Logical OR operator.
+        #
+        def | (rhs)
+          BinaryOperator.new(:or, self, rhs)
+        end
+      
+        # Logical XOR operator.
+        #
+        def ^ (rhs)
+          BinaryOperator.new(:xor, self, rhs)
+        end
+      
+        # Logical NOT operator.
+        #
+        # IMPORTANT: It inverts bits padding with zeroes till the nearest byte boundary.
+        # It means that setting the left-most bit to 1 and inverting will result in 01111111 not 0.
+        #
+        # Corresponding Redis commands:
+        #
+        # SETBIT "a" 0 1
+        # BITOP NOT "b" "a"
+        # BITCOUNT "b"
+        # => (integer) 7
+        #
+        def ~
+          UnaryOperator.new(:not, self)
+        end
+      end
+    end
+  end
+end

data/lib/redis/bitops/queries/unary_operator.rb

@@ -0,0 +1,48 @@
+class Redis
+  module Bitops
+    module Queries
+
+      # A unary bitwise operator. Currently only NOT is supported by redis.
+      #
+      class UnaryOperator
+        include MaterializationHelpers
+        include TreeBuildingHelpers
+        include LazyEvaluation
+        
+        # Create a new bitwise operator 'op' with one argument 'arg'.
+        #
+        def initialize(op, arg)
+          @op = op
+          @arg = arg
+        end
+      
+        # Runs the expression tree against the redis database, saving the results
+        # in bitmap 'dest'.
+        #
+        def materialize(dest)
+          temp_intermediates = []
+          result, = resolve_operand(@arg, dest, temp_intermediates)
+          result.bitop(@op, dest)
+        ensure
+          temp_intermediates.each(&:delete!)
+        end
+      
+        # Optimizes the expression tree by combining operands for neighboring identical operators.
+        # Because NOT operator in Redis can only accept one operand, no optimization is made
+        # for the operand but the children are optimized recursively.
+        #
+        def optimize!(parent_op = nil)
+          @arg.optimize!(@op) if @arg.respond_to?(:optimize!)
+          self
+        end
+      
+        # Finds the first bitmap factory in the expression tree.
+        # Required by LazyEvaluation and MaterializationHelpers.
+        #
+        def bitmap_factory
+          @arg.bitmap_factory or raise "Internal error. Cannot get redis connection."
+        end
+      end
+    end
+  end
+end

data/lib/redis/bitops/sparse_bitmap.rb

@@ -0,0 +1,125 @@
+require 'set'
+
+class Redis
+  module Bitops
+    
+    # A sparse bitmap using multiple key to store its data to save memory.
+    #
+    # Note: When adding new public methods, revise the LazyEvaluation module.
+    #
+    class SparseBitmap < Bitmap
+    
+      # Creates a new sparse bitmap stored in 'redis' under 'root_key'.
+      #
+      def initialize(root_key, redis, bytes_per_chunk = nil)
+        @bytes_per_chunk = bytes_per_chunk || Redis::Bitops.configuration.default_bytes_per_chunk
+        super(root_key, redis)
+      end
+    
+      # Returns the number of set bits.
+      #
+      def bitcount
+        chunk_keys.map { |key| @redis.bitcount(key) }.reduce(:+) || 0
+      end
+     
+      # Deletes the bitmap and all its keys.
+      #
+      def delete!
+        chunk_keys.each do |key|
+          @redis.del(key)
+        end
+        super
+      end
+     
+      # Redis BITOP operator 'op' (one of :and, :or, :xor or :not) on operands
+      # (bitmaps). The result is stored in 'result'.
+      #
+      def bitop(op, *operands, result)
+        # TODO: Optimization is possible for AND. We can use an intersection of each operand
+        # chunk numbers to minimize the number of database accesses.
+      
+        all_keys = self.chunk_keys + (operands.map(&:chunk_keys).flatten! || [])
+        unique_chunk_numbers = Set.new(chunk_numbers(all_keys))
+      
+        maybe_multi(level: :bitmap, watch: all_keys) do
+          unique_chunk_numbers.each do |i|
+            @redis.bitop(op, result.chunk_key(i), self.chunk_key(i), *operands.map { |o| o.chunk_key(i) })
+          end
+        end
+        result
+      end
+
+      def chunk_keys
+        @redis.keys("#{@root_key}:chunk:*")
+      end
+    
+      def chunk_key(i)
+        "#{@root_key}:chunk:#{i}"
+      end
+
+      # Returns lambda creating SparseBitmap objects using @redis as the connection.
+      #
+      def bitmap_factory
+        lambda { |key| @redis.sparse_bitmap(key, @bytes_per_chunk) }
+      end
+    
+      # Copy this bitmap to 'dest' bitmap.
+      #
+      def copy_to(dest)
+      
+        # Copies all source chunks to destination chunks and deletes remaining destination chunk keys.
+      
+        source_keys = self.chunk_keys
+        dest_keys = dest.chunk_keys
+
+        maybe_multi(level: :bitmap, watch: source_keys + dest_keys) do
+          source_chunks = Set.new(chunk_numbers(source_keys))
+          source_chunks.each do |i|
+            copy(chunk_key(i), dest.chunk_key(i))
+          end
+          dest_chunks = Set.new(chunk_numbers(dest_keys))
+          (dest_chunks - source_chunks).each do |i|
+            @redis.del(dest.chunk_key(i))
+          end
+        end
+      end
+
+      protected
+    
+      def bits_per_chunk
+        @bytes_per_chunk * 8
+      end
+    
+      def key(pos)
+        chunk_key(chunk_number(pos))
+      end
+   
+      def offset(pos)
+        pos.modulo bits_per_chunk
+      end
+    
+      def chunk_number(pos) 
+        (pos / bits_per_chunk).to_i
+      end
+
+      def chunk_numbers(keys)
+        keys.map { |key| key.split(":").last.to_i }
+      end
+    
+      # Maybe pipeline/make atomic based on the configuration.
+      #
+      def maybe_multi(options = {}, &block)
+        current_level = options[:level] or raise "Specify the current transaction level."
+      
+        if Redis::Bitops.configuration.transaction_level == current_level
+          watched_keys = options[:watch]
+          @redis.watch(watched_keys) if watched_keys
+          @redis.multi(&block)
+        else
+          block.call
+        end
+      end
+    
+    end
+  end
+end

data/spec/redis/bitops/bitmap_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe Redis::Bitops::Bitmap, redis_cleanup: true, redis_key_prefix: "rsb:" do
+  it_should_behave_like "a bitmap", :bitmap
+end
+
+describe "Redis#bitmap" do
+  it_should_behave_like "a bitmap factory method", :bitmap, Redis::Bitops::Bitmap
+end

data/spec/redis/bitops/queries/binary_operator_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe Redis::Bitops::Queries::BinaryOperator do
+  let(:a) { double("a") }
+  let(:b) { double("b") }
+  let(:c) { double("c") }
+  let(:d) { double("d") }
+  let(:e) { double("e") }
+  
+  let(:redis) { double("redis", del: nil) }
+  let(:result) { double("output", redis: redis) }
+
+  it "optimizes the expression tree" do
+    a.should_receive(:bitop).with(:and, result, d, e, result)
+    b.should_receive(:bitop).with(:or, c, result)
+    expr = Redis::Bitops::Queries::BinaryOperator.new(:and, 
+      a, 
+      Redis::Bitops::Queries::BinaryOperator.new(:and, 
+        Redis::Bitops::Queries::BinaryOperator.new(:or, b, c), 
+        Redis::Bitops::Queries::BinaryOperator.new(:and, d, e)))
+    expr.optimize!
+    expr.materialize(result)
+  end
+end

data/spec/redis/bitops/queries/unary_operator_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe Redis::Bitops::Queries::UnaryOperator do
+  let(:a) { double("a") }
+  let(:b) { double("b") }
+  let(:c) { double("c") }
+  let(:d) { double("d") }
+  let(:e) { double("e") }
+  
+  let(:redis) { double("redis", del: nil) }
+  let(:result) { double("result", redis: redis) }
+
+  it "optimizes the expression tree" do
+    a.should_receive(:bitop).with(:and, result, d, e, result)
+    b.should_receive(:bitop).with(:or, c, result)
+    result.should_receive(:bitop).with(:not, result)
+    expr = 
+      Redis::Bitops::Queries::UnaryOperator.new(:not, 
+        Redis::Bitops::Queries::BinaryOperator.new(:and, 
+          a, 
+          Redis::Bitops::Queries::BinaryOperator.new(:and, 
+            Redis::Bitops::Queries::BinaryOperator.new(:or, b, c), 
+            Redis::Bitops::Queries::BinaryOperator.new(:and, d, e))))
+    expr.optimize!
+    expr.materialize(result)
+  end
+end

data/spec/redis/bitops/sparse_bitmap_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+
+describe Redis::Bitops::SparseBitmap, redis_cleanup: true, redis_key_prefix: "rsb:" do
+  it_should_behave_like "a bitmap", :sparse_bitmap
+
+  let(:redis) { Redis.new }
+  let(:bytes_per_chunk) { 4 }
+  let(:bits_per_chunk) { bytes_per_chunk * 8 }
+  let(:a) { redis.sparse_bitmap("rsb:a", bytes_per_chunk) }
+  let(:b) { redis.sparse_bitmap("rsb:b", bytes_per_chunk) }
+  let(:c) { redis.sparse_bitmap("rsb:c", bytes_per_chunk) }
+  let(:result) { redis.sparse_bitmap("rsb:output", bytes_per_chunk) }
+
+  describe "edge cases" do
+    before do
+      a[0] = true
+      a[bits_per_chunk - 1] = true
+      a[bits_per_chunk] = true
+      a[2 * bits_per_chunk - 1] = true
+      a[2 * bits_per_chunk] = true
+      
+      b[0] = true
+      b[bits_per_chunk] = true
+      b[2 * bits_per_chunk - 1] = true
+    end
+
+    describe "#[]" do
+      it "handles bits arround chunk boundaries" do
+        a.chunk_keys.should have(3).items
+        set_bits = 
+          (0..(3 * bits_per_chunk)).inject([]) { |acc, i|
+            acc << i if a[i]
+            acc
+          }
+        set_bits.should match_array([
+          0, 
+          bits_per_chunk - 1, 
+          bits_per_chunk, 
+          2 * bits_per_chunk - 1,
+          2 * bits_per_chunk])
+      end
+    end
+  
+    describe "#bitcount" do
+      it "handles bits around chunk boundaries" do
+        a.bitcount.should == 5
+      end
+    end
+  
+    describe "#operator |" do 
+      it "handles bits around chunk boundaries" do
+        result << (a | b)
+        result.bitcount.should == 5
+      end
+    end
+  
+    describe "#operator &" do 
+      it "handles bits around chunk boundaries" do
+        result << (a & b)
+        result.bitcount.should == 3
+      end
+    end
+  
+    describe "#operator ~" do
+      it "handles bits around chunk boundaries" do
+        result << (~(a & b))
+        result[0].should be_false
+        result[1].should be_true
+        result[bits_per_chunk - 1].should be_true
+        result[bits_per_chunk].should be_false
+        result[bits_per_chunk + 1].should be_true
+        result[2 * bits_per_chunk - 2].should be_true
+        result[2 * bits_per_chunk - 1].should be_false
+        result[2 * bits_per_chunk].should be_true
+      end
+    end
+  
+    describe "#operator ^" do
+      it "handles bits around chunk boundaries" do
+        result << (a ^ b)
+        result.bitcount.should == 2
+      end
+    end
+
+    describe "#copy_to" do
+      it "overrides all chunks in the target bitmap" do
+        # Fix expression with bits set using [] after evaluation doesn't materialize the newly set bits.
+        result[4*bits_per_chunk + 1] = true
+        a.copy_to(result)
+        result.bitcount.should == a.bitcount
+        result[4*bits_per_chunk + 1].should be_false
+      end
+    end
+  end
+end
+
+describe "Redis#sparse_bitmap" do
+  it_should_behave_like "a bitmap factory method", :sparse_bitmap, Redis::Bitops::SparseBitmap
+end