cassandra 0.12.1 → 0.13.0

data/lib/cassandra/composite.rb

@@ -0,0 +1,118 @@
+
+class Cassandra
+  class Composite
+    include ::Comparable
+    attr_reader :parts
+    attr_reader :column_slice
+
+    def initialize(*parts)
+      options = {}
+      if parts.last.is_a?(Hash)
+        options = parts.pop
+      end
+      @column_slice = options[:slice]
+      raise ArgumentError if @column_slice != nil && ![:before, :after].include?(@column_slice)
+
+      if parts.length == 1 && parts[0].instance_of?(self.class)
+        @column_slice = parts[0].column_slice
+        @parts = parts[0].parts
+      elsif parts.length == 1 && parts[0].instance_of?(String) && @column_slice.nil? && valid_packed_composite?(parts[0])
+        unpack(parts[0])
+      else
+        @parts = parts
+      end
+    end
+
+    def [](*args)
+      return @parts[*args]
+    end
+
+    def pack
+      packed = @parts.map do |part|
+        [part.length].pack('n') + part + "\x00"
+      end
+      if @column_slice
+        part = @parts[-1]
+        packed[-1] = [part.length].pack('n') + part + slice_end_of_component
+      end
+      return packed.join('')
+    end
+
+    def to_s
+      return pack
+    end
+
+    def <=>(other)
+      if !other.instance_of?(self.class)
+        return @parts.first <=> other
+      end
+      eoc = slice_end_of_component.unpack('c')[0]
+      other_eoc = other.slice_end_of_component.unpack('c')[0]
+      @parts.zip(other.parts).each do |a, b|
+        next if a == b
+        if a.nil? && b.nil?
+          return eoc <=> other_eoc
+        end
+
+        if a.nil?
+          return @column_slice == :after ? 1 : -1
+        end
+        if b.nil?
+          return other.column_slice == :after ? -1 : 1
+        end
+        return -1 if a < b
+        return 1 if a > b
+      end
+      return 0
+    end
+
+    def inspect
+      return "#<Composite:#{@column_slice} #{@parts.inspect}>"
+    end
+
+    def slice_end_of_component
+      return "\x01" if @column_slice == :after
+      return "\xFF" if @column_slice == :before
+      return "\x00"
+    end
+
+    private
+    def unpack(packed_string)
+      parts = []
+      end_of_component = nil
+      while packed_string.length > 0
+        length = packed_string.slice(0, 2).unpack('n')[0]
+        parts << packed_string.slice(2, length)
+        end_of_component = packed_string.slice(2 + length, 1)
+
+        packed_string = packed_string.slice(3 + length, packed_string.length)
+      end
+      @column_slice = :after if end_of_component == "\x01"
+      @column_slice = :before if end_of_component == "\xFF"
+      @parts = parts
+    end
+
+    def valid_packed_composite?(packed_string)
+      while packed_string.length > 0
+        length = packed_string.slice(0, 2).unpack('n')[0]
+        return false if length.nil? || length + 3 > packed_string.length
+
+        end_of_component = packed_string.slice(2 + length, 1)
+        if length + 3 != packed_string.length
+          return false if end_of_component != "\x00"
+        end
+
+        packed_string = packed_string.slice(3 + length, packed_string.length)
+      end
+      return true
+    end
+
+    def hash
+      return to_s.hash
+    end
+
+    def eql?(other)
+      return to_s == other.to_s
+    end
+  end
+end

data/lib/cassandra/mock.rb

@@ -393,11 +393,16 @@ class Cassandra
         if (start_key.nil? || key >= start_key) && (finish_key.nil? || key <= finish_key)
           if columns
             #ret[key] = columns.inject(OrderedHash.new){|hash, column_name| hash[column_name] = cf(column_family)[key][column_name]; hash;}
-            ret[key] = columns_to_hash(column_family, cf(column_family)[key].select{|k,v| columns.include?(k)})
+            selected_hash = OrderedHash.new
+            cf(column_family)[key].each do |k, v|
+              selected_hash.[]=(k, v, cf(column_family)[key].timestamps[k]) if columns.include?(k)
+            end
+            ret[key] = columns_to_hash(column_family, selected_hash)
             ret[key] = apply_count(ret[key], count, reversed)
             blk.call(key,ret[key]) unless blk.nil?
           else
             #ret[key] = apply_range(cf(column_family)[key], column_family, start, finish, !is_super(column_family))
+            start, finish = finish, start if reversed
             ret[key] = apply_range(columns_to_hash(column_family, cf(column_family)[key]), column_family, start, finish)
             ret[key] = apply_count(ret[key], count, reversed)
             blk.call(key,ret[key]) unless blk.nil?
@@ -446,7 +451,12 @@ class Cassandra
 
       new_stuff = new_stuff.to_a.inject({}){|h,k| h[k[0].to_s] = k[1]; h }
 
-      OrderedHash[old_stuff.merge(new_stuff).sort{|a,b| a[0] <=> b[0]}]
+      new_stuff.each { |k,v| old_stuff.[]=(k, v, (Time.now.to_f * 1000000).to_i) }
+      hash = OrderedHash.new
+      old_stuff.sort{ |a,b| a[0] <=> b[0] }.each do |k, v|
+        hash.[]=(k, v, old_stuff.timestamps[k])
+      end
+      hash
     end
 
     def columns_to_hash(column_family, columns)
@@ -454,15 +464,17 @@ class Cassandra
       output = OrderedHash.new
 
       columns.each do |column_name, value|
+        timestamp = columns.timestamps[column_name]
         column = column_class.new(column_name)
 
         if [Hash, OrderedHash].include?(value.class)
           output[column] ||= OrderedHash.new
           value.each do |sub_column, sub_column_value|
-            output[column][sub_column_class.new(sub_column)] = sub_column_value
+            timestamp = value.timestamps[sub_column]
+            output[column].[]=(sub_column_class.new(sub_column), sub_column_value, timestamp)
           end
         else
-          output[column_class.new(column_name)] = value
+          output.[]=(column_class.new(column_name), value, timestamp)
         end
       end
 
@@ -475,7 +487,7 @@ class Cassandra
         keys = keys.reverse if reversed
         keys = keys[0...count]
         keys.inject(OrderedHash.new) do |memo, key|
-          memo[key] = row[key]
+          memo.[]=(key, row[key], row.timestamps[key])
           memo
         end
       else
@@ -489,7 +501,7 @@ class Cassandra
       ret = OrderedHash.new
       row.keys.each do |key|
         if (start.nil? || key >= start) && (finish.nil? || key <= finish)
-          ret[key] = row[key]
+          ret.[]=(key, row[key], row.timestamps[key])
         end
       end
       ret

data/test/cassandra_mock_test.rb

@@ -22,6 +22,12 @@ class CassandraMockTest < CassandraTest
 
     @uuids = (0..6).map {|i| SimpleUUID::UUID.new(Time.at(2**(24+i))) }
     @longs = (0..6).map {|i| Long.new(Time.at(2**(24+i))) }
+    @composites = [
+      Cassandra::Composite.new([5].pack('N'), "zebra"),
+      Cassandra::Composite.new([5].pack('N'), "aardvark"),
+      Cassandra::Composite.new([1].pack('N'), "elephant"),
+      Cassandra::Composite.new([10].pack('N'), "kangaroo"),
+    ]
   end
 
   def test_setup
@@ -57,6 +63,24 @@ class CassandraMockTest < CassandraTest
     end
   end
   
+  def test_get_range_reversed_slice
+    data = 4.times.map { |i| ["body-#{i.to_s}", "v"] }
+    hash = Cassandra::OrderedHash[data]
+    sliced_hash = Cassandra::OrderedHash[data.reverse[1..-1]]
+    
+    @twitter.insert(:Statuses, "all-keys", hash)
+    
+    columns = @twitter.get_range(
+      :Statuses,
+      :start => sliced_hash.keys.first,
+      :reversed => true
+    )["all-keys"]
+    
+    columns.each do |column|
+      assert_equal sliced_hash.shift, column
+    end
+  end
+  
   def test_get_range_count
     data = 3.times.map { |i| ["body-#{i.to_s}", "v"] }
     hash = Cassandra::OrderedHash[data]
@@ -76,4 +100,23 @@ class CassandraMockTest < CassandraTest
       @twitter.insert(:UserRelationships, 'a', ['u1','u2'])
     }
   end
+  
+  def test_column_timestamps
+    base_time = Time.now
+    @twitter.insert(:Statuses, "time-key", { "body" => "value" })
+
+    results = @twitter.get(:Statuses, "time-key")
+    assert(results.timestamps["body"] / 1000000 >= base_time.to_i)
+  end
+  
+  def test_supercolumn_timestamps
+    base_time = Time.now
+    @twitter.insert(:StatusRelationships, "time-key", { "super" => { @uuids[1] => "value" }})
+
+    results = @twitter.get(:StatusRelationships, "time-key")
+    assert_nil(results.timestamps["super"])
+    
+    columns = results["super"]
+    assert(columns.timestamps[@uuids[1]] / 1000000 >= base_time.to_i)
+  end
 end

data/test/cassandra_test.rb

@@ -21,6 +21,12 @@ class CassandraTest < Test::Unit::TestCase
 
     @uuids = (0..6).map {|i| SimpleUUID::UUID.new(Time.at(2**(24+i))) }
     @longs = (0..6).map {|i| Long.new(Time.at(2**(24+i))) }
+    @composites = [
+      Cassandra::Composite.new([5].pack('N'), "zebra"),
+      Cassandra::Composite.new([5].pack('N'), "aardvark"),
+      Cassandra::Composite.new([1].pack('N'), "elephant"),
+      Cassandra::Composite.new([10].pack('N'), "kangaroo"),
+    ]
   end
 
   def test_inspect
@@ -43,7 +49,7 @@ class CassandraTest < Test::Unit::TestCase
   end
 
   def test_get_key
-    
+
     @twitter.insert(:Users, key, {'body' => 'v', 'user' => 'v'})
     assert_equal({'body' => 'v', 'user' => 'v'}, @twitter.get(:Users, key))
     assert_equal(['body', 'user'].sort, @twitter.get(:Users, key).timestamps.keys.sort)
@@ -280,35 +286,46 @@ class CassandraTest < Test::Unit::TestCase
 
   def test_get_range_block
     k = key
+
+    values = {}
     5.times do |i|
-      @twitter.insert(:Statuses, k+i.to_s, {"body-#{i.to_s}" => 'v'})
+      values[k+i.to_s] = {"body-#{i.to_s}" => 'v'}
     end
 
-    values = (0..4).collect{|n| { :key => "test_get_range_block#{n}", :columns => { "body-#{n}" => "v" }} }.reverse
+    values.each {|key, columns| @twitter.insert(:Statuses, key, columns) }
 
     returned_value = @twitter.get_range(:Statuses, :start_key => k.to_s, :key_count => 5) do |key,columns|
-       expected = values.pop
-       assert_equal expected[:key], key
-       assert_equal expected[:columns], columns
+       expected = values.delete(key)
+       assert_equal expected, columns
     end
 
-    assert_equal [], values
+    assert values.length < 5
     assert_nil returned_value
   end
-  
+
   def test_get_range_reversed
     data = 3.times.map { |i| ["body-#{i.to_s}", "v"] }
     hash = Cassandra::OrderedHash[data]
     reversed_hash = Cassandra::OrderedHash[data.reverse]
-    
+
     @twitter.insert(:Statuses, "all-keys", hash)
-    
+
     columns = @twitter.get_range(:Statuses, :reversed => true)["all-keys"]
     columns.each do |column|
       assert_equal reversed_hash.shift, column
     end
   end
 
+  def test_get_range_with_start_key_and_key_count
+    hash = {"name" => "value"}
+    @twitter.insert(:Statuses, "a-key", hash)
+    @twitter.insert(:Statuses, "b-key", hash)
+    @twitter.insert(:Statuses, "c-key", hash)
+
+    results = @twitter.get_range(:Statuses, :start_key => "b-key", :key_count => 1)
+    assert_equal ["b-key"], results.keys
+  end
+
   def test_each_key
     k = key
     keys_yielded = []
@@ -490,12 +507,12 @@ class CassandraTest < Test::Unit::TestCase
 
   def test_count_columns
     columns = (1..200).inject(Hash.new){|h,v| h['column' + v.to_s] = v.to_s; h;}
-    
+
     @twitter.insert(:Statuses, key, columns)
     assert_equal 200, @twitter.count_columns(:Statuses, key, :count => 200)
-    assert_equal 100, @twitter.count_columns(:Statuses, key)    
+    assert_equal 100, @twitter.count_columns(:Statuses, key)
     assert_equal 55, @twitter.count_columns(:Statuses, key, :count => 55)
-    
+
   end
 
   def test_count_super_columns
@@ -539,40 +556,43 @@ class CassandraTest < Test::Unit::TestCase
       @twitter.insert(:Users, k + '3', {'body' => 'bogus', 'user' => 'v3'})
       @twitter.insert(:Users, k + '3', {'body' => 'v3', 'location' => 'v3'})
       @twitter.insert(:Statuses, k + '3', {'body' => 'v'})
+      @twitter.add(:UserCounters, 'bob', 5, 'tweet_count') if CASSANDRA_VERSION.to_f >= 0.8
 
       assert_equal({'delete_me' => 'v0', 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # Written
       assert_equal({'body' => 'v1', 'user' => 'v1'}, @twitter.get(:Users, k + '1')) # Written
       assert_equal({}, @twitter.get(:Users, k + '2')) # Not yet written
       assert_equal({}, @twitter.get(:Statuses, k + '3')) # Not yet written
+      assert_equal({}, @twitter.get(:UserCounters, 'bob')) if CASSANDRA_VERSION.to_f >= 0.8 # Not yet written
 
-      @twitter.remove(:Users, k + '1') # Full row 
+      @twitter.remove(:Users, k + '1') # Full row
       assert_equal({'body' => 'v1', 'user' => 'v1'}, @twitter.get(:Users, k + '1')) # Not yet removed
 
       @twitter.remove(:Users, k + '0', 'delete_me') # A single column of the row
       assert_equal({'delete_me' => 'v0', 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # Not yet removed
-      
+
       @twitter.remove(:Users, k + '4')
       @twitter.insert(:Users, k + '4', {'body' => 'v4', 'user' => 'v4'})
       assert_equal({}, @twitter.get(:Users, k + '4')) # Not yet written
 
       # SuperColumns
       # Add and delete new sub columns to the user timeline supercolumn
-      @twitter.insert(:StatusRelationships, k, {'user_timelines' => new_subcolumns }) 
+      @twitter.insert(:StatusRelationships, k, {'user_timelines' => new_subcolumns })
       @twitter.remove(:StatusRelationships, k, 'user_timelines' , subcolumn_to_delete ) # Delete the first of the initial_subcolumns from the user_timeline supercolumn
       assert_equal(initial_subcolumns, @twitter.get(:StatusRelationships, k, 'user_timelines')) # No additions or deletes reflected yet
-      # Delete a complete supercolumn 
+      # Delete a complete supercolumn
       @twitter.remove(:StatusRelationships, k, 'dummy_supercolumn' ) # Delete the full dummy supercolumn
-      assert_equal({@uuids[5] => 'value'}, @twitter.get(:StatusRelationships, k, 'dummy_supercolumn')) # dummy supercolumn not yet deleted 
+      assert_equal({@uuids[5] => 'value'}, @twitter.get(:StatusRelationships, k, 'dummy_supercolumn')) # dummy supercolumn not yet deleted
     end
 
     assert_equal({'body' => 'v2', 'user' => 'v2'}, @twitter.get(:Users, k + '2')) # Written
     assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}, @twitter.get(:Users, k + '3')) # Written and compacted
     assert_equal({'body' => 'v4', 'user' => 'v4'}, @twitter.get(:Users, k + '4')) # Written
     assert_equal({'body' => 'v'}, @twitter.get(:Statuses, k + '3')) # Written
+    assert_equal({'tweet_count' => 5}, @twitter.get(:UserCounters, 'bob')) if CASSANDRA_VERSION.to_f >= 0.8 # Written
     assert_equal({}, @twitter.get(:Users, k + '1')) # Removed
-    
+
     assert_equal({ 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # 'delete_me' column removed
-    
+
 
     assert_equal({'body' => 'v2', 'user' => 'v2'}.keys.sort, @twitter.get(:Users, k + '2').timestamps.keys.sort) # Written
     assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}.keys.sort, @twitter.get(:Users, k + '3').timestamps.keys.sort) # Written and compacted
@@ -582,7 +602,7 @@ class CassandraTest < Test::Unit::TestCase
     # Final result: initial_subcolumns - initial_subcolumns.first + new_subcolumns
     resulting_subcolumns = initial_subcolumns.merge(new_subcolumns).reject{|k2,v| k2 == subcolumn_to_delete }
     assert_equal(resulting_subcolumns, @twitter.get(:StatusRelationships, key, 'user_timelines'))
-    assert_equal({}, @twitter.get(:StatusRelationships, key, 'dummy_supercolumn')) # dummy supercolumn deleted 
+    assert_equal({}, @twitter.get(:StatusRelationships, key, 'dummy_supercolumn')) # dummy supercolumn deleted
 
   end
 
@@ -823,6 +843,75 @@ class CassandraTest < Test::Unit::TestCase
       assert_equal(1, @twitter.get(:UserCounterAggregates, 'bob', 'DAU', 'today'))
       assert_equal(2, @twitter.get(:UserCounterAggregates, 'bob', 'DAU', 'tomorrow'))
     end
+    
+    def test_reading_rows_with_super_column_counter
+      assert_nil @twitter.add(:UserCounterAggregates, 'bob', 1, 'DAU', 'today')
+      assert_nil @twitter.add(:UserCounterAggregates, 'bob', 2, 'DAU', 'tomorrow')
+      result = @twitter.get(:UserCounterAggregates, 'bob')
+      assert_equal(1, result.size)
+      assert_equal(2, result.first.size)
+      assert_equal("DAU", result.first[0])
+      assert_equal(1, result.first[1]["today"])
+      assert_equal(2, result.first[1]["tomorrow"])
+    end  
+    
+    def test_composite_column_type_conversion
+      columns = {}
+      @composites.each_with_index do |c, index|
+        columns[c] = "value-#{index}"
+      end
+      @type_conversions.insert(:CompositeColumnConversion, key, columns)
+      columns_in_order = [
+        Cassandra::Composite.new([1].pack('N'), "elephant"),
+        Cassandra::Composite.new([5].pack('N'), "aardvark"),
+        Cassandra::Composite.new([5].pack('N'), "zebra"),
+        Cassandra::Composite.new([10].pack('N'), "kangaroo"),
+      ]
+      assert_equal(columns_in_order, @type_conversions.get(:CompositeColumnConversion, key).keys)
+
+      column_slice = @type_conversions.get(:CompositeColumnConversion, key,
+        :start => Cassandra::Composite.new([1].pack('N')),
+        :finish => Cassandra::Composite.new([10].pack('N')),
+      ).keys
+      assert_equal(columns_in_order[0..-2], column_slice)
+
+      column_slice = @type_conversions.get(:CompositeColumnConversion, key,
+        :start => Cassandra::Composite.new([5].pack('N')),
+        :finish => Cassandra::Composite.new([5].pack('N'), :slice => :after),
+      ).keys
+      assert_equal(columns_in_order[1..2], column_slice)
+
+      column_slice = @type_conversions.get(:CompositeColumnConversion, key,
+        :start => Cassandra::Composite.new([5].pack('N'), :slice => :after).to_s,
+      ).keys
+      assert_equal([columns_in_order[-1]], column_slice)
+
+      column_slice = @type_conversions.get(:CompositeColumnConversion, key,
+        :finish => Cassandra::Composite.new([10].pack('N'), :slice => :before).to_s,
+      ).keys
+      assert_equal(columns_in_order[0..-2], column_slice)
+
+      assert_equal('value-2', @type_conversions.get(:CompositeColumnConversion, key, columns_in_order.first))
+    end
+  end
+
+  def test_column_timestamps
+    base_time = Time.now
+    @twitter.insert(:Statuses, "time-key", { "body" => "value" })
+
+    results = @twitter.get(:Statuses, "time-key")
+    assert(results.timestamps["body"] / 1000000 >= base_time.to_i)
+  end
+
+  def test_supercolumn_timestamps
+    base_time = Time.now
+    @twitter.insert(:StatusRelationships, "time-key", { "super" => { @uuids[1] => "value" }})
+
+    results = @twitter.get(:StatusRelationships, "time-key")
+    assert_nil(results.timestamps["super"])
+
+    columns = results["super"]
+    assert(columns.timestamps[@uuids[1]] / 1000000 >= base_time.to_i)
   end
 
   private