cassandra 0.2.3 → 0.4

data/lib/cassandra/long.rb

@@ -0,0 +1,39 @@
+
+class Cassandra
+  # A temporally-ordered Long class for use in Cassandra column names
+  class Long < Comparable
+    ENTROPY = 2**12
+    MAX = 2**64
+    
+    def initialize(bytes = nil)      
+      case bytes
+      when String
+        raise TypeError, "8 bytes required" if bytes.size != 8
+        @bytes = bytes
+      when Integer
+        raise TypeError, "Integer must be between 0 and 2**64" if bytes < 0 or bytes > MAX
+        @bytes = [bytes].pack("Q")
+      when NilClass
+        # Time.stamp is 52 bytes, so we have 12 bytes of entropy left over
+        @bytes = [Time.stamp * ENTROPY + rand(ENTROPY)].pack("Q")
+      else
+        raise TypeError, "Can't convert from #{bytes.class}"
+      end
+    end
+
+    def to_i
+      @to_i ||= @bytes.unpack("Q")[0]
+    end    
+    
+    def inspect
+      ints = @bytes.unpack("Q")
+      "<Cassandra::Long##{object_id} time: #{
+          Time.at((ints[0] / ENTROPY) / 1_000_000).inspect
+        }, usecs: #{
+          (ints[0] / ENTROPY) % 1_000_000
+        }, jitter: #{
+          ints[0] % ENTROPY
+        }>"
+    end      
+  end  
+end

data/lib/cassandra/ordered_hash.rb

@@ -0,0 +1,135 @@
+
+class Cassandra
+  # Hash is ordered in Ruby 1.9!
+  if RUBY_VERSION >= '1.9'
+    OrderedHash = ::Hash
+  else  
+    #  Copyright (c) 2004-2009 David Heinemeier Hansson
+    #  
+    #  Permission is hereby granted, free of charge, to any person obtaining
+    #  a copy of this software and associated documentation files (the
+    #  "Software"), to deal in the Software without restriction, including
+    #  without limitation the rights to use, copy, modify, merge, publish,
+    #  distribute, sublicense, and/or sell copies of the Software, and to
+    #  permit persons to whom the Software is furnished to do so, subject to
+    #  the following conditions:
+    #  
+    #  The above copyright notice and this permission notice shall be
+    #  included in all copies or substantial portions of the Software.
+    #  
+    #  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    #  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    #  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    #  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+    #  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    #  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    #  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+  
+    class OrderedHash < Hash
+      require 'enumerator'    
+    
+      def self.[](*array)
+        hash = new
+        array.each_slice(2) { |key, value| hash[key] = value }
+        hash
+      end
+
+      def initialize(*args, &block)
+        super
+        @keys = []
+      end
+
+      def initialize_copy(other)
+        super
+        # make a deep copy of keys
+        @keys = other.keys
+      end
+
+      def []=(key, value)
+        @keys << key if !has_key?(key)
+        super
+      end
+
+      def delete(key)
+        if has_key? key
+          index = @keys.index(key)
+          @keys.delete_at index
+        end
+        super
+      end
+      
+      def delete_if
+        super
+        sync_keys!
+        self
+      end
+
+      def reject!
+        super
+        sync_keys!
+        self
+      end
+
+      def reject(&block)
+        dup.reject!(&block)
+      end
+
+      def keys
+        @keys.dup
+      end
+
+      def values
+        @keys.collect { |key| self[key] }
+      end
+
+      def to_hash
+        self
+      end
+
+      def each_key
+        @keys.each { |key| yield key }
+      end
+
+      def each_value
+        @keys.each { |key| yield self[key]}
+      end
+
+      def each
+        @keys.each {|key| yield [key, self[key]]}
+      end
+
+      alias_method :each_pair, :each
+
+      def clear
+        super
+        @keys.clear
+        self
+      end
+
+      def shift
+        k = @keys.first
+        v = delete(k)
+        [k, v]
+      end
+
+      def merge!(other_hash)
+        other_hash.each {|k,v| self[k] = v }
+        self
+      end
+
+      def merge(other_hash)
+        dup.merge!(other_hash)
+      end
+
+      def inspect
+        "#<OrderedHash #{super}>"
+      end
+
+    private
+
+      def sync_keys!
+        @keys.delete_if {|k| !has_key?(k)}
+      end
+    end
+  end
+end

data/lib/cassandra/safe_client.rb

@@ -0,0 +1,21 @@
+
+module CassandraThrift
+  module Cassandra
+    class SafeClient  
+      def initialize(client, transport)
+        @client = client 
+        @transport = transport
+      end
+      
+      def method_missing(*args)
+        @client.send(*args)
+      rescue IOError, UnavailableException
+        @transport.close rescue nil
+        @transport.open
+        raise if defined?(once)
+        once = true
+        retry
+      end
+    end
+  end
+end

data/lib/cassandra/serialization.rb

@@ -0,0 +1,57 @@
+
+class Cassandra
+  module Serialization
+    module String
+      def self.dump(object);
+        object.to_s
+      end
+
+      def self.load(object)
+        object
+      end
+    end
+
+    module Marshal
+      def self.dump(object)
+        ::Marshal.dump(object)
+      end
+
+      def self.load(object)
+        ::Marshal.load(object)
+      end
+    end
+
+    module JSON
+      def self.dump(object)
+        ::JSON.dump(object)
+      end
+
+      begin
+        require 'yajl/json_gem'
+        def self.load(object)
+          ::JSON.load(object)
+        end
+      rescue LoadError
+        require 'json/ext'
+        def self.load(object)
+          ::JSON.load("[#{object}]").first # :-(
+        end
+      end
+    end
+    
+    module CompressedJSON
+      def self.dump(object)
+        Zlib::Deflate.deflate(JSON.dump(object))
+      end
+
+      def self.load(object)
+        JSON.load(Zlib::Inflate.inflate(object))
+      end    
+    end
+
+    # module Avro
+    #  # Someday!
+    # end
+  end
+end
+

data/lib/cassandra/time.rb

@@ -0,0 +1,9 @@
+
+class << Time
+  def now_in_useconds
+    time = Time.now
+    time.to_i * 1_000_000 + time.usec
+  end  
+  alias :stamp :now_in_useconds
+end
+        

data/lib/cassandra/uuid.rb

@@ -0,0 +1,44 @@
+
+class Cassandra
+  # A temporally-ordered UUID class for use in Cassandra column names
+  class UUID < Comparable
+    UINT = 2**32
+    LONG = 2**64
+    MAX = 2**128
+    
+    def initialize(bytes = nil)      
+      case bytes
+      when String
+        raise TypeError, "16 bytes required" if bytes.size != 16
+        @bytes = bytes
+      when Integer
+        raise TypeError, "Integer must be between 0 and 2**128" if bytes < 0 or bytes > MAX
+        @bytes = [bytes / LONG, bytes % LONG].pack("QQ")
+      when NilClass
+        @bytes = [Time.stamp, Process.pid, rand(UINT)].pack("QII")
+      else
+        raise TypeError, "Can't convert from #{bytes.class}"
+      end
+    end
+    
+    def to_i
+      @to_i ||= begin
+        ints = @bytes.unpack("QQ")
+        ints[0] * 2**64 + ints[1]        
+      end
+    end
+    
+    def inspect
+      ints = @bytes.unpack("QII")
+      "<Cassandra::UUID##{object_id} time: #{
+          Time.at(ints[0] / 1_000_000).inspect
+        }, usecs: #{
+          ints[0] % 1_000_000
+        }, pid: #{
+          ints[1]
+        }, jitter: #{
+          ints[2]
+        }>"
+    end      
+  end  
+end

data/quickstart.sh

@@ -0,0 +1,12 @@
+
+if [ ! -e cassandra ]; then
+  rm -rf data
+  cd vendor
+  tar xjf cassandra.tar.bz2
+  mv cassandra ..
+  cd ../cassandra
+  ant
+  cd ..
+fi
+
+env CASSANDRA_INCLUDE=`pwd`/conf/cassandra.in.sh cassandra/bin/cassandra -f

data/test/cassandra_client_test.rb

@@ -0,0 +1,302 @@
+
+require 'test/unit'
+require "#{File.expand_path(File.dirname(__FILE__))}/../lib/cassandra"
+
+begin; require 'ruby-debug'; rescue LoadError; end
+
+class CassandraTest < Test::Unit::TestCase
+  include Cassandra::Constants
+
+  def setup
+    @twitter = Cassandra.new('Twitter', '127.0.0.1')
+    @twitter.clear_keyspace!
+    @blogs = Cassandra.new('Multiblog', '127.0.0.1')
+    @blogs.clear_keyspace!
+  end
+
+  def test_inspect
+    assert_nothing_raised do
+      @blogs.inspect
+      @twitter.inspect
+    end
+  end
+
+  def test_connection_reopens
+    assert_raises(NoMethodError) do
+      @twitter.insert(:Statuses, 1, {'body' => 'v'})
+    end
+    assert_nothing_raised do
+      @twitter.insert(:Statuses, key, {'body' => 'v'})
+    end
+  end
+
+  def test_get_key_name_sorted
+    @twitter.insert(:Users, key, {'body' => 'v', 'user' => 'v'})
+    assert_equal({'body' => 'v', 'user' => 'v'}, @twitter.get(:Users, key))
+    assert_equal({}, @twitter.get(:Users, 'bogus'))
+  end
+
+  def test_get_key_name_sorted_preserving_order
+    # In-order hash is preserved
+    hash = OrderedHash['a', nil, 'b', nil, 'c', nil, 'd', nil,]
+    @twitter.insert(:Users, key, hash)
+    assert_equal(hash.keys, @twitter.get(:Users, key).keys)
+
+    @twitter.remove(:Users, key)
+
+    # Out-of-order hash is returned sorted
+    hash = OrderedHash['b', nil, 'c', nil, 'd', nil, 'a', nil]
+    @twitter.insert(:Users, key, hash)
+    assert_equal(hash.keys.sort, @twitter.get(:Users, key).keys)
+    assert_not_equal(hash.keys, @twitter.get(:Users, key).keys)
+  end
+
+  def test_get_key_time_sorted
+    @twitter.insert(:Statuses, key, {'body' => 'v', 'user' => 'v'})
+    assert_equal({'body' => 'v', 'user' => 'v'}, @twitter.get(:Statuses, key))
+    assert_equal({}, @twitter.get(:Statuses, 'bogus'))
+  end
+
+  def test_get_value
+    @twitter.insert(:Statuses, key, {'body' => 'v'})
+    assert_equal 'v', @twitter.get(:Statuses, key, 'body')
+    assert_nil @twitter.get(:Statuses, 'bogus', 'body')
+
+    assert @twitter.exists?(:Statuses, key, 'body')
+    assert_nil @twitter.exists?(:Statuses, 'bogus', 'body')
+  end
+
+  def test_get_value_nil
+    @twitter.insert(:Statuses, key, {'body' => nil})
+    assert_nil @twitter.get(:Statuses, key, 'body')
+    assert @twitter.exists?(:Statuses, key, 'body')
+  end
+
+  def test_get_super_key
+    columns = {'user_timelines' => {Long.new => '4', Long.new => '5'}}
+    @twitter.insert(:StatusRelationships, key, columns)
+    assert_equal(columns, @twitter.get(:StatusRelationships, key))
+    assert_equal({}, @twitter.get(:StatusRelationships, 'bogus'))
+  end
+
+  def test_get_several_super_keys
+    columns = {
+      'user_timelines' => {Long.new => 'v1'},
+      'mentions_timelines' => {Long.new => 'v2'}}
+    @twitter.insert(:StatusRelationships, key, columns)
+
+    assert_equal(columns, @twitter.get(:StatusRelationships, key))
+    assert_equal({}, @twitter.get(:StatusRelationships, 'bogus'))
+  end
+
+  def test_get_super_sub_keys_with_limit
+    columns = {Long.new => 'v1'}
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => columns})
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => {Long.new => 'v2'}})
+    assert_equal(columns, @twitter.get(:StatusRelationships, key, "user_timelines", nil, 1))
+  end
+
+  def test_get_super_sub_keys_with_ranges
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => {Long.new => 'v1'}})
+    first_column = {Long.new => 'v2'}
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => first_column})
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => {Long.new => 'v3', Long.new => 'v4'}})    
+    last_column = {Long.new => 'v5'}
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => last_column})
+    
+    keys = @twitter.get(:StatusRelationships, key, "user_timelines").keys
+    assert_equal keys.sort, keys
+    
+    assert_equal(first_column, @twitter.get(:StatusRelationships, key, "user_timelines", nil, 1, first_column.keys.first..''))
+    assert_equal(3, @twitter.get(:StatusRelationships, key, "user_timelines", nil, 100, last_column.keys.first..first_column.keys.first).size)
+  end
+
+  def test_get_super_sub_key
+    columns = {Long.new => 'v', Long.new => 'v'}
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => columns})
+    assert_equal(columns, @twitter.get(:StatusRelationships, key, 'user_timelines'))
+    assert_equal({}, @twitter.get(:StatusRelationships, 'bogus', 'user_timelines'))
+  end
+
+  def test_get_super_value
+    columns = {Long.new => 'v'}
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => columns})
+    assert_equal('v', @twitter.get(:StatusRelationships, key, 'user_timelines', columns.keys.first))
+    assert_nil @twitter.get(:StatusRelationships, 'bogus', 'user_timelines', columns.keys.first)
+  end
+
+  def test_get_key_range
+    @twitter.insert(:Statuses, '2', {'body' => '1'})
+    @twitter.insert(:Statuses, '3', {'body' => '1'})
+    @twitter.insert(:Statuses, '4', {'body' => '1'})
+    @twitter.insert(:Statuses, '5', {'body' => '1'})
+    @twitter.insert(:Statuses, '6', {'body' => '1'})
+    assert_equal(['3', '4', '5'], @twitter.get_key_range(:Statuses, '3'..'5'))
+  end
+
+  def test_multi_get
+    @twitter.insert(:Users, key + '1', {'body' => 'v1', 'user' => 'v1'})
+    @twitter.insert(:Users, key + '2', {'body' => 'v2', 'user' => 'v2'})
+    assert_equal(
+      OrderedHash[key + '1', {'body' => 'v1', 'user' => 'v1'}, key + '2', {'body' => 'v2', 'user' => 'v2'}, 'bogus', {}],
+      @twitter.multi_get(:Users, [key + '1', key + '2', 'bogus']))
+    assert_equal(
+      OrderedHash[key + '2', {'body' => 'v2', 'user' => 'v2'}, 'bogus', {}, key + '1', {'body' => 'v1', 'user' => 'v1'}],
+      @twitter.multi_get(:Users, [key + '2', 'bogus', key + '1']))
+  end
+
+  def test_remove_key
+    @twitter.insert(:Statuses, key, {'body' => 'v'})
+    assert_equal({'body' => 'v'}, @twitter.get(:Statuses, key))
+
+    @twitter.remove(:Statuses, key)
+    assert_equal({}, @twitter.get(:Statuses, key))
+    assert_equal 0, @twitter.count(:Statuses)
+  end
+
+  def test_remove_value
+    @twitter.insert(:Statuses, key, {'body' => 'v'})
+    @twitter.remove(:Statuses, key, 'body')
+    assert_nil @twitter.get(:Statuses, key, 'body')
+  end
+
+  def test_remove_super_key
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => {Long.new => 'v'}})
+    @twitter.remove(:StatusRelationships, key)
+    assert_equal({}, @twitter.get(:StatusRelationships, key))
+  end
+
+  def test_remove_super_sub_key
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => {Long.new => 'v'}})
+    @twitter.remove(:StatusRelationships, key, 'user_timelines')
+    assert_equal({}, @twitter.get(:StatusRelationships, key, 'user_timelines'))
+  end
+
+  def test_remove_super_value
+    columns = {Long.new => 'v'}
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => columns})
+    @twitter.remove(:StatusRelationships, key, 'user_timelines', columns.keys.first)
+    assert_nil @twitter.get(:StatusRelationships, key, 'user_timelines', columns.keys.first)
+  end
+
+  def test_clear_column_family
+    @twitter.insert(:Statuses, key + "1", {'body' => '1'})
+    @twitter.insert(:Statuses, key + "2", {'body' => '2'})
+    @twitter.insert(:Statuses, key + "3", {'body' => '3'})
+    @twitter.clear_column_family!(:Statuses)
+    assert_equal 0, @twitter.count(:Statuses)
+  end
+
+  def test_insert_key
+    @twitter.insert(:Statuses, key, {'body' => 'v', 'user' => 'v'})
+    assert_equal({'body' => 'v', 'user' => 'v'}, @twitter.get(:Statuses, key))
+  end
+
+  def test_insert_super_key
+    columns = {Long.new => 'v', Long.new => 'v'}
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => columns})
+    assert_equal(columns, @twitter.get(:StatusRelationships, key, 'user_timelines'))
+  end
+
+  def test_get_columns
+    @twitter.insert(:Statuses, key, {'body' => 'v1', 'user' => 'v2'})
+    assert_equal(['v1' , 'v2'], @twitter.get_columns(:Statuses, key, ['body', 'user']))
+  end
+
+  def test_get_column_values_super
+    user_columns, mentions_columns = {Long.new => 'v1'}, {Long.new => 'v2'}
+    @twitter.insert(:StatusRelationships, key,
+      {'user_timelines' => user_columns, 'mentions_timelines' => mentions_columns})
+    assert_equal [user_columns, mentions_columns],
+      @twitter.get_columns(:StatusRelationships, key, ['user_timelines', 'mentions_timelines'])
+  end
+
+  def test_multi_get_columns
+    @twitter.insert(:Users, key + '1', {'body' => 'v1', 'user' => 'v1'})
+    @twitter.insert(:Users, key + '2', {'body' => 'v2', 'user' => 'v2'})
+    assert_equal(
+      OrderedHash[key + '1', ['v1', 'v1'], key + '2', ['v2', 'v2'], 'bogus', [nil, nil]],
+      @twitter.multi_get_columns(:Users, [key + '1', key + '2', 'bogus'], ['body', 'user']))
+    assert_equal(
+      OrderedHash[key + '2', ['v2', 'v2'], 'bogus', [nil, nil], key + '1', ['v1', 'v1']],
+      @twitter.multi_get_columns(:Users, [key + '2', 'bogus', key + '1'], ['body', 'user']))
+  end
+
+  # Not supported
+  #  def test_get_columns_super_sub
+  #    @twitter.insert(:StatusRelationships, key, {
+  #      'user_timelines' => {Long.new => 'v1'},
+  #      'mentions_timelines' => {Long.new => 'v2'}})
+  #    assert_equal ['v1', 'v2'],
+  #      @twitter.get_columns(:StatusRelationships, key, 'user_timelines', ['1', key])
+  #  end
+
+  def test_count_keys
+    @twitter.insert(:Statuses, key + "1", {'body' => '1'})
+    @twitter.insert(:Statuses, key + "2", {'body' => '2'})
+    @twitter.insert(:Statuses, key + "3", {'body' => '3'})
+    assert_equal 3, @twitter.count(:Statuses)
+  end
+
+  def test_count_columns
+    @twitter.insert(:Statuses, key, {'body' => 'v1', 'user' => 'v2'})
+    assert_equal 2, @twitter.count_columns(:Statuses, key)
+  end
+
+  def test_count_super_columns
+    @twitter.insert(:StatusRelationships, key, {
+      'user_timelines' => {Long.new => 'v1'},
+      'mentions_timelines' => {Long.new => 'v2'}})
+    assert_equal 2, @twitter.count_columns(:StatusRelationships, key)
+  end
+
+  def test_count_super_sub_columns
+    @twitter.insert(:StatusRelationships, key, {'user_timelines' => {Long.new => 'v1', Long.new => 'v2'}})
+    assert_equal 2, @twitter.count_columns(:StatusRelationships, key, 'user_timelines')
+  end
+
+  def test_multi_count_columns
+    @twitter.insert(:Users, key + '1', {'body' => 'v1', 'user' => 'v1'})
+    @twitter.insert(:Users, key + '2', {'body' => 'v2', 'user' => 'v2'})
+    assert_equal(
+      OrderedHash[key + '1', 2, key + '2', 2, 'bogus', 0],
+      @twitter.multi_count_columns(:Users, [key + '1', key + '2', 'bogus']))
+    assert_equal(
+      OrderedHash[key + '2', 2, 'bogus', 0, key + '1', 2],
+      @twitter.multi_count_columns(:Users, [key + '2', 'bogus', key + '1']))
+  end
+
+  def test_batch_insert
+    @twitter.insert(:Users, key + '1', {'body' => 'v1', 'user' => 'v1'})
+
+    @twitter.batch do
+      @twitter.insert(:Users, key + '2', {'body' => 'v2', 'user' => 'v2'})
+      @twitter.insert(:Users, key + '3', {'body' => 'bogus', 'user' => 'v3'})
+      @twitter.insert(:Users, key + '3', {'body' => 'v3', 'location' => 'v3'})
+      @twitter.insert(:Statuses, key + '3', {'body' => 'v'})
+
+      assert_equal({'body' => 'v1', 'user' => 'v1'}, @twitter.get(:Users, key + '1')) # Written
+      assert_equal({}, @twitter.get(:Users, key + '2')) # Not yet written
+      assert_equal({}, @twitter.get(:Statuses, key + '3')) # Not yet written
+
+      @twitter.remove(:Users, key + '1')
+      assert_equal({'body' => 'v1', 'user' => 'v1'}, @twitter.get(:Users, key + '1')) # Not yet removed
+
+      @twitter.remove(:Users, key + '4')
+      @twitter.insert(:Users, key + '4', {'body' => 'v4', 'user' => 'v4'})
+      assert_equal({}, @twitter.get(:Users, key + '4')) # Not yet written
+    end
+
+    assert_equal({'body' => 'v2', 'user' => 'v2'}, @twitter.get(:Users, key + '2')) # Written
+    assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}, @twitter.get(:Users, key + '3')) # Written and compacted
+    assert_equal({'body' => 'v4', 'user' => 'v4'}, @twitter.get(:Users, key + '4')) # Written
+    assert_equal({'body' => 'v'}, @twitter.get(:Statuses, key + '3')) # Written
+    assert_equal({}, @twitter.get(:Users, key + '1')) # Removed
+  end
+
+  private
+
+  def key
+    caller.first[/`(.*?)'/, 1]
+  end
+end