cassandra_client 0.1

data/lib/cassandra_client/ordered_hash.rb

@@ -0,0 +1,134 @@
+#  Copyright (c) 2004-2009 David Heinemeier Hansson
+#  
+#  Permission is hereby granted, free of charge, to any person obtaining
+#  a copy of this software and associated documentation files (the
+#  "Software"), to deal in the Software without restriction, including
+#  without limitation the rights to use, copy, modify, merge, publish,
+#  distribute, sublicense, and/or sell copies of the Software, and to
+#  permit persons to whom the Software is furnished to do so, subject to
+#  the following conditions:
+#  
+#  The above copyright notice and this permission notice shall be
+#  included in all copies or substantial portions of the Software.
+#  
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+#  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+#  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+#  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+class CassandraClient
+  # Hash is ordered in Ruby 1.9!
+  if RUBY_VERSION >= '1.9'
+    OrderedHash = ::Hash
+  else
+    class OrderedHash < Hash #:nodoc:
+      require 'enumerator'    
+    
+      def self.[](*array)
+        hash = new
+        array.each_slice(2) { |key, value| hash[key] = value }
+        hash
+      end
+
+      def initialize(*args, &block)
+        super
+        @keys = []
+      end
+
+      def initialize_copy(other)
+        super
+        # make a deep copy of keys
+        @keys = other.keys
+      end
+
+      def []=(key, value)
+        @keys << key if !has_key?(key)
+        super
+      end
+
+      def delete(key)
+        if has_key? key
+          index = @keys.index(key)
+          @keys.delete_at index
+        end
+        super
+      end
+      
+      def delete_if
+        super
+        sync_keys!
+        self
+      end
+
+      def reject!
+        super
+        sync_keys!
+        self
+      end
+
+      def reject(&block)
+        dup.reject!(&block)
+      end
+
+      def keys
+        @keys.dup
+      end
+
+      def values
+        @keys.collect { |key| self[key] }
+      end
+
+      def to_hash
+        self
+      end
+
+      def each_key
+        @keys.each { |key| yield key }
+      end
+
+      def each_value
+        @keys.each { |key| yield self[key]}
+      end
+
+      def each
+        @keys.each {|key| yield [key, self[key]]}
+      end
+
+      alias_method :each_pair, :each
+
+      def clear
+        super
+        @keys.clear
+        self
+      end
+
+      def shift
+        k = @keys.first
+        v = delete(k)
+        [k, v]
+      end
+
+      def merge!(other_hash)
+        other_hash.each {|k,v| self[k] = v }
+        self
+      end
+
+      def merge(other_hash)
+        dup.merge!(other_hash)
+      end
+
+      def inspect
+        "#<OrderedHash #{super}>"
+      end
+
+    private
+
+      def sync_keys!
+        @keys.delete_if {|k| !has_key?(k)}
+      end
+    end
+  end
+end

data/lib/cassandra_client/serialization.rb

@@ -0,0 +1,64 @@
+
+class CassandraClient
+  module Serialization
+    module String
+      def dump(object); 
+        object.to_s
+      end
+      
+      def load(object)
+        object
+      end
+    end
+  
+    module Marshal
+      def dump(object)
+        ::Marshal.dump(object)
+      end
+
+      def load(object)
+        ::Marshal.load(object)
+      end
+    end
+
+    module JSON
+      def dump(object)
+        ::JSON.dump(object)
+      end
+
+      begin
+        require 'yajl/json_gem'
+        def load(object)
+          ::JSON.load(object)
+        end
+      rescue LoadError      
+        require 'json/ext'        
+        def load(object)
+          ::JSON.load("[#{object}]").first # :-(
+        end
+      end
+    end
+      
+    # module Avro
+    #  # Someday!
+    # end
+    
+    # Decorate all available modules with compression
+    self.constants.each do |module_name|
+      eval <<-MODULE
+        module Compressed#{module_name}
+          include #{module_name}
+          def dump(object)
+            Zlib::Deflate.deflate(super(object))
+          end
+    
+          def load(object)        
+            super(Zlib::Inflate.inflate(object))
+          end
+        end
+      MODULE
+    end
+    
+  end
+end
+

data/lib/cassandra_client/table.rb

@@ -0,0 +1,202 @@
+class CassandraClient
+  class Table
+    attr_reader :name, :schema, :parent
+    
+    MAX_INT = 2**31 - 1
+  
+    def initialize(name, parent)
+      @parent = parent
+      @client = parent.client
+      @block_for = parent.block_for
+
+      @name = name
+      @schema = @client.describeTable(@name)
+      extend(parent.serialization)
+    end
+    
+    def inspect(full = true)
+      string = "#<CassandraClient::Table:#{object_id}, @name=#{name.inspect}"
+      string += ", @schema={#{schema.map {|name, hash| ":#{name} => #{hash['type'].inspect}"}.join(', ')}}, @parent=#{parent.inspect(false)}" if full
+      string + ">"
+    end
+  
+    ## Write
+    
+    # Insert a row for a key. Pass a flat hash for a regular column family, and 
+    # a nested hash for a super column family.
+    def insert(key, column_family, hash, timestamp = now)
+      column_family = column_family.to_s    
+      insert = is_super(column_family) ? :insert_super : :insert_standard
+      send(insert, key, column_family, hash, timestamp)
+    end
+    
+    private
+  
+    def insert_standard(key, column_family, hash, timestamp = now)
+      mutation = Batch_mutation_t.new(
+        :table => @name, 
+        :key => key, 
+        :cfmap => {column_family => hash_to_columns(hash, timestamp)})
+      @client.batch_insert(mutation, @block_for)
+    end 
+  
+    def insert_super(key, column_family, hash, timestamp = now)
+      mutation = Batch_mutation_super_t.new(
+        :table => @name, 
+        :key => key, 
+        :cfmap => {column_family => hash_to_super_columns(hash, timestamp)})
+      @client.batch_insert_superColumn(mutation, @block_for)
+    end 
+    
+    public
+    
+    ## Delete
+    
+    # Remove the element at the column_family:key:super_column:column 
+    # path you request.
+    def remove(key, column_family, super_column = nil, column = nil, timestamp = now)
+      column_family = column_family.to_s
+      column_family += ":#{super_column}" if super_column
+      column_family += ":#{column}" if column
+      @client.remove(@name, key, column_family, timestamp, @block_for )
+    end
+    
+    # Remove all rows in the column family you request.
+    def remove_all(column_family)
+      get_key_range(column_family).each do |key| 
+        remove(key, column_family)
+      end
+    end
+    
+    ## Read
+  
+    # Count the elements at the column_family:key:super_column path you 
+    # request.
+    def count_columns(key, column_family, super_column = nil)
+      column_family = column_family.to_s
+      column_family += ":#{super_column}" if super_column
+      @client.get_column_count(@name, key, column_family)
+    end
+    
+    # Return a list of single values for the elements at the
+    # column_family:key:super_column:column path you request.
+    def get_columns(key, column_family, super_columns, columns = nil)
+      column_family = column_family.to_s
+      get_slice_by_names = (is_super(column_family) && !columns) ? :get_slice_super_by_names : :get_slice_by_names
+      if super_columns and columns
+        column_family += ":#{super_columns}" 
+        columns = Array(columns)
+      else
+        columns = Array(super_columns)
+      end
+          
+      hash = columns_to_hash(@client.send(get_slice_by_names, @name, key, column_family, columns))
+      columns.map { |column| hash[column] }
+    end
+          
+    # Return a hash (actually, a CassandraClient::OrderedHash) or a single value 
+    # representing the element at the column_family:key:super_column:column 
+    # path you request.
+    def get(key, column_family, super_column = nil, column = nil, offset = -1, limit = 100)
+      column_family = column_family.to_s
+      column_family += ":#{super_column}" if super_column
+      column_family += ":#{column}" if column          
+      
+      # You have got to be kidding
+      if is_super(column_family)
+        if column
+          load(@client.get_column(@name, key, column_family).value)
+        elsif super_column
+          columns_to_hash(@client.get_superColumn(@name, key, column_family).columns)
+        else
+          columns_to_hash(@client.get_slice_super(@name, key, "#{column_family}:", offset, limit))
+        end
+      else
+        if super_column
+          load(@client.get_column(@name, key, column_family).value)
+        elsif is_sorted_by_time(column_family)
+          result = columns_to_hash(@client.get_columns_since(@name, key, column_family, 0))
+
+          # FIXME Hack until get_slice on a time-sorted column family works again
+          result = OrderedHash[*flatten_once(result.to_a[offset, limit])] if offset > -1
+          result
+        else
+          columns_to_hash(@client.get_slice(@name, key, "#{column_family}:", offset, limit))
+        end 
+      end
+    rescue NotFoundException
+      is_super(column_family) && !column ? {} : nil
+    end  
+    
+    # FIXME
+    # def get_recent(key, column_family, super_column = nil, column = nil, timestamp = 0)
+    # end
+  
+    # Return a list of keys in the column_family you request. Requires the
+    # table to be partitioned with OrderPreservingHash.
+    def get_key_range(key_range, column_family = nil, limit = 100)      
+      column_family, key_range = key_range, ''..'' unless column_family
+      column_families = Array(column_family).map {|c| c.to_s}
+      @client.get_key_range(@name, column_families, key_range.begin, key_range.end, limit)
+    end
+    
+    # Count all rows in the column_family you request. Requires the table 
+    # to be partitioned with OrderPreservingHash.
+    def count(key_range, column_family = nil, limit = MAX_INT)
+      get_key_range(key_range, column_family, limit).size
+    end
+      
+    private
+    
+    def is_super(column_family)
+      column_family_property(column_family, 'type') == 'Super'
+    end
+
+    def is_sorted_by_time(column_family)
+      column_family_property(column_family, 'sort') == 'Time'
+    end
+    
+    def column_family_property(column_family_or_path, key)
+      column_family = column_family_or_path.to_s.split(':').first    
+      @schema[column_family][key]
+    rescue NoMethodError
+      raise AccessError, "Invalid column family \":#{column_family}\""    
+    end
+    
+    def columns_to_hash(columns)
+      hash = ::CassandraClient::OrderedHash.new
+      Array(columns).each do |c| 
+        if c.is_a?(SuperColumn_t)
+          hash[c.name] = columns_to_hash(c.columns)
+        else
+          hash[c.columnName] = load(c.value)
+        end
+      end
+      hash
+    end  
+    
+    def hash_to_columns(hash, timestamp)
+      hash.map do |column, value|
+        Column_t.new(:columnName => column, :value => dump(value), :timestamp => timestamp)
+      end    
+    end
+    
+    def hash_to_super_columns(hash, timestamp)
+      hash.map do |super_column, columns|
+        SuperColumn_t.new(:name => super_column, :columns => hash_to_columns(columns, timestamp))
+      end
+    end
+    
+    def time_in_microseconds
+      time = Time.now
+      time.to_i * 1_000_000 + time.usec
+    end
+    alias :now :time_in_microseconds
+    
+    def flatten_once(array)
+      result = []
+      array.each { |el| result.concat(el) }
+      result
+    end    
+  end
+end

data/lib/cassandra_client.rb

@@ -0,0 +1,14 @@
+
+require 'zlib'
+require 'rubygems'
+require 'thrift'
+
+HERE = File.expand_path(File.dirname(__FILE__))
+
+require "#{HERE}/cassandra_client/client"
+require "#{HERE}/cassandra_client/table"
+require "#{HERE}/cassandra_client/serialization"
+require "#{HERE}/cassandra_client/ordered_hash"
+
+$LOAD_PATH << "#{HERE}/../vendor/gen-rb"
+require 'cassandra'

data/quickstart.sh

@@ -0,0 +1,10 @@
+
+if [ ! -e cassandra-r789419 ]; then
+  cd vendor
+  tar xjvf cassandra-r789419.tar.bz2
+  mv cassandra-r789419 ..
+  cd ..
+fi
+
+export CASSANDRA_INCLUDE=`pwd`/conf/cassandra.in.sh
+cassandra-r789419/bin/cassandra -f

data/test/cassandra_client_test.rb

@@ -0,0 +1,204 @@
+
+require 'test/unit'
+require "#{File.expand_path(File.dirname(__FILE__))}/../lib/cassandra_client"
+
+begin; require 'ruby-debug'; rescue LoadError; end
+
+class CassandraClientTest < Test::Unit::TestCase
+  def setup
+    @client = CassandraClient.new('127.0.0.1')
+    @client.remove_all
+    @statuses = @client.table('Statuses')
+    @users = @client.table('Users')
+  end
+  
+  def test_inspect
+    assert_nothing_raised do
+      @statuses.inspect
+      @client.inspect
+    end
+  end
+  
+  def test_connection_reopens
+    assert_raises(NoMethodError) do
+      @statuses.insert(1, :row, {'body' => 'v'})
+    end
+    assert_nothing_raised do
+      @statuses.insert(key, :row, {'body' => 'v'})
+    end
+  end  
+
+  def test_get_key_name_sorted
+    @users.insert(key, :row, {'body' => 'v', 'user' => 'v'})
+    assert_equal({'body' => 'v', 'user' => 'v'}, @users.get(key, :row))
+    assert_equal({}, @users.get('bogus', :row))
+  end
+  
+  def test_get_key_name_sorted_preserving_order
+    # In-order hash is preserved
+    hash = CassandraClient::OrderedHash['a', '', 'b', '', 'c', '', 'd', '',]    
+    @users.insert(key, :row, hash)
+    assert_equal(hash.keys, @users.get(key, :row).keys)
+    
+    @users.remove(key, :row)
+        
+    # Out-of-order hash is returned sorted
+    hash = CassandraClient::OrderedHash['b', '', 'c', '', 'd', '', 'a', '']    
+    @users.insert(key, :row, hash)
+    assert_equal(hash.keys.sort, @users.get(key, :row).keys)
+    assert_not_equal(hash.keys, @users.get(key, :row).keys)
+  end  
+
+  def test_get_key_time_sorted
+    @statuses.insert(key, :row, {'body' => 'v', 'user' => 'v'})
+    assert_equal({'body' => 'v', 'user' => 'v'}, @statuses.get(key, :row))
+    assert_equal({}, @statuses.get('bogus', :row))
+  end
+    
+  def test_get_key_time_sorted_with_limit
+    @statuses.insert(key, :row, {'first' => 'v'})
+    @statuses.insert(key, :row, {'second' => 'v'})
+    assert_equal({'second' => 'v'}, @statuses.get(key, :row, nil, nil, 0, 1))
+  end    
+
+  def test_get_value
+    @statuses.insert(key, :row, {'body' => 'v'})
+    assert_equal 'v', @statuses.get(key, :row, 'body')
+    assert_nil @statuses.get('bogus', :row, 'body')
+  end
+    
+  def test_get_super_key
+    @statuses.insert(key, :relationships, {'user_timelines' => {'4' => 'v', '5' => 'v'}})
+    assert_equal({'user_timelines' => {'4' => 'v', '5' => 'v'}}, @statuses.get(key, :relationships))
+    assert_equal({}, @statuses.get('bogus', :relationships))
+  end
+
+  def test_get_super_key_multi
+    @statuses.insert(key, :relationships, {
+      'user_timelines' => {'1' => 'v1'}, 
+      'mentions_timelines' => {'2' => 'v2'}})
+    assert_equal({
+      'user_timelines' => {'1' => 'v1'}, 
+      'mentions_timelines' => {'2' => 'v2'}}, @statuses.get(key, :relationships))
+    assert_equal({}, @statuses.get('bogus', :relationships))
+  end
+
+  def test_get_super_sub_key
+    @statuses.insert(key, :relationships, {'user_timelines' => {'4' => 'v', '5' => 'v'}})
+    assert_equal({'4' => 'v', '5' => 'v'}, @statuses.get(key, :relationships, 'user_timelines'))
+    assert_equal({}, @statuses.get('bogus', :relationships, 'user_timelines'))
+  end
+  
+  def test_get_super_value
+    @statuses.insert(key, :relationships, {'user_timelines' => {'1' => 'v'}})
+    assert_equal('v', @statuses.get(key, :relationships, 'user_timelines', '1'))
+    assert_nil @statuses.get('bogus', :relationships, 'user_timelines', '1')
+  end  
+  
+  def test_get_key_range
+    @statuses.insert('3', :row, {'body' => 'v'})
+    @statuses.insert('4', :row, {'body' => 'v'})
+    @statuses.insert('5', :row, {'body' => 'v'})
+    assert_equal(['3', '4', '5'], @statuses.get_key_range('3'..'5', :row))
+  end
+
+  # Not supported
+  #  def test_get_key_range_super
+  #    @statuses.insert('3', :relationships, {'user_timelines' => {'1' => 'v'}})
+  #    @statuses.insert('4', :relationships, {'user_timelines' => {'1' => 'v'}})
+  #    @statuses.insert('5', :relationships, {'user_timelines' => {'1' => 'v'}})
+  #    assert_equal(['3', '4', '5'], @statuses.get_key_range('3'..'5', :relationships, 'user_timelines'))
+  #  end
+  
+  def test_remove_key
+    @statuses.insert(key, :row, {'body' => 'v'})
+    @statuses.remove(key, :row)
+    assert_equal({}, @statuses.get(key, :row))
+  end
+
+  def test_remove_value
+    @statuses.insert(key, :row, {'body' => 'v'})
+    @statuses.remove(key, :row, 'body')
+    assert_nil @statuses.get(key, :row, 'body')    
+  end
+
+  def test_remove_super_key
+    @statuses.insert(key, :relationships, {'user_timelines' => {'1' => 'v'}})
+    @statuses.remove(key, :relationships)
+    assert_equal({}, @statuses.get(key, :relationships))
+  end
+
+  def test_remove_super_sub_key
+    @statuses.insert(key, :relationships, {'user_timelines' => {'1' => 'v'}})
+    @statuses.remove(key, :relationships, 'user_timelines')
+    assert_equal({}, @statuses.get(key, :relationships, 'user_timelines'))
+  end
+
+  def test_remove_super_value
+    @statuses.insert(key, :relationships, {'user_timelines' => {'1' => 'v'}})
+    @statuses.remove(key, :relationships, 'user_timelines', '1')
+    assert_nil @statuses.get(key, :relationships, 'user_timelines', '1')    
+  end
+
+  def test_insert_key
+    @statuses.insert(key, :row, {'body' => 'v', 'user' => 'v'})
+    assert_equal({'body' => 'v', 'user' => 'v'}, @statuses.get(key, :row))  
+  end
+
+  def test_insert_super_key
+    @statuses.insert(key, :relationships, {'user_timelines' => {'1' => 'v', key => 'v'}})
+    assert_equal({'1' => 'v' , key => 'v'}, @statuses.get(key, :relationships, 'user_timelines'))  
+  end
+  
+  def test_get_column_values
+    @statuses.insert(key, :row, {'body' => 'v1', 'user' => 'v2'})
+    assert_equal(['v1' , 'v2'], @statuses.get_columns(key, :row, ['body', 'user']))
+  end  
+
+  def test_get_column_values_super
+    @statuses.insert(key, :relationships, {
+      'user_timelines' => {'1' => 'v1'}, 
+      'mentions_timelines' => {'2' => 'v2'}})
+    assert_equal [{'1' => 'v1'}, {'2' => 'v2'}], 
+      @statuses.get_columns(key, :relationships, ['user_timelines', 'mentions_timelines'])
+  end  
+
+  # Not supported
+  #  def test_get_columns_super_sub
+  #    @statuses.insert(key, :relationships, {
+  #      'user_timelines' => {'1' => 'v1'}, 
+  #      'mentions_timelines' => {'2' => 'v2'}})
+  #    assert_equal ['v1', 'v2'], 
+  #      @statuses.get_columns(key, :relationships, 'user_timelines', ['1', key])
+  #  end    
+  
+  def test_count_keys
+    @statuses.insert(key + "1", :row, {'body' => 'v1'})
+    @statuses.insert(key + "2", :row, {'body' => 'v1'})
+    @statuses.insert(key + "3", :row, {'body' => 'v1'})
+    assert_equal 3, @statuses.count(:row)  
+  end
+  
+  def test_count_columns
+    @statuses.insert(key, :row, {'body' => 'v1', 'user' => 'v2'})
+    assert_equal 2, @statuses.count_columns(key, :row)
+  end 
+
+  def test_count_super_columns
+    @statuses.insert(key, :relationships, {
+      'user_timelines' => {'1' => 'v1'}, 
+      'mentions_timelines' => {'2' => 'v2'}})
+    assert_equal 2, @statuses.count_columns(key, :relationships)
+  end 
+
+  def test_count_super_sub_columns
+    @statuses.insert(key, :relationships, {'user_timelines' => {'1' => 'v1', key => 'v2'}})
+    assert_equal 2, @statuses.count_columns(key, :relationships, 'user_timelines')
+  end
+  
+  private
+  
+  def key
+    caller.first[/`(.*?)'/, 1]
+  end
+end