cassilds 0.9.1

data/lib/cassandra/mock.rb

@@ -0,0 +1,326 @@
+require 'nokogiri'
+
+class SimpleUUID::UUID
+  def >=(other)
+    (self <=> other) >= 0
+  end
+
+  def <=(other)
+    (self <=> other) <= 0
+  end
+end
+
+class Cassandra
+  class Mock
+    include ::Cassandra::Helpers
+    include ::Cassandra::Columns
+
+    def initialize(keyspace, storage_xml)
+      @keyspace = keyspace
+      @column_name_class = {}
+      @sub_column_name_class = {}
+      @storage_xml = storage_xml
+      clear_keyspace!
+    end
+
+    def disconnect!
+    end
+
+    def clear_keyspace!
+      @data = {}
+    end
+
+    def clear_column_family!(column_family)
+      @data[column_family.to_sym] = OrderedHash.new
+    end
+
+    def insert(column_family, key, hash_or_array, options = {})
+      if @batch
+        @batch << [:insert, column_family, key, hash_or_array, options]
+      else
+        raise ArgumentError if key.nil?
+        if column_family_type(column_family) == 'Standard'
+          insert_standard(column_family, key, hash_or_array)
+        else
+          insert_super(column_family, key, hash_or_array)
+        end
+      end
+    end
+
+    def insert_standard(column_family, key, hash_or_array)
+      old = cf(column_family)[key] || OrderedHash.new
+      cf(column_family)[key] = merge_and_sort(old, hash_or_array)
+    end
+
+    def insert_super(column_family, key, hash)
+      raise ArgumentError unless hash.is_a?(Hash)
+      cf(column_family)[key] ||= OrderedHash.new
+
+      hash.keys.each do |sub_key|
+        old = cf(column_family)[key][sub_key] || OrderedHash.new
+        cf(column_family)[key][sub_key] = merge_and_sort(old, hash[sub_key])
+      end
+    end
+
+    def batch
+      @batch = []
+      yield
+      b = @batch
+      @batch = nil
+      b.each do |mutation|
+        send(*mutation)
+      end
+    ensure
+      @batch = nil
+    end
+
+    def get(column_family, key, *columns_and_options)
+      column_family, column, sub_column, options =
+        extract_and_validate_params_for_real(column_family, [key], columns_and_options, READ_DEFAULTS)
+      if column_family_type(column_family) == 'Standard'
+        get_standard(column_family, key, column, options)
+      else
+        get_super(column_family, key, column, sub_column, options)
+      end
+    end
+
+    def get_standard(column_family, key, column, options)
+      row = cf(column_family)[key] || OrderedHash.new
+      if column
+        row[column]
+      else
+        row = apply_range(row, column_family, options[:start], options[:finish])
+        apply_count(row, options[:count], options[:reversed])
+      end
+    end
+
+    def get_super(column_family, key, column, sub_column, options)
+      if column
+        if sub_column
+          if cf(column_family)[key] &&
+             cf(column_family)[key][column] &&
+             cf(column_family)[key][column][sub_column]
+            cf(column_family)[key][column][sub_column]
+          else
+            nil
+          end
+        else
+          row = cf(column_family)[key] && cf(column_family)[key][column] ?
+            cf(column_family)[key][column] :
+            OrderedHash.new
+          row = apply_range(row, column_family, options[:start], options[:finish], false)
+          apply_count(row, options[:count], options[:reversed])
+        end
+      elsif cf(column_family)[key]
+        cf(column_family)[key]
+      else
+        OrderedHash.new
+      end
+    end
+
+    def exists?(column_family, key, column=nil)
+      !!get(column_family, key, column)
+    end
+
+    def multi_get(column_family, keys, *columns_and_options)
+      column_family, column, sub_column, options = extract_and_validate_params_for_real(column_family, keys, columns_and_options, READ_DEFAULTS)
+      keys.inject(OrderedHash.new) do |hash, key|
+        hash[key] = get(column_family, key)
+        hash
+      end
+    end
+
+    def remove(column_family, key, *columns_and_options)
+      column_family, column, sub_column, options = extract_and_validate_params_for_real(column_family, key, columns_and_options, WRITE_DEFAULTS)
+      if @batch
+        @batch << [:remove, column_family, key, column]
+      else
+        if column
+          if sub_column
+            cf(column_family)[key][column].delete(sub_column)
+          else
+            cf(column_family)[key].delete(column)
+          end
+        else
+          cf(column_family).delete(key)
+        end
+      end
+    end
+
+    def get_columns(column_family, key, *columns_and_options)
+      column_family, columns, sub_columns, options = extract_and_validate_params_for_real(column_family, key, columns_and_options, READ_DEFAULTS)
+      d = get(column_family, key)
+
+
+      if sub_columns
+        sub_columns.collect do |sub_column|
+          d[columns][sub_column]
+        end
+      else
+        columns.collect do |column|
+          d[column]
+        end
+      end
+    end
+
+    def count_columns(column_family, key, column=nil)
+      get(column_family, key, column).keys.length
+    end
+
+    def multi_get_columns(column_family, keys, columns)
+      keys.inject(OrderedHash.new) do |hash, key|
+        hash[key] = get_columns(column_family, key, columns)
+        hash
+      end
+    end
+
+    def multi_count_columns(column_family, keys)
+      keys.inject(OrderedHash.new) do |hash, key|
+        hash[key] = count_columns(column_family, key)
+        hash
+      end
+    end
+
+    def get_range(column_family, options = {})
+      column_family, _, _, options = extract_and_validate_params_for_real(column_family, "", [options], READ_DEFAULTS)
+      _get_range(column_family, options[:start], options[:finish], options[:count]).keys
+    end
+
+    def count_range(column_family, options={})
+      count = 0
+      l = []
+      start_key = ''
+      while (l = get_range(column_family, options.merge(:count => 1000, :start => start_key))).size > 0
+        count += l.size
+        start_key = l.last.succ
+      end
+      count
+    end
+
+    def schema(load=true)
+      if !load && !@schema
+        []
+      else
+        @schema ||= schema_for_keyspace(@keyspace)
+      end
+    end
+
+    private
+
+    def _get_range(column_family, start, finish, count)
+      ret = OrderedHash.new
+      start  = to_compare_with_type(start,  column_family)
+      finish = to_compare_with_type(finish, column_family)
+      cf(column_family).keys.sort.each do |key|
+        break if ret.keys.size >= count
+        if (start.nil? || key >= start) && (finish.nil? || key <= finish)
+          ret[key] = cf(column_family)[key]
+        end
+      end
+      ret
+    end
+
+    def schema_for_keyspace(keyspace)
+      doc = read_storage_xml
+      ret = {}
+      doc.css("Keyspaces Keyspace[@Name='#{keyspace}']").css('ColumnFamily').each do |cf|
+        ret[cf['Name']] = {}
+        if cf['CompareSubcolumnsWith']
+          ret[cf['Name']]['CompareSubcolumnsWith'] = 'org.apache.cassandra.db.marshal.' + cf['CompareSubcolumnsWith']
+        end
+        if cf['CompareWith']
+          ret[cf['Name']]['CompareWith'] = 'org.apache.cassandra.db.marshal.' + cf['CompareWith']
+        end
+        if cf['ColumnType'] == 'Super'
+          ret[cf['Name']]['Type'] = 'Super'
+        else
+          ret[cf['Name']]['Type'] = 'Standard'
+        end
+      end
+      ret
+    end
+
+    def read_storage_xml
+      @doc ||= Nokogiri::XML(open(@storage_xml))
+    end
+
+    def extract_and_validate_params_for_real(column_family, keys, args, options)
+      column_family, columns, sub_column, options = extract_and_validate_params(column_family, keys, args, options)
+      options[:start] = nil if options[:start] == ''
+      options[:finish] = nil if options[:finish] == ''
+      [column_family, to_compare_with_types(columns, column_family), to_compare_with_types(sub_column, column_family, false), options]
+    end
+
+    def to_compare_with_types(column_names, column_family, standard=true)
+      if column_names.is_a?(Array)
+        column_names.collect do |name|
+          to_compare_with_type(name, column_family, standard)
+        end
+      else
+        to_compare_with_type(column_names, column_family, standard)
+      end
+    end
+
+    def to_compare_with_type(column_name, column_family, standard=true)
+      return column_name if column_name.nil?
+      klass = if standard
+        schema[column_family.to_s]["CompareWith"]
+      else
+        schema[column_family.to_s]["CompareSubcolumnsWith"]
+      end
+
+      case klass
+      when "org.apache.cassandra.db.marshal.UTF8Type", "org.apache.cassandra.db.marshal.BytesType"
+        column_name
+      when "org.apache.cassandra.db.marshal.TimeUUIDType"
+        SimpleUUID::UUID.new(column_name)
+      when "org.apache.cassandra.db.marshal.LongType"
+        Long.new(column_name)
+      else
+        raise "Unknown column family type: #{klass.inspect}"
+      end
+    end
+
+    def column_family_type(column_family)
+      schema[column_family.to_s]['Type']
+    end
+
+    def cf(column_family)
+      @data[column_family.to_sym] ||= OrderedHash.new
+    end
+
+    def merge_and_sort(old_stuff, new_stuff)
+      if new_stuff.is_a?(Array)
+        new_stuff = new_stuff.inject({}){|h,k| h[k] = nil; h }
+      end
+      OrderedHash[old_stuff.merge(new_stuff).sort{|a,b| a[0] <=> b[0]}]
+    end
+
+    def apply_count(row, count, reversed=false)
+      if count
+        keys = row.keys.sort
+        keys = keys.reverse if reversed
+        keys = keys[0...count]
+        keys.inject(OrderedHash.new) do |memo, key|
+          memo[key] = row[key]
+          memo
+        end
+      else
+        row
+      end
+    end
+
+    def apply_range(row, column_family, strt, fin, standard=true)
+      start  = to_compare_with_type(strt, column_family, standard)
+      finish = to_compare_with_type(fin,  column_family, standard)
+      ret = OrderedHash.new
+      row.keys.each do |key|
+        if (start.nil? || key >= start) && (finish.nil? || key <= finish)
+          ret[key] = row[key]
+        end
+      end
+      ret
+    end
+
+  end
+end

data/lib/cassandra/ordered_hash.rb

@@ -0,0 +1,200 @@
+# OrderedHash is namespaced to prevent conflicts with other implementations
+class Cassandra
+  # Hash is ordered in Ruby 1.9!
+  if RUBY_VERSION >= '1.9'
+    OrderedHashInt = ::Hash
+  else
+    class OrderedHashInt < Hash #:nodoc:
+      def initialize(*args, &block)
+        super
+        @keys = []
+      end
+
+      def self.[](*args)
+        ordered_hash = new
+
+        if (args.length == 1 && args.first.is_a?(Array))
+          args.first.each do |key_value_pair|
+            next unless (key_value_pair.is_a?(Array))
+            ordered_hash[key_value_pair[0]] = key_value_pair[1]
+          end
+
+          return ordered_hash
+        end
+
+        unless (args.size % 2 == 0)
+          raise ArgumentError.new("odd number of arguments for Hash")
+        end
+
+        args.each_with_index do |val, ind|
+          next if (ind % 2 != 0)
+          ordered_hash[val] = args[ind + 1]
+        end
+
+        ordered_hash
+      end
+
+      def initialize_copy(other)
+        super
+        # make a deep copy of keys
+        @keys = other.keys
+      end
+
+      def []=(key, value)
+        @keys << key if !has_key?(key)
+        super
+      end
+
+      def delete(key)
+        if has_key? key
+          index = @keys.index(key)
+          @keys.delete_at index
+        end
+        super
+      end
+
+      def delete_if
+        super
+        sync_keys!
+        self
+      end
+
+      def reject!
+        super
+        sync_keys!
+        self
+      end
+
+      def reject(&block)
+        dup.reject!(&block)
+      end
+
+      def keys
+        @keys.dup
+      end
+
+      def values
+        @keys.collect { |key| self[key] }
+      end
+
+      def to_hash
+        self
+      end
+
+      def to_a
+        @keys.map { |key| [ key, self[key] ] }
+      end
+
+      def each_key
+        @keys.each { |key| yield key }
+      end
+
+      def each_value
+        @keys.each { |key| yield self[key]}
+      end
+
+      def each
+        @keys.each {|key| yield [key, self[key]]}
+      end
+
+      alias_method :each_pair, :each
+
+      def clear
+        super
+        @keys.clear
+        self
+      end
+
+      def shift
+        k = @keys.first
+        v = delete(k)
+        [k, v]
+      end
+
+      def merge!(other_hash)
+        other_hash.each {|k,v| self[k] = v }
+        self
+      end
+
+      def merge(other_hash)
+        dup.merge!(other_hash)
+      end
+
+      # When replacing with another hash, the initial order of our keys must come from the other hash -ordered or not.
+      def replace(other)
+        super
+        @keys = other.keys
+        self
+      end
+
+    private
+
+      def sync_keys!
+        @keys.delete_if {|k| !has_key?(k)}
+      end
+    end
+  end
+
+  class OrderedHash < OrderedHashInt #:nodoc:
+    def initialize(*args, &block)
+      @timestamps = Hash.new
+      super
+    end
+
+    def initialize_copy(other)
+      @timestamps = other.timestamps
+      super
+    end
+
+    def []=(key, value, timestamp = nil)
+      @timestamps[key] = timestamp
+      super(key, value)
+    end
+
+    def delete(key)
+      @timestamps.delete(key)
+      super
+    end
+
+    def delete_if
+      @timestamps.delete_if
+      super
+    end
+
+    def reject!
+      @timestamps.reject!
+      super
+    end
+
+    def timestamps
+      @timestamps.dup
+    end
+
+    def clear
+      @timestamps.clear
+      super
+    end
+
+    def shift
+      k, v = super
+      @timestamps.delete(k)
+      [k, v]
+    end
+
+    def replace(other)
+      @timestamps = other.timestamps
+      super
+    end
+
+    def inspect
+      "#<OrderedHash #{super}--TimeStamps: #{@timestamps.inspect}>"
+    end
+
+  private
+
+    def sync_keys!
+      @timestamps.delete_if {|k,v| !has_key?(k)}
+      super
+    end
+  end
+end

data/lib/cassandra/time.rb

@@ -0,0 +1,11 @@
+
+class Time
+  def self.stamp
+    Time.now.utc.stamp
+  end
+  
+  def stamp
+    to_i * 1_000_000 + usec
+  end
+end
+        

data/lib/cassandra.rb

@@ -0,0 +1,39 @@
+require 'rubygems'
+gem 'thrift', '~> 0.5.0'
+require 'thrift'
+gem 'thrift_client', '~> 0.6.0'
+require 'thrift_client'
+gem 'simple_uuid' , '~> 0.1.0'
+require 'simple_uuid'
+
+require 'json' unless defined?(JSON)
+
+here = File.expand_path(File.dirname(__FILE__))
+
+class Cassandra ; end
+unless Cassandra.respond_to?(:VERSION)
+  require "#{here}/cassandra/0.7"
+end
+
+$LOAD_PATH << "#{here}/../vendor/#{Cassandra.VERSION}/gen-rb"
+require "#{here}/../vendor/#{Cassandra.VERSION}/gen-rb/cassandra"
+
+$LOAD_PATH << "#{here}"
+
+require 'cassandra/helpers'
+require 'cassandra/array'
+require 'cassandra/time'
+require 'cassandra/comparable'
+require 'cassandra/long'
+require 'cassandra/ordered_hash'
+require 'cassandra/columns'
+require "cassandra/#{Cassandra.VERSION}/columns"
+require "cassandra/#{Cassandra.VERSION}/protocol"
+require "cassandra/cassandra"
+require "cassandra/#{Cassandra.VERSION}/cassandra"
+unless Cassandra.VERSION.eql?("0.6")
+  require "cassandra/#{Cassandra.VERSION}/column_family"
+  require "cassandra/#{Cassandra.VERSION}/keyspace"
+end
+require 'cassandra/constants'
+require 'cassandra/debug' if ENV['DEBUG']

data/test/cassandra_client_test.rb

@@ -0,0 +1,20 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+class CassandraClientTest < Test::Unit::TestCase
+  include Cassandra::Constants
+  
+  def setup
+    @twitter = Cassandra.new('Twitter', "127.0.0.1:9160", :retries => 2, :exception_classes => [])
+  end
+  
+  def test_client_method_is_called
+    assert_nil @twitter.instance_variable_get(:@client)
+    @twitter.insert(:Statuses, key, {'1' => 'v', '2' => 'v', '3' => 'v'})
+    assert_not_nil @twitter.instance_variable_get(:@client)
+  end
+  
+  def key
+    caller.first[/`(.*?)'/, 1]
+  end
+  
+end

data/test/cassandra_mock_test.rb

@@ -0,0 +1,73 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+require 'cassandra_test'
+require 'cassandra/mock'
+
+class CassandraMockTest < CassandraTest
+  include Cassandra::Constants
+
+  def setup
+    storage_xml_path = File.expand_path(File.join(File.dirname(File.dirname(__FILE__)), 'conf', 'storage-conf.xml'))
+    @twitter = Cassandra::Mock.new('Twitter', storage_xml_path)
+    @twitter.clear_keyspace!
+
+    @blogs = Cassandra::Mock.new('Multiblog', storage_xml_path)
+    @blogs.clear_keyspace!
+
+    @blogs_long = Cassandra::Mock.new('MultiblogLong', storage_xml_path)
+    @blogs_long.clear_keyspace!
+
+    @uuids = (0..6).map {|i| SimpleUUID::UUID.new(Time.at(2**(24+i))) }
+    @longs = (0..6).map {|i| Long.new(Time.at(2**(24+i))) }
+  end
+
+  def test_setup
+    assert @twitter
+    assert @blogs
+    assert @blogs_long
+  end
+  
+  def test_schema_for_keyspace
+    data = {
+      "StatusRelationships"=>{
+          "CompareSubcolumnsWith"=>"org.apache.cassandra.db.marshal.TimeUUIDType", 
+          "CompareWith"=>"org.apache.cassandra.db.marshal.UTF8Type",
+          "Type"=>"Super"},
+      "StatusAudits"=>{
+        "CompareWith"=>"org.apache.cassandra.db.marshal.UTF8Type",
+        "Type"=>"Standard"}, 
+      "Statuses"=>{
+        "CompareWith"=>"org.apache.cassandra.db.marshal.UTF8Type",
+        "Type"=>"Standard"}, 
+      "UserRelationships"=>{
+        "CompareSubcolumnsWith"=>"org.apache.cassandra.db.marshal.TimeUUIDType", 
+        "CompareWith"=>"org.apache.cassandra.db.marshal.UTF8Type",
+        "Type"=>"Super"}, 
+      "UserAudits"=>{
+        "CompareWith"=>"org.apache.cassandra.db.marshal.UTF8Type", 
+        "Type"=>"Standard"},
+      "Users"=>{"CompareWith"=>"org.apache.cassandra.db.marshal.UTF8Type", "Type"=>"Standard"},
+      "TimelinishThings"=>
+        {"CompareWith"=>"org.apache.cassandra.db.marshal.BytesType", "Type"=>"Standard"}
+    }
+    stuff = @twitter.send(:schema_for_keyspace, 'Twitter')
+    data.keys.each do |k|
+      assert_equal data[k], stuff[k], k
+    end
+  end
+
+  def test_sorting_row_keys
+    @twitter.insert(:Statuses, 'b', {:text => 'foo'})
+    @twitter.insert(:Statuses, 'a', {:text => 'foo'})
+    assert_equal ['a'], @twitter.get_range(:Statuses, :count => 1)
+  end
+
+  def test_inserting_array_for_indices
+    @twitter.insert(:TimelinishThings, 'a', ['1','2'])
+    row = @twitter.get(:TimelinishThings, 'a')
+    assert_equal({'1' => nil, '2' => nil}, row)
+
+    assert_raises(ArgumentError) {
+      @twitter.insert(:UserRelationships, 'a', ['u1','u2'])
+    }
+  end
+end