cassandra 0.8.2 → 0.9.0

data/lib/cassandra/{protocol.rb → 0.6/protocol.rb}

@@ -74,7 +74,7 @@ class Cassandra
     def _get_range(column_family, start, finish, count, consistency)
       column_parent = CassandraThrift::ColumnParent.new(:column_family => column_family)
       predicate = CassandraThrift::SlicePredicate.new(:slice_range => CassandraThrift::SliceRange.new(:start => '', :finish => ''))
-      range = CassandraThrift::KeyRange.new(:start_key => start, :end_key => finish)
+      range = CassandraThrift::KeyRange.new(:start_key => start, :end_key => finish, :count => count)
       client.get_range_slices(@keyspace, column_parent, predicate, range, 1)
     end
     
@@ -89,4 +89,4 @@ class Cassandra
       client.get_range_slices(@keyspace, column_parent, predicate, range, 1).each{|i| yield i.key }
     end
   end
-end
+end

data/lib/cassandra/0.7.rb

@@ -0,0 +1,7 @@
+class Cassandra
+  def self.VERSION
+    "0.7"
+  end
+end
+
+require "#{File.expand_path(File.dirname(__FILE__))}/../cassandra"

data/lib/cassandra/0.7/cassandra.rb

@@ -0,0 +1,195 @@
+class Cassandra
+  def self.DEFAULT_TRANSPORT_WRAPPER
+    Thrift::FramedTransport
+  end
+
+  def login!(username, password)
+    @auth_request = CassandraThrift::AuthenticationRequest.new
+    @auth_request.credentials = {'username' => username, 'password' => password}
+    client.login(@auth_request)
+  end
+
+  def inspect
+    "#<Cassandra:#{object_id}, @keyspace=#{keyspace.inspect}, @schema={#{
+      Array(schema(false).cf_defs).map {|cfdef| ":#{cfdef.name} => #{cfdef.column_type}"}.join(', ')
+    }}, @servers=#{servers.inspect}>"
+  end
+
+  def keyspace=(ks)
+    client.set_keyspace(ks) if check_keyspace(ks)
+    @schema = nil; @keyspace = ks
+  end
+
+  def keyspaces
+    client.describe_keyspaces.to_a.collect {|ksdef| ksdef.name }
+  end
+
+  def schema(load=true)
+    if !load && !@schema
+      Cassandra::Keyspace.new
+    else
+      @schema ||= client.describe_keyspace(@keyspace)
+    end
+  end
+
+  def schema_agreement?
+    client.describe_schema_versions().length == 1
+  end
+
+  def version
+    client.describe_version()
+  end
+
+  def cluster_name
+    @cluster_name ||= client.describe_cluster_name()
+  end
+
+  def ring
+    client.describe_ring(@keyspace)
+  end
+
+  def partitioner
+    client.describe_partitioner()
+  end
+
+  ## Delete
+
+  # Remove all rows in the column family you request.
+  def truncate!(column_family)
+    #each_key(column_family) do |key|
+    #  remove(column_family, key, options)
+    #end
+    client.truncate(column_family)
+  end
+
+  # Remove all rows in the keyspace.
+  def clear_keyspace!
+    schema.cf_defs.each { |cfdef| truncate!(cfdef.name) }
+  end
+
+### Read
+
+  def add_column_family(cf_def)
+    begin
+      res = client.system_add_column_family(cf_def)
+    rescue CassandraThrift::TimedOutException => te
+      puts "Timed out: #{te.inspect}"
+    end
+    @schema = nil
+    res
+  end
+
+  def drop_column_family(cf_name)
+    begin
+      res = client.system_drop_column_family(cf_name)
+    rescue CassandraThrift::TimedOutException => te
+      puts "Timed out: #{te.inspect}"
+    end
+    @schema = nil
+    res
+  end
+
+  def rename_column_family(old_name, new_name)
+    begin
+      res = client.system_rename_column_family(old_name, new_name)
+    rescue CassandraThrift::TimedOutException => te
+      puts "Timed out: #{te.inspect}"
+    end
+    @schema = nil
+    res
+  end
+
+  def add_keyspace(ks_def)
+    begin
+      res = client.system_add_keyspace(ks_def)
+    rescue CassandraThrift::TimedOutException => toe
+      puts "Timed out: #{toe.inspect}"
+    rescue Thrift::TransportException => te
+      puts "Timed out: #{te.inspect}"
+    end
+    @keyspaces = nil
+    res
+  end
+
+  def drop_keyspace(ks_name)
+    begin
+      res = client.system_drop_keyspace(ks_name)
+    rescue CassandraThrift::TimedOutException => toe
+      puts "Timed out: #{toe.inspect}"
+    rescue Thrift::TransportException => te
+      puts "Timed out: #{te.inspect}"
+    end
+    keyspace = "system" if ks_name.eql?(@keyspace)
+    @keyspaces = nil
+    res
+  end
+
+  def rename_keyspace(old_name, new_name)
+    begin
+      res = client.system_rename_keyspace(old_name, new_name)
+    rescue CassandraThrift::TimedOutException => toe
+      puts "Timed out: #{toe.inspect}"
+    rescue Thrift::TransportException => te
+      puts "Timed out: #{te.inspect}"
+    end
+    keyspace = new_name if old_name.eql?(@keyspace)
+    @keyspaces = nil
+    res
+  end
+
+  # Open a batch operation and yield self. Inserts and deletes will be queued
+  # until the block closes, and then sent atomically to the server.  Supports
+  # the <tt>:consistency</tt> option, which overrides the consistency set in
+  # the individual commands.
+  def batch(options = {})
+    _, _, _, options =
+      extract_and_validate_params(schema.cf_defs.first.name, "", [options], WRITE_DEFAULTS)
+
+    @batch = []
+    yield(self)
+    compact_mutations!
+
+    @batch.each do |mutation|
+      case mutation.first
+      when :remove
+        _remove(*mutation[1])
+      else
+        _mutate(*mutation)
+      end
+    end
+  ensure
+    @batch = nil
+  end
+
+  protected
+
+  def client
+    if @client.nil? || @client.current_server.nil?
+      reconnect!
+      @client.set_keyspace(@keyspace) if check_keyspace
+    end
+    @client
+  end
+
+  def reconnect!
+    @servers = all_nodes
+    @client = new_client
+  end
+
+  def check_keyspace(ks = @keyspace)
+    !(unless (_keyspaces = keyspaces()).include?(ks)
+      raise AccessError, "Keyspace #{ks.inspect} not found. Available: #{_keyspaces.inspect}"
+    end)
+  end
+
+  def all_nodes
+    if @auto_discover_nodes && !@keyspace.eql?("system")
+      ips = (new_client.describe_ring(@keyspace).map {|range| range.endpoints}).flatten.uniq
+      port = @servers.first.split(':').last
+      ips.map{|ip| "#{ip}:#{port}" }
+    else
+      @servers
+    end
+  end
+
+end

data/lib/cassandra/0.7/column_family.rb

@@ -0,0 +1,3 @@
+class Cassandra
+  class ColumnFamily < CassandraThrift::CfDef ; end
+end

data/lib/cassandra/0.7/columns.rb

@@ -0,0 +1,59 @@
+
+class Cassandra
+  # A bunch of crap, mostly related to introspecting on column types
+  module Columns #:nodoc:
+
+    def is_super(column_family)
+      @is_super[column_family] ||= column_family_property(column_family, 'column_type') == "Super"
+    end
+
+    def column_name_class(column_family)
+      @column_name_class[column_family] ||= column_name_class_for_key(column_family, "comparator_type")
+    end
+
+    def sub_column_name_class(column_family)
+      @sub_column_name_class[column_family] ||= column_name_class_for_key(column_family, "subcomparator_type")
+    end
+
+    def column_family_property(column_family, key)
+      cfdef = schema.cf_defs.find {|cfdef| cfdef.name == column_family }
+      unless cfdef
+        raise AccessError, "Invalid column family \"#{column_family}\""
+      end
+      cfdef.send(key)
+    end
+
+    private
+
+    def _standard_insert_mutation(column_family, column_name, value, timestamp, ttl = nil)
+      CassandraThrift::Mutation.new(
+        :column_or_supercolumn => CassandraThrift::ColumnOrSuperColumn.new(
+          :column => CassandraThrift::Column.new(
+            :name      => column_name_class(column_family).new(column_name).to_s,
+            :value     => value,
+            :timestamp => timestamp,
+            :ttl       => ttl
+          )
+        )
+      )
+    end
+
+    def _super_insert_mutation(column_family, super_column_name, sub_columns, timestamp, ttl = nil)
+      CassandraThrift::Mutation.new(:column_or_supercolumn => 
+        CassandraThrift::ColumnOrSuperColumn.new(
+          :super_column => CassandraThrift::SuperColumn.new(
+            :name => column_name_class(column_family).new(super_column_name).to_s,
+            :columns => sub_columns.collect { |sub_column_name, sub_column_value|
+              CassandraThrift::Column.new(
+                :name      => sub_column_name_class(column_family).new(sub_column_name).to_s,
+                :value     => sub_column_value.to_s,
+                :timestamp => timestamp,
+                :ttl       => ttl
+              )
+            }
+          )
+        )
+      )
+    end
+  end
+end

data/lib/cassandra/0.7/keyspace.rb

@@ -0,0 +1,3 @@
+class Cassandra
+  class Keyspace < CassandraThrift::KsDef ; end
+end

data/lib/cassandra/0.7/protocol.rb

@@ -0,0 +1,95 @@
+
+class Cassandra
+  # Inner methods for actually doing the Thrift calls
+  module Protocol #:nodoc:
+    private
+
+    def _mutate(mutation_map, consistency_level)
+      client.batch_mutate(mutation_map, consistency_level)
+    end
+
+    def _remove(key, column_path, timestamp, consistency_level)
+      client.remove(key, column_path, timestamp, consistency_level)
+    end
+
+    def _count_columns(column_family, key, super_column, consistency)
+      client.get_count(key,
+        CassandraThrift::ColumnParent.new(:column_family => column_family, :super_column => super_column),
+        consistency
+      )
+    end
+
+    def _get_columns(column_family, key, columns, sub_columns, consistency)
+      result = if is_super(column_family)
+        if sub_columns
+          columns_to_hash(column_family, client.get_slice(key,
+            CassandraThrift::ColumnParent.new(:column_family => column_family, :super_column => columns),
+            CassandraThrift::SlicePredicate.new(:column_names => sub_columns),
+            consistency))
+        else
+          columns_to_hash(column_family, client.get_slice(key,
+            CassandraThrift::ColumnParent.new(:column_family => column_family),
+            CassandraThrift::SlicePredicate.new(:column_names => columns),
+            consistency))
+        end
+      else
+        columns_to_hash(column_family, client.get_slice(key,
+          CassandraThrift::ColumnParent.new(:column_family => column_family),
+          CassandraThrift::SlicePredicate.new(:column_names => columns),
+          consistency))
+      end
+
+      klass = column_name_class(column_family)
+      (sub_columns || columns).map { |name| result[klass.new(name)] }
+    end
+
+    def _multiget(column_family, keys, column, sub_column, count, start, finish, reversed, consistency)
+      # Single values; count and range parameters have no effect
+      if is_super(column_family) and sub_column
+        predicate = CassandraThrift::SlicePredicate.new(:column_names => [column])
+        column_parent = CassandraThrift::ColumnParent.new(:column_family => column_family, :super_column => column)
+        multi_sub_columns_to_hash!(column_family, client.multiget_slice(keys, column_parent, predicate, consistency))
+
+      elsif !is_super(column_family) and column
+        predicate = CassandraThrift::SlicePredicate.new(:column_names => [column])
+        column_parent = CassandraThrift::ColumnParent.new(:column_family => column_family)
+        multi_columns_to_hash!(column_family, client.multiget_slice(keys, column_parent, predicate, consistency))
+
+      # Slices
+      else
+        predicate = CassandraThrift::SlicePredicate.new(:slice_range =>
+          CassandraThrift::SliceRange.new(
+            :reversed => reversed,
+            :count => count,
+            :start => start,
+            :finish => finish))
+
+        if is_super(column_family) and column
+          column_parent = CassandraThrift::ColumnParent.new(:column_family => column_family, :super_column => column)
+          multi_sub_columns_to_hash!(column_family, client.multiget_slice(keys, column_parent, predicate, consistency))
+        else
+          column_parent = CassandraThrift::ColumnParent.new(:column_family => column_family)
+          multi_columns_to_hash!(column_family, client.multiget_slice(keys, column_parent, predicate, consistency))
+        end
+      end
+    end
+
+    def _get_range(column_family, start, finish, count, consistency)
+      column_parent = CassandraThrift::ColumnParent.new(:column_family => column_family)
+      predicate = CassandraThrift::SlicePredicate.new(:slice_range => CassandraThrift::SliceRange.new(:start => '', :finish => ''))
+      range = CassandraThrift::KeyRange.new(:start_key => start, :end_key => finish, :count => count)
+      client.get_range_slices(column_parent, predicate, range, 1)
+    end
+
+    def _get_range_keys(column_family, start, finish, count, consistency)
+      _get_range(column_family, start, finish, count, consistency).collect{|i| i.key }
+    end
+
+    def each_key(column_family)
+      column_parent = CassandraThrift::ColumnParent.new(:column_family => column_family.to_s)
+      predicate = CassandraThrift::SlicePredicate.new(:column_names => [])
+      range = CassandraThrift::KeyRange.new(:start_key => '', :end_key => '')
+      client.get_range_slices(column_parent, predicate, range, 1).each{|i| yield i.key }
+    end
+  end
+end

data/lib/cassandra/cassandra.rb

@@ -3,6 +3,10 @@
 Create a new Cassandra client instance. Accepts a keyspace name, and optional host and port.
 
   client = Cassandra.new('twitter', '127.0.0.1:9160')
+  
+If the server requires authentication, you must authenticate before make calls
+
+  client.login!('username','password')
 
 You can then make calls to the server via the <tt>client</tt> instance.
 
@@ -43,7 +47,8 @@ class Cassandra
   WRITE_DEFAULTS = {
     :count => 1000,
     :timestamp => nil,
-    :consistency => Consistency::ONE
+    :consistency => Consistency::ONE,
+    :ttl => nil
   }.freeze
 
   READ_DEFAULTS = {
@@ -59,7 +64,7 @@ class Cassandra
     :thrift_client_class => ThriftClient
   }.freeze
 
-  attr_reader :keyspace, :servers, :schema, :thrift_client_options, :thrift_client_class
+  attr_reader :keyspace, :servers, :schema, :thrift_client_options, :thrift_client_class, :auth_request
 
   # Create a new Cassandra instance and open the connection.
   def initialize(keyspace, servers = "127.0.0.1:9160", thrift_client_options = {})
@@ -67,6 +72,7 @@ class Cassandra
     @column_name_class = {}
     @sub_column_name_class = {}
     @auto_discover_nodes = true
+    thrift_client_options[:transport_wrapper] ||= Cassandra.DEFAULT_TRANSPORT_WRAPPER
     @thrift_client_options = THRIFT_DEFAULTS.merge(thrift_client_options)
     @thrift_client_class = @thrift_client_options[:thrift_client_class]
     @keyspace = keyspace
@@ -83,9 +89,15 @@ class Cassandra
   end
 
   def keyspaces
-    @keyspaces ||= client.get_string_list_property("keyspaces")
+    @keyspaces ||= client.describe_keyspaces()
   end
-
+  
+  def login!(username, password)
+    @auth_request = CassandraThrift::AuthenticationRequest.new
+    @auth_request.credentials = {'username' => username, 'password' => password}
+    client.login(@keyspace, @auth_request)
+  end
+  
   def inspect
     "#<Cassandra:#{object_id}, @keyspace=#{keyspace.inspect}, @schema={#{
       schema(false).map {|name, hash| ":#{name} => #{hash['type'].inspect}"}.join(', ')
@@ -95,8 +107,8 @@ class Cassandra
 ### Write
 
   # Insert a row for a key. Pass a flat hash for a regular column family, and
-  # a nested hash for a super column family. Supports the <tt>:consistency</tt>
-  # and <tt>:timestamp</tt> options.
+  # a nested hash for a super column family. Supports the <tt>:consistency</tt>,
+  # <tt>:timestamp</tt> and <tt>:ttl</tt> options.
   def insert(column_family, key, hash, options = {})
     column_family, _, _, options = extract_and_validate_params(column_family, key, [options], WRITE_DEFAULTS)
 
@@ -104,13 +116,13 @@ class Cassandra
     mutation_map = if is_super(column_family)
       {
         key => {
-          column_family => hash.collect{|k,v| _super_insert_mutation(column_family, k, v, timestamp) }
+          column_family => hash.collect{|k,v| _super_insert_mutation(column_family, k, v, timestamp, options[:ttl]) }
         }
       }
     else
       {
         key => {
-          column_family => hash.collect{|k,v| _standard_insert_mutation(column_family, k, v, timestamp)}
+          column_family => hash.collect{|k,v| _standard_insert_mutation(column_family, k, v, timestamp, options[:ttl])}
         }
       }
     end
@@ -118,7 +130,7 @@ class Cassandra
     @batch ? @batch << [mutation_map, options[:consistency]] : _mutate(mutation_map, options[:consistency])
   end
 
-  
+
   ## Delete
 
   # _mutate the element at the column_family:key:[column]:[sub_column]
@@ -130,28 +142,10 @@ class Cassandra
     args = {:column_family => column_family}
     columns = is_super(column_family) ? {:super_column => column, :column => sub_column} : {:column => column}
     column_path = CassandraThrift::ColumnPath.new(args.merge(columns))
-    
-    mutation = [:remove, [key, column_path, options[:timestamp] || Time.stamp, options[:consistency]]]
-    
-    @batch ? @batch << mutation : _remove(*mutation[1])
-  end
 
-  # Remove all rows in the column family you request. Supports options
-  # <tt>:consistency</tt> and <tt>:timestamp</tt>.
-  # FIXME May not currently delete all records without multiple calls. Waiting
-  # for ranged remove support in Cassandra.
-  def clear_column_family!(column_family, options = {})
-    each_key(column_family) do |key|
-      remove(column_family, key, options)
-    end
-  end
+    mutation = [:remove, [key, column_path, options[:timestamp] || Time.stamp, options[:consistency]]]
 
-  # Remove all rows in the keyspace. Supports options <tt>:consistency</tt> and
-  # <tt>:timestamp</tt>.
-  # FIXME May not currently delete all records without multiple calls. Waiting
-  # for ranged remove support in Cassandra.
-  def clear_keyspace!(options = {})
-    schema.keys.each { |column_family| clear_column_family!(column_family, options) }
+    @batch ? @batch << mutation : _remove(*mutation[1])
   end
 
 ### Read
@@ -213,7 +207,11 @@ class Cassandra
   def exists?(column_family, key, *columns_and_options)
     column_family, column, sub_column, options = 
       extract_and_validate_params(column_family, key, columns_and_options, READ_DEFAULTS)
-    _multiget(column_family, [key], column, sub_column, 1, nil, nil, nil, options[:consistency])[key]
+    if column
+      _multiget(column_family, [key], column, sub_column, 1, nil, nil, nil, options[:consistency])[key]
+    else
+      _multiget(column_family, [key], nil, nil, 1, '', '', false, options[:consistency])[key]
+    end
   end
 
   # Return a list of keys in the column_family you request. Requires the
@@ -233,16 +231,16 @@ class Cassandra
     get_range(column_family, options).select{|r| r.columns.length > 0}.compact.length
   end
 
-  # Open a batch operation and yield. Inserts and deletes will be queued until
-  # the block closes, and then sent atomically to the server.  Supports the
-  # <tt>:consistency</tt> option, which overrides the consistency set in
+  # Open a batch operation and yield self. Inserts and deletes will be queued
+  # until the block closes, and then sent atomically to the server.  Supports
+  # the <tt>:consistency</tt> option, which overrides the consistency set in
   # the individual commands.
   def batch(options = {})
     _, _, _, options = 
       extract_and_validate_params(schema.keys.first, "", [options], WRITE_DEFAULTS)
 
     @batch = []
-    yield
+    yield(self)
     compact_mutations!
 
     @batch.each do |mutation|
@@ -256,7 +254,7 @@ class Cassandra
   ensure
     @batch = nil
   end
-
+  
   protected
 
   def calling_method
@@ -268,43 +266,8 @@ class Cassandra
     #TODO re-do this rollup
   end
 
-  def schema(load=true)
-    if !load && !@schema
-      []
-    else
-      @schema ||= client.describe_keyspace(@keyspace)
-    end
-  end
-
-  def client
-    reconnect! if @client.nil?
-    @client
-  end
-
-  def reconnect!
-    @servers = all_nodes
-    @client = new_client
-    check_keyspace
-  end
-
-  def check_keyspace
-    unless (keyspaces = client.get_string_list_property("keyspaces")).include?(@keyspace)
-      raise AccessError, "Keyspace #{@keyspace.inspect} not found. Available: #{keyspaces.inspect}"
-    end
-  end
-
   def new_client
     thrift_client_class.new(CassandraThrift::Cassandra::Client, @servers, @thrift_client_options)
   end
-
-  def all_nodes
-    if @auto_discover_nodes
-      ips = ::JSON.parse(new_client.get_string_property('token map')).values
-      port = @servers.first.split(':').last
-      ips.map{|ip| "#{ip}:#{port}" }
-    else
-      @servers
-    end
-  end
-
-end
+  
+end