cassandra 0.15.0 → 0.16.0

data/conf/1.1/log4j-server.properties

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# for production, you should probably set pattern to %c instead of %l.
+# (%l is slower.)
+
+# output messages into a rolling log file as well as stdout
+log4j.rootLogger=INFO,stdout,R
+
+# stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%5p %d{HH:mm:ss,SSS} %m%n
+
+# rolling log file
+log4j.appender.R=org.apache.log4j.RollingFileAppender
+log4j.appender.R.maxFileSize=20MB
+log4j.appender.R.maxBackupIndex=50
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=%5p [%t] %d{ISO8601} %F (line %L) %m%n
+# Edit the next line to point to your logs directory
+log4j.appender.R.File=/var/log/cassandra/system.log
+
+# Application logging options
+#log4j.logger.org.apache.cassandra=DEBUG
+#log4j.logger.org.apache.cassandra.db=DEBUG
+#log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG
+
+# Adding this to avoid thrift logging disconnect errors.
+log4j.logger.org.apache.thrift.server.TNonblockingServer=ERROR
+

data/conf/1.1/schema.json

@@ -0,0 +1,72 @@
+{"Twitter":{
+    "Users":{
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Standard"},
+    "UserAudits":{
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Standard"},
+    "UserCounters":{
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Standard",
+      "default_validation_class":"CounterColumnType"},
+    "UserCounterAggregates":{
+      "subcomparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Super",
+      "default_validation_class":"CounterColumnType"},
+    "UserRelationships":{
+      "subcomparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Super"},
+    "Usernames":{
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Standard"},
+    "Statuses":{
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Standard"},
+    "StatusAudits":{
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Standard"},
+    "StatusRelationships":{
+      "subcomparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Super"},
+    "Indexes":{
+      "comparator_type":"org.apache.cassandra.db.marshal.UTF8Type",
+      "column_type":"Super"},
+    "TimelinishThings":{
+      "comparator_type":"org.apache.cassandra.db.marshal.BytesType",
+      "column_type":"Standard"}
+  },
+"Multiblog":{
+    "Blogs":{
+      "comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
+      "column_type":"Standard"},
+    "Comments":{
+      "comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
+      "column_type":"Standard"}
+  },
+"MultiblogLong":{
+    "Blogs":{
+      "comparator_type":"org.apache.cassandra.db.marshal.LongType",
+      "column_type":"Standard"},
+    "Comments":{
+      "comparator_type":"org.apache.cassandra.db.marshal.LongType",
+      "column_type":"Standard"}
+  },
+"TypeConversions":{
+    "UUIDColumnConversion":{
+      "comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
+      "column_type":"Standard"},
+    "SuperUUID":{
+      "subcomparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
+      "comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
+      "column_type":"Super"},
+    "CompositeColumnConversion":{
+      "comparator_type":"org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.UTF8Type)",
+      "column_type":"Standard"},
+    "DynamicComposite":{
+      "comparator_type":"org.apache.cassandra.db.marshal.DynamicCompositeType(u=>org.apache.cassandra.db.marshal.UUIDType,t=>org.apache.cassandra.db.marshal.TimeUUIDType,s=>org.apache.cassandra.db.marshal.UTF8Type,b=>org.apache.cassandra.db.marshal.BytesType,a=>org.apache.cassandra.db.marshal.AsciiType,l=>org.apache.cassandra.db.marshal.LongType,x=>org.apache.cassandra.db.marshal.LexicalUUIDType,i=>org.apache.cassandra.db.marshal.IntegerType)",
+      "column_type":"Standard"}
+  }
+}

data/conf/1.1/schema.txt

@@ -0,0 +1,57 @@
+create keyspace Twitter with
+  placement_strategy = 'org.apache.cassandra.locator.SimpleStrategy' AND
+  strategy_options = {replication_factor:1};
+use Twitter;
+create column family Users with comparator = 'UTF8Type';
+create column family UserAudits with comparator = 'UTF8Type';
+create column family UserCounters with comparator = 'UTF8Type' and
+  default_validation_class = CounterColumnType;
+create column family UserCounterAggregates with column_type = 'Super'
+  and comparator = 'UTF8Type' and
+  subcomparator = 'UTF8Type' and
+  default_validation_class = CounterColumnType;
+create column family UserRelationships with
+  comparator = 'UTF8Type' and
+  column_type = 'Super' and
+  subcomparator = 'TimeUUIDType';
+create column family Usernames with comparator = 'UTF8Type';
+create column family Statuses
+  with comparator = 'UTF8Type'
+  and column_metadata = [
+    {column_name: 'tags', validation_class: 'BytesType', index_type: 'KEYS'}
+  ];
+create column family StatusAudits with comparator = 'UTF8Type';
+create column family StatusRelationships with
+  comparator = 'UTF8Type' and
+  column_type = 'Super' and
+  subcomparator = 'TimeUUIDType';
+create column family Indexes with
+  comparator = 'UTF8Type' and
+  column_type = 'Super';
+create column family TimelinishThings with
+  comparator = 'BytesType';
+
+create keyspace Multiblog with
+  placement_strategy = 'org.apache.cassandra.locator.SimpleStrategy' AND
+  strategy_options = {replication_factor:1};
+use Multiblog;
+create column family Blogs with comparator = 'TimeUUIDType';
+create column family Comments with comparator = 'TimeUUIDType';
+
+
+create keyspace MultiblogLong with
+  placement_strategy = 'org.apache.cassandra.locator.SimpleStrategy' AND
+  strategy_options = {replication_factor:1};
+use MultiblogLong;
+create column family Blogs with comparator = 'LongType';
+create column family Comments with comparator = 'LongType';
+
+create keyspace TypeConversions with
+  placement_strategy = 'org.apache.cassandra.locator.SimpleStrategy' AND
+  strategy_options = {replication_factor:1};
+use TypeConversions;
+create column family UUIDColumnConversion with comparator = TimeUUIDType;
+create column family SuperUUID with comparator = TimeUUIDType and column_type = Super;
+create column family CompositeColumnConversion with comparator = 'CompositeType(IntegerType, UTF8Type)';
+create column family DynamicComposite with comparator ='DynamicCompositeType
+   (a=>AsciiType,b=>BytesType,i=>IntegerType,x=>LexicalUUIDType,l=>LongType,t=>TimeUUIDType,s=>UTF8Type,u=>UUIDType)';

data/lib/cassandra.rb

@@ -28,6 +28,7 @@ require 'cassandra/dynamic_composite'
 require 'cassandra/ordered_hash'
 require 'cassandra/columns'
 require 'cassandra/protocol'
+require 'cassandra/batch'
 require "cassandra/#{Cassandra.VERSION}/columns"
 require "cassandra/#{Cassandra.VERSION}/protocol"
 require "cassandra/cassandra"

data/lib/cassandra/1.0/cassandra.rb

@@ -1,25 +1 @@
-class Cassandra
-
-  ## Counters
-
-  # Add a value to the counter in cf:key:super column:column
-  def add(column_family, key, value, *columns_and_options)
-    column_family, column, sub_column, options = extract_and_validate_params(column_family, key, columns_and_options, WRITE_DEFAULTS)
-
-    mutation_map = if is_super(column_family)
-      {
-        key => {
-          column_family => [_super_counter_mutation(column_family, column, sub_column, value)]
-        }
-      }
-    else
-      {
-        key => {
-          column_family => [_standard_counter_mutation(column_family, column, value)]
-        }
-      }
-    end
-
-    @batch ? @batch << [mutation_map, options[:consistency]] : _mutate(mutation_map, options[:consistency])
-  end
-end
+require "#{File.expand_path(File.dirname(__FILE__))}/../0.8/cassandra"

data/lib/cassandra/1.0/columns.rb

@@ -1,28 +1 @@
-class Cassandra
-  module Columns #:nodoc:
-    def _standard_counter_mutation(column_family, column_name, value)
-      CassandraThrift::Mutation.new(
-        :column_or_supercolumn => CassandraThrift::ColumnOrSuperColumn.new(
-          :counter_column => CassandraThrift::CounterColumn.new(
-            :name      => column_name_class(column_family).new(column_name).to_s,
-            :value     => value
-          )
-        )
-      )
-    end
-
-    def _super_counter_mutation(column_family, super_column_name, sub_column, value)
-      CassandraThrift::Mutation.new(:column_or_supercolumn =>
-        CassandraThrift::ColumnOrSuperColumn.new(
-          :counter_super_column => CassandraThrift::SuperColumn.new(
-            :name => column_name_class(column_family).new(super_column_name).to_s,
-            :columns => [CassandraThrift::CounterColumn.new(
-              :name      => sub_column_name_class(column_family).new(sub_column).to_s,
-              :value     => value
-            )]
-          )
-        )
-      )
-    end
-  end
-end
+require "#{File.expand_path(File.dirname(__FILE__))}/../0.8/columns"

data/lib/cassandra/1.0/protocol.rb

@@ -1,12 +1 @@
-require "#{File.expand_path(File.dirname(__FILE__))}/../0.7/protocol"
-
-class Cassandra
-  # Inner methods for actually doing the Thrift calls
-  module Protocol #:nodoc:
-    private
-
-    def _remove_counter(key, column_path, consistency_level)
-      client.remove_counter(key, column_path, consistency_level)
-    end
-  end
-end
+require "#{File.expand_path(File.dirname(__FILE__))}/../0.8/protocol"

data/lib/cassandra/1.1.rb

@@ -0,0 +1,7 @@
+class Cassandra
+  def self.VERSION
+    "1.1"
+  end
+end
+
+require "#{File.expand_path(File.dirname(__FILE__))}/../cassandra"

data/lib/cassandra/1.1/cassandra.rb

@@ -0,0 +1 @@
+require "#{File.expand_path(File.dirname(__FILE__))}/../1.0/cassandra"

data/lib/cassandra/1.1/columns.rb

@@ -0,0 +1 @@
+require "#{File.expand_path(File.dirname(__FILE__))}/../1.0/columns"

data/lib/cassandra/1.1/protocol.rb

@@ -0,0 +1 @@
+require "#{File.expand_path(File.dirname(__FILE__))}/../1.0/protocol"

data/lib/cassandra/batch.rb

@@ -0,0 +1,41 @@
+class Cassandra
+  class Batch
+    include Enumerable
+
+    def initialize(cassandra, options) 
+      @queue_size = options.delete(:queue_size) || 0
+      @cassandra = cassandra
+      @options = options
+      @batch_queue = []
+    end
+
+    ##
+    # Append mutation to the batch queue
+    # Flush the batch queue if full
+    #
+    def <<(mutation)
+      @batch_queue << mutation
+      if @queue_size > 0 and @batch_queue.length >= @queue_size
+        begin
+          @cassandra.flush_batch(@options)
+        ensure
+          @batch_queue = []
+        end
+      end
+    end
+
+    ##
+    # Implement each method (required by Enumerable)
+    #
+    def each(&block)
+      @batch_queue.each(&block)
+    end
+
+    ##
+    # Queue size
+    #
+    def length
+      @batch_queue.length
+    end
+  end
+end

data/lib/cassandra/cassandra.rb

@@ -839,19 +839,31 @@ class Cassandra
 
   ##
   # Open a batch operation and yield self. Inserts and deletes will be queued
-  # until the block closes, and then sent atomically to the server.
+  # until the block closes or the queue is full(if option :queue_size is set),
+  # and then sent atomically to the server.
   #
   # Supports the :consistency option, which overrides the consistency set in
   # the individual commands.
   #
   def batch(options = {})
+    @batch = Cassandra::Batch.new(self, options)
+
     _, _, _, options =
       extract_and_validate_params(schema.cf_defs.first.name, "", [options], WRITE_DEFAULTS)
 
-      @batch = []
-      yield(self)
-      compacted_map,seen_clevels = compact_mutations!
-      clevel = if options[:consistency] != nil # Override any clevel from individual mutations if
+    yield(self)
+    flush_batch(options)
+  ensure
+    @batch = nil
+  end
+
+  ##
+  # Send the batch queue to the server
+  #
+  def flush_batch(options)
+    compacted_map,seen_clevels = compact_mutations!
+
+    clevel = if options[:consistency] != nil # Override any clevel from individual mutations if
                  options[:consistency]
                elsif seen_clevels.length > 1 # Cannot choose which CLevel to use if there are several ones
                  raise "Multiple consistency levels used in the batch, and no override...cannot pick one"
@@ -859,11 +871,9 @@ class Cassandra
                  seen_clevels.first
                end
 
-      _mutate(compacted_map,clevel)
-  ensure
-    @batch = nil
+    _mutate(compacted_map,clevel)
   end
-
+  
   ##
   # Create secondary index.
   #

data/lib/cassandra/mock.rb

@@ -74,16 +74,21 @@ class Cassandra
       end
     end
 
-    def batch
-      @batch = []
+    def batch(options={})
+      @batch = Cassandra::Batch.new(self, options)
       yield
+      flush_batch(options)
+    ensure
+      @batch = nil
+    end
+
+    def flush_batch(options)
       b = @batch
       @batch = nil
       b.each do |mutation|
         send(*mutation)
       end
-    ensure
-      @batch = nil
+      @batch = b
     end
 
     def get(column_family, key, *columns_and_options)

data/test/cassandra_test.rb

@@ -8,16 +8,16 @@ class CassandraTest < Test::Unit::TestCase
   end
 
   def setup
-    @twitter = Cassandra.new('Twitter', "127.0.0.1:9160", :retries => 2, :connect_timeout => 0.1, :timeout => 5, :exception_classes => [])
+    @twitter = Cassandra.new('Twitter', "127.0.0.1:9160", :retries => 2, :connect_timeout => 1, :timeout => 5, :exception_classes => [])
     @twitter.clear_keyspace!
 
-    @blogs = Cassandra.new('Multiblog', "127.0.0.1:9160", :retries => 2, :connect_timeout => 0.1, :timeout => 5, :exception_classes => [])
+    @blogs = Cassandra.new('Multiblog', "127.0.0.1:9160", :retries => 2, :connect_timeout => 1, :timeout => 5, :exception_classes => [])
     @blogs.clear_keyspace!
 
-    @blogs_long = Cassandra.new('MultiblogLong', "127.0.0.1:9160", :retries => 2, :connect_timeout => 0.1, :timeout => 5, :exception_classes => [])
+    @blogs_long = Cassandra.new('MultiblogLong', "127.0.0.1:9160", :retries => 2, :connect_timeout => 1, :timeout => 5, :exception_classes => [])
     @blogs_long.clear_keyspace!
 
-    @type_conversions = Cassandra.new('TypeConversions', "127.0.0.1:9160", :retries => 2, :connect_timeout => 0.1, :timeout => 5, :exception_classes => [])
+    @type_conversions = Cassandra.new('TypeConversions', "127.0.0.1:9160", :retries => 2, :connect_timeout => 1, :timeout => 5, :exception_classes => [])
     @type_conversions.clear_keyspace!
 
     Cassandra::WRITE_DEFAULTS[:consistency] = Cassandra::Consistency::ONE
@@ -615,6 +615,78 @@ class CassandraTest < Test::Unit::TestCase
 
   end
 
+  def test_batch_queue_size
+    k = key
+
+    @twitter.insert(:Users, k + '0', {'delete_me' => 'v0', 'keep_me' => 'v0'})
+    @twitter.insert(:Users, k + '1', {'body' => 'v1', 'user' => 'v1'})
+    initial_subcolumns = {@uuids[1] => 'v1', @uuids[2] => 'v2'}
+    @twitter.insert(:StatusRelationships, k, {'user_timelines' => initial_subcolumns, 'dummy_supercolumn' => {@uuids[5] => 'value'}})
+    assert_equal(initial_subcolumns, @twitter.get(:StatusRelationships, k, 'user_timelines'))
+    assert_equal({@uuids[5] => 'value'}, @twitter.get(:StatusRelationships, k, 'dummy_supercolumn'))
+    new_subcolumns = {@uuids[3] => 'v3', @uuids[4] => 'v4'}
+    subcolumn_to_delete = initial_subcolumns.keys.first # the first column of the initial set
+
+    @twitter.batch(:queue_size => 5) do
+      # Normal Columns
+      @twitter.insert(:Users, k + '2', {'body' => 'v2', 'user' => 'v2'})
+      @twitter.insert(:Users, k + '3', {'body' => 'bogus', 'user' => 'v3'})
+      @twitter.insert(:Users, k + '3', {'body' => 'v3', 'location' => 'v3'})
+      @twitter.insert(:Statuses, k + '3', {'body' => 'v'})
+
+      assert_equal({'delete_me' => 'v0', 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # Written
+      assert_equal({'body' => 'v1', 'user' => 'v1'}, @twitter.get(:Users, k + '1')) # Written
+      assert_equal({}, @twitter.get(:Users, k + '2')) # Not yet written
+      assert_equal({}, @twitter.get(:Statuses, k + '3')) # Not yet written
+      assert_equal({}, @twitter.get(:UserCounters, 'bob')) if CASSANDRA_VERSION.to_f >= 0.8 # Written
+
+      if CASSANDRA_VERSION.to_f >= 0.8
+        @twitter.add(:UserCounters, 'bob', 5, 'tweet_count')
+      else
+        @twitter.insert(:Users, k + '2', {'body' => 'v2', 'user' => 'v2'})
+      end
+      # Flush!
+
+      assert_equal({'body' => 'v2', 'user' => 'v2'}, @twitter.get(:Users, k + '2')) # Written
+      assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}, @twitter.get(:Users, k + '3')) # Written and compacted
+      assert_equal({'body' => 'v'}, @twitter.get(:Statuses, k + '3')) # Written
+      assert_equal({'tweet_count' => 5}, @twitter.get(:UserCounters, 'bob')) if CASSANDRA_VERSION.to_f >= 0.8 # Written
+
+      @twitter.remove(:Users, k + '1') # Full row
+      @twitter.remove(:Users, k + '0', 'delete_me') # A single column of the row
+      @twitter.remove(:Users, k + '4')
+      @twitter.insert(:Users, k + '4', {'body' => 'v4', 'user' => 'v4'})
+
+      assert_equal({'body' => 'v1', 'user' => 'v1'}, @twitter.get(:Users, k + '1')) # Not yet removed
+      assert_equal({'delete_me' => 'v0', 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # Not yet removed
+      assert_equal({}, @twitter.get(:Users, k + '4')) # Not yet written
+
+      @twitter.insert(:Users, k + '5', {'body' => 'v5', 'user' => 'v5'})
+      # Flush!
+
+      assert_equal({'body' => 'v4', 'user' => 'v4'}, @twitter.get(:Users, k + '4')) # Written
+      assert_equal({}, @twitter.get(:Users, k + '1')) # Removed
+      assert_equal({ 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # 'delete_me' column removed
+
+      assert_equal({'body' => 'v2', 'user' => 'v2'}.keys.sort, @twitter.get(:Users, k + '2').timestamps.keys.sort) # Written
+      assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}.keys.sort, @twitter.get(:Users, k + '3').timestamps.keys.sort) # Written and compacted
+      assert_equal({'body' => 'v4', 'user' => 'v4'}.keys.sort, @twitter.get(:Users, k + '4').timestamps.keys.sort) # Written
+      assert_equal({'body' => 'v'}.keys.sort, @twitter.get(:Statuses, k + '3').timestamps.keys.sort) # Written
+
+      # SuperColumns
+      # Add and delete new sub columns to the user timeline supercolumn
+      @twitter.insert(:StatusRelationships, k, {'user_timelines' => new_subcolumns })
+      @twitter.remove(:StatusRelationships, k, 'user_timelines' , subcolumn_to_delete ) # Delete the first of the initial_subcolumns from the user_timeline supercolumn
+      # Delete a complete supercolumn
+      @twitter.remove(:StatusRelationships, k, 'dummy_supercolumn' ) # Delete the full dummy supercolumn
+    end
+
+    # Final result: initial_subcolumns - initial_subcolumns.first + new_subcolumns
+    resulting_subcolumns = initial_subcolumns.merge(new_subcolumns).reject{|k2,v| k2 == subcolumn_to_delete }
+    assert_equal(resulting_subcolumns, @twitter.get(:StatusRelationships, key, 'user_timelines'))
+    assert_equal({}, @twitter.get(:StatusRelationships, key, 'dummy_supercolumn')) # dummy supercolumn deleted
+  end
+
   def test_complain_about_nil_key
     assert_raises(ArgumentError) do
       @twitter.insert(:Statuses, nil, {'text' => 'crap'})