datastax_rails 1.2.3 → 2.0.3

data/lib/datastax_rails/cassandra_only_model.rb

@@ -1,27 +1,33 @@
 module DatastaxRails
-  # A base class designed for models that will only interact with Cassandra.
-  # Classes that inherit from this will not generate Solr schemas or have
+  # A module designed for models that will only interact with Cassandra.
+  # Classes that include this will not generate Solr schemas or have
   # any communication with Solr.  If an application only uses these models
   # then it should be possible to run with pure Cassandra and no Solr at all.
-  class CassandraOnlyModel < Base
-    self.abstract_class = true
+  #
+  # If you want to search by anything other than primary_key, you will need
+  # to add CQL indexes as they are not created by default.
+  #
+  #   class Model < DatastaxRails::Base
+  #     include DatastaxRails::CassandraOnlyModel
+  #
+  #     uuid :id
+  #     string :name, :cql_index => true
+  #   end
+  module CassandraOnlyModel
+    extend ActiveSupport::Concern
+
+    included do
+      default_scope -> {with_cassandra}
+      self.storage_method = :cql
+    end
     
-    # XXX: This is a complete hack until we properly detect database types
-    def self.encode_attributes(attributes)
-      encoded = {}
-      attributes.each do |column_name, value|
-        encoded[column_name.to_s] = attribute_definitions[column_name.to_sym].coder.encode(value)
-        if attribute_definitions[column_name.to_sym].coder.options[:cassandra_type] == 'timestamp'
-          encoded[column_name.to_s] = encoded[column_name.to_s][0..-2]
-        elsif attribute_definitions[column_name.to_sym].coder.options[:cassandra_type] == 'int'
-          encoded[column_name.to_s] = encoded[column_name.to_s].to_i
-        elsif attribute_definitions[column_name.to_sym].coder.options[:cassandra_type] == 'boolean'
-          encoded[column_name.to_s] = encoded[column_name.to_s] == '1'
-        end
+    module ClassMethods
+      def attribute(name, options)
+        opts = options.update(:solr_index => false,    :solr_store => false,
+                              :multi_valued => false,  :sortable => false,
+                              :tokenized => false,     :fulltext => false)
+        super(name, opts)
       end
-      encoded
     end
-    
-    default_scope with_cassandra
   end
 end

data/lib/datastax_rails/column.rb

@@ -0,0 +1,384 @@
+require 'set'
+
+module DatastaxRails
+  class Column
+    TRUE_VALUES = [true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON'].to_set
+    FALSE_VALUES = [false, 0, '0', 'f', 'F', 'false', 'FALSE', 'off', 'OFF'].to_set
+    
+    module Format
+      ISO_DATE = /\A(\d{4})-(\d\d)-(\d\d)\z/
+      ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/
+      SOLR_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ".force_encoding('utf-8').freeze
+    end
+
+    attr_reader :name, :type, :cql_type, :solr_type, :options
+    attr_accessor :primary, :coder, :default
+
+    alias :encoded? :coder
+
+    # Instantiates a new column in the table.
+    #
+    # +name+ is the column's name as specified in the schema. e.g., 'first_name' in
+    # <tt>first_name text</tt>.
+    # +default+ is the type-casted default value that will be applied to a new record
+    # if no value is given.
+    # +type+ is the type of the column. Usually this will match the cql_type, but
+    # there are exceptions (e.g., date)
+    # +cql_type+ is the type of column as specified in the schema. e.g., 'text' in
+    # <tt>first_name text</tt>.
+    # +solr_type+ overrides the normal CQL <-> SOLR type mapping (uncommon)
+    def initialize(name, default, type, options = {})
+      @name      = name
+      @type      = type.to_sym
+      raise ArgumentError, "Unknown type #{type}" unless self.klass
+      options[:holds] = 'string' if collection? && options[:holds].blank?
+      @options   = configure_options(@type, options).with_indifferent_access
+      @cql_type  = compute_cql_type(@type, @options)
+      @solr_type = compute_solr_type(@type, @options)
+      @default   = extract_default(default)
+      @primary   = nil
+      @coder     = nil
+    end
+    
+    def configure_options(type, options)
+      case type.to_sym
+      when :set, :list, :map then
+        configure_options(options[:holds], options).merge(:multi_valued => true)
+      when :binary then
+        {:solr_index => false,   :solr_store => false, 
+         :multi_valued => false, :sortable => false, 
+         :tokenized => false,    :fulltext => false,
+         :cql_index => false}
+      when :boolean, :date, :time, :timestamp, :datetime, :float, :integer, :uuid then
+        {:solr_index => true,    :solr_store => true,
+         :multi_valued => false, :sortable => true,
+         :tokenized => false,    :fulltext => false,
+         :cql_index => false}
+      when :string then
+        {:solr_index => true,    :solr_store => true,
+         :multi_valued => false, :sortable => true,
+         :tokenized => false,    :fulltext => true,
+         :cql_index => false}
+      when :text then
+        {:solr_index => true,    :solr_store => true,
+         :multi_valued => false, :sortable => false,
+         :tokenized => true,     :fulltext => true,
+         :cql_index => false}
+      else
+        raise ArgumentError, "Unknown Type: #{type.to_s}"
+      end.merge(options)
+    end
+    
+    # Returns +true+ if the column is either of type ascii or text.
+    def text?
+      [:ascii, :text].include?(type)
+    end
+
+    # Returns +true+ if the column is either of type integer, float or decimal.
+    def number?
+      [:decimal, :double, :float, :integer].include?(type)
+    end
+    
+    # Returns +true+ if the column is of type binary
+    def binary?
+      [:binary].include?(type)
+    end
+    
+    def collection?
+      [:set, :list, :map].include?(type)
+    end
+
+    def has_default?
+      !default.nil?
+    end
+
+    # Returns the Ruby class that corresponds to the abstract data type.
+    def klass
+      case type
+      when :integer                        then Fixnum
+      when :float                          then Float
+      when :decimal, :double               then BigDecimal
+      when :timestamp, :time, :datetime    then Time
+      when :date                           then Date
+      when :text, :string, :binary, :ascii then String
+      when :boolean                        then Object
+      when :uuid                           then ::Cql::TimeUuid
+      when :list                           then DatastaxRails::Types::DynamicList
+      when :set                            then DatastaxRails::Types::DynamicSet
+      when :map                            then DatastaxRails::Types::DynamicMap
+      end
+    end
+
+    # Casts value (which can be a String) to an appropriate instance.
+    def type_cast(value, record = nil, dest_type = nil)
+      return nil if value.nil?
+      return coder.load(value) if encoded?
+
+      klass = self.class
+
+      case dest_type || type
+      when :string, :text        then value.to_s
+      when :ascii                then value.force_encoding('ascii')
+      when :integer              then klass.value_to_integer(value)
+      when :float                then value.to_f
+      when :decimal              then klass.value_to_decimal(value)
+      when :datetime, :timestamp then klass.string_to_time(value)
+      when :time                 then klass.string_to_dummy_time(value)
+      when :date                 then klass.value_to_date(value)
+      when :binary               then klass.binary_to_string(value)
+      when :boolean              then klass.value_to_boolean(value)
+      when :uuid, :timeuuid      then klass.value_to_uuid(value)
+      when :list, :set           then wrap_collection(value.collect {|v| type_cast(v, record, @options[:holds])}, record)
+      when :map                  then wrap_collection(value.each {|k,v| value[k] = type_cast(v, record, @options[:holds])}.stringify_keys, record)
+      else value
+      end
+    end
+    
+    def wrap_collection(collection, record)
+      klass.new(record, name, collection)
+    end
+    
+    # Cql-rb does a really good job of typecasting, so for the most part we
+    # just pass in the native types.  The only exceptions are for UUIDs that
+    # are passed in as strings and dates.
+    def type_cast_for_cql3(value, dest_type = nil)
+      return nil if value.nil?
+      return coder.dump(value) if encoded?
+      
+      case (dest_type || type)
+      when :uuid             then value.is_a?(::Cql::Uuid) ? value : self.class.value_to_uuid(value)
+      when :date             then value.to_time
+      when :list, :set       then list_to_cql3_value(value)
+      when :map              then map_to_cql3_value(value)
+      else value
+      end
+    end
+    
+    # By contrast, since Solr isn't doing things like prepared statements
+    # it doesn't know what the types are so we have to handle any casting
+    # or encoding ourselves.
+    def type_cast_for_solr(value, dest_type = nil)
+      return nil if value.nil?
+      return coder.dump(value) if encoded?
+      
+      case (dest_type || type)
+      when :boolean                            then value ? 'true' : 'false'
+      when :date, :time, :datetime, :timestamp then value.strftime(Format::SOLR_TIME_FORMAT)
+      when :list, :set                         then self.list_to_solr_value(value)
+      when :map                                then self.map_to_solr_value(value)
+      else value
+      end
+    end
+    
+    def list_to_solr_value(value)
+      value.map {|v| type_cast_for_solr(v, @options[:holds])}
+    end
+    
+    def map_to_solr_value(value)
+      value.each {|k,v| value[k] = type_cast_for_solr(v, @options[:holds])}
+    end
+    
+    def list_to_cql3_value(value)
+      value.map {|v| type_cast_for_cql3(v, @options[:holds])}
+    end
+    
+    def map_to_cql3_value(value)
+      value.dup.each {|k,v| value[k] = type_cast_for_cql3(v, @options[:holds])}
+      value
+    end
+
+    # Returns the human name of the column name.
+    #
+    # ===== Examples
+    #  Column.new('sales_stage', ...).human_name # => 'Sales stage'
+    def human_name
+      Base.human_attribute_name(@name)
+    end
+
+    def extract_default(default)
+      case type
+      when :map      then {}#lambda {|rec| DatastaxRails::Types::DynamicMap.new(rec, self.name.to_s, {})}
+      when :list     then []#lambda {|rec| DatastaxRails::Types::DynamicList.new(rec, self.name.to_s, [])}
+      when :set      then Set.new#lambda {|set| DatastaxRails::Types::DynamicSet.new(rec, self.name.to_s, Set.new)}
+      else default
+      end
+    end
+
+    # Used to convert from Strings to BLOBs
+    def string_to_binary(value)
+      self.class.string_to_binary(value)
+    end
+    
+    def full_solr_range
+      if solr_type == 'date'
+        '[* TO *]'
+      else
+        '[\"\" TO *]'
+      end
+    end
+
+    class << self
+      # Used to convert from Strings to BLOBs
+      def string_to_binary(value)
+        # TODO: Figure out what Cassandra's blobs look like
+        value
+      end
+
+      # Used to convert from BLOBs to Strings
+      def binary_to_string(value)
+        # TODO: Figure out what Cassandra's blobs look like
+        value
+      end
+
+      def value_to_date(value)
+        if value.is_a?(String)
+          return nil if value.empty?
+          fast_string_to_date(value) || fallback_string_to_date(value)
+        elsif value.respond_to?(:to_date)
+          value.to_date
+        else
+          value
+        end
+      end
+
+      def string_to_time(string)
+        return string unless string.is_a?(String)
+        return nil if string.empty?
+
+        fast_string_to_time(string) || fallback_string_to_time(string)
+      end
+      
+      def string_to_dummy_time(string)
+        return string unless string.is_a?(String)
+        return nil if string.empty?
+
+        dummy_time_string = "2000-01-01 #{string}"
+
+        fast_string_to_time(dummy_time_string) || begin
+          time_hash = Date._parse(dummy_time_string)
+          return nil if time_hash[:hour].nil?
+          new_time(*time_hash.values_at(:year, :mon, :mday, :hour, :min, :sec, :sec_fraction))
+        end
+      end
+
+      # convert something to a boolean
+      def value_to_boolean(value)
+        if value.is_a?(String) && value.empty?
+          nil
+        else
+          TRUE_VALUES.include?(value)
+        end
+      end
+
+      # Used to convert values to integer.
+      def value_to_integer(value)
+        case value
+        when TrueClass, FalseClass
+          value ? 1 : 0
+        else
+          value.to_i rescue nil
+        end
+      end
+
+      # convert something to a BigDecimal
+      def value_to_decimal(value)
+        # Using .class is faster than .is_a? and
+        # subclasses of BigDecimal will be handled
+        # in the else clause
+        if value.class == BigDecimal
+          value
+        elsif value.respond_to?(:to_d)
+          value.to_d
+        else
+          value.to_s.to_d
+        end
+      end
+      
+      # convert something to a TimeUuid
+      def value_to_uuid(value)
+        if value.is_a?(::Cql::Uuid)
+          value
+        else
+          ::Cql::TimeUuid.new(value) rescue nil
+        end
+      end
+      
+      protected
+        # '0.123456' -> 123456
+        # '1.123456' -> 123456
+        def microseconds(time)
+          time[:sec_fraction] ? (time[:sec_fraction] * 1_000_000).to_i : 0
+        end
+
+        def new_date(year, mon, mday)
+          if year && year != 0
+            Date.new(year, mon, mday) rescue nil
+          end
+        end
+
+        def new_time(year, mon, mday, hour, min, sec, microsec, offset = nil)
+          # Treat 0000-00-00 00:00:00 as nil.
+          return nil if year.nil? || (year == 0 && mon == 0 && mday == 0)
+
+          if offset
+            time = Time.utc(year, mon, mday, hour, min, sec, microsec) rescue nil
+            return nil unless time
+
+            time -= offset
+            Base.default_timezone == :utc ? time : time.getlocal
+          else
+            Time.public_send(Base.default_timezone, year, mon, mday, hour, min, sec, microsec) rescue nil
+          end
+        end
+
+        def fast_string_to_date(string)
+          if string =~ Format::ISO_DATE
+            new_date $1.to_i, $2.to_i, $3.to_i
+          end
+        end
+
+        # Doesn't handle time zones.
+        def fast_string_to_time(string)
+          if string =~ Format::ISO_DATETIME
+            microsec = ($7.to_r * 1_000_000).to_i
+            new_time $1.to_i, $2.to_i, $3.to_i, $4.to_i, $5.to_i, $6.to_i, microsec
+          end
+        end
+
+        def fallback_string_to_date(string)
+          new_date(*::Date._parse(string, false).values_at(:year, :mon, :mday))
+        end
+
+        def fallback_string_to_time(string)
+          time_hash = Date._parse(string)
+          time_hash[:sec_fraction] = microseconds(time_hash)
+
+          new_time(*time_hash.values_at(:year, :mon, :mday, :hour, :min, :sec, :sec_fraction))
+        end
+    end
+    
+    private
+      def compute_cql_type(field_type, options)
+        options[:cql_type] || case field_type.to_sym
+        when :integer                            then 'int'
+        when :time, :date, :timestamp, :datetime then 'timestamp' 
+        when :binary                             then 'blob'
+        when :list                               then "list<#{compute_cql_type(options[:holds], options)}>"
+        when :set                                then "set<#{compute_cql_type(options[:holds], options)}>"
+        when :map                                then "map<text, #{compute_cql_type(options[:holds], options)}>"
+        when :string                             then 'text'
+        else field_type.to_s
+        end
+      end
+      
+      def compute_solr_type(field_type, options)
+        options[:solr_type] || case field_type.to_sym
+        when :integer                            then 'int'
+        when :decimal                            then 'double'
+        when :timestamp, :time, :datetime        then 'date'
+        when :list, :set, :map                   then compute_solr_type(options[:holds], options)
+        else field_type.to_s
+        end
+      end
+  end
+end

data/lib/datastax_rails/connection.rb

@@ -1,7 +1,8 @@
 # require 'datastax_rails/rsolr_client_wrapper'
 require 'rsolr/client_cert'
 require 'rest_client'
-require "cassandra-cql/1.2"
+require 'cql'
+
 module DatastaxRails
   # The connection module holds all the code for establishing and maintaining a connection to
   # Datastax Exterprise.  This includes both the Cassandra and Solr connections.
@@ -27,12 +28,7 @@ module DatastaxRails
       #
       # @return [String] the hostname or ip address of the current server
       def current_server
-        thrift_client.instance_variable_get(:@current_server).to_s.split(/\:/).first
-      end
-      
-      # Returns the thrift client object
-      def thrift_client
-        self.connection.instance_variable_get(:@connection)
+        connection.current_connection.instance_variable_get(:@connection).host
       end
       
       # Establish a Cassandra connection to DSE.  datastax.yml will be read and the current environment's
@@ -41,14 +37,13 @@ module DatastaxRails
       # The following is an example production configuration document.  Assume that your setup consists
       # of three datacenters each with three servers and RF=3 (i.e., you're storing your data 9 times)
       #
-      #   servers: ["10.1.2.5:9160", "10.1.2.6:9160", "10.1.2.7:9160"]
+      #   servers: ["10.1.2.5"]
+      #   port: 9042
       #   keyspace: "datastax_rails_production"
       #   strategy_class: "org.apache.cassandra.locator.NetworkTopologyStrategy"
       #   strategy_options: {"DS1": "3", "DS2": "3", "DS3": "3"} 
       #   connection_options:
       #     timeout: 10
-      #     retries: 2
-      #     server_max_requests: 1000
       #   solr:
       #     port: 8983
       #     path: /solr
@@ -58,8 +53,12 @@ module DatastaxRails
       #       key: config/datastax_rails.key
       #       keypass: changeme
       #
-      # The +servers+ entry should be a list of all of the servers in your local datacenter.  These
-      # are the servers that DSR will attempt to connect to and will round-robin through.
+      # The +servers+ entry should be a list of all seed nodes for servers you wish to connect to.  DSR
+      # will automatically connect to all nodes in the cluster or in the datacenter if you are using multiple
+      # datacenters.  You can safely just list all nodes in a particular datacenter if you would like.
+      #
+      # The port to connect to, this port will be used for all nodes. Because the `system.peers` table does not contain the port that the nodes are 
+      # listening on, the port must be the same for all nodes.
       #
       # Since we're using the NetworkTopologyStrategy for our locator, it is important that you configure
       # cassandra-topology.properties.  See the DSE documentation at http://www.datastax.com for more
@@ -94,7 +93,7 @@ module DatastaxRails
         DatastaxRails::Base.config = spec.with_indifferent_access
         spec.reverse_merge!(DEFAULT_OPTIONS)
         connection_options = spec[:connection_options] || {}
-        self.connection = CassandraCQL::Database.new(spec[:servers], {:keyspace => spec[:keyspace], :cql_version => spec[:cql_version]}, connection_options.symbolize_keys)
+        self.connection = ::Cql::Client.connect(:hosts => spec[:servers], :keyspace => spec[:keyspace], :connection_timeout => spec[:connection_options][:timeout])
       end
       
       # Returns the base portion of the URL for connecting to SOLR based on the current Cassandra server.