datastax_rails 1.2.3 → 2.0.3

data/lib/datastax_rails/relation/search_methods.rb

@@ -11,6 +11,8 @@ module DatastaxRails
     #
     #   Model.where(:name => 'johndoe', :active => true).allow_filtering
     #
+    # NOTE that this only applies when doing a search via a cassandra index.
+    #
     # @return [DatastaxRails::Relation] a new Relation object
     def allow_filtering
       clone.tap do |r|
@@ -24,14 +26,22 @@ module DatastaxRails
     # 
     #   Model.consistency(:local_quorum).find("12345")
     #
-    # Note that Solr searches (basically anything but find by id) don't allow you
-    # to specify the consistency level.  DSR sort of gets around this by taking the
-    # search results and then going to Cassandra to retrieve the objects by ID using
-    # the consistency you specified.  However, it is possible that you might not get
-    # all of the records you are expecting if the SOLR node you were talking to hasn't
-    # been updated yet with the results.  In practice, this should not happen for
-    # records that were created over your connection, but it is possible for other
-    # connections to create records that you can't see yet.
+    # Note that Solr searches don't allow you to specify the consistency level.
+    # DSR sort of gets around this by taking the search results and then going
+    # to Cassandra to retrieve the objects by ID using the consistency you specified.
+    # However, it is possible that you might not get all of the records you are 
+    # expecting if the SOLR node you were talking to hasn't been updated yet with 
+    # the results. In practice, this should not happen for records that were created 
+    # over your connection, but it is possible for other connections to create records
+    # that you can't see yet.
+    #
+    # Valid consistency levels are:
+    # * :any
+    # * :one
+    # * :quorum
+    # * :local_quorum (if using Network Topology)
+    # * :each_quorum (if using Network Topology)
+    # * :all
     #
     # @param level [Symbol, String] the level to set the consistency at
     # @return [DatastaxRails::Relation] a new Relation object
@@ -118,7 +128,8 @@ module DatastaxRails
     
     # WillPaginate compatible method for paginating
     #
-    #   Model.paginate(:page => 2, :per_page => 10)
+    #   Model.paginate(page: 2, per_page: 10)
+    #
     # @param options [Hash] the options to pass to paginate
     # @option options [String, Fixnum] :page the page number to retrieve
     # @option options [String, Fixnum] :per_page the number of records to include on a page
@@ -144,7 +155,7 @@ module DatastaxRails
     # after-the-fact as the groups are returned as hash of Collection objects.
     #
     # Because SOLR is doing the grouping work, we can only group on single-valued
-    # fields (i.e., not +text+ or +array+ attributes). In the future, SOLR may
+    # fields (i.e., not +text+ or collections). In the future, SOLR may
     # support grouping on multi-valued fields.
     #
     # NOTE: Group names will be lower-cased
@@ -172,11 +183,11 @@ module DatastaxRails
     # the value is :desc
     #
     #   Model.order(:name)
-    #   Model.order(:name => :desc)
+    #   Model.order(name: :desc)
     #
     # WARNING: If this call is combined with #with_cassandra, you can only
-    # order on the clustering column of the primary key.  If this doesn't
-    # mean anything to you, then you probably don't want to use these together.
+    # order on the cluster_by column.  If this doesn't mean anything to you, 
+    # then you probably don't want to use these together.
     #
     # @param attribute [Symbol, String, Hash] the attribute to sort by and optionally the direction to sort in
     # @return [DatastaxRails::Relation] a new Relation object
@@ -243,12 +254,13 @@ module DatastaxRails
     #
     # The exception to this rule is when an attribute is lazy-loaded (e.g., binary).
     # In that case, it is never retrieved until you call the getter method.
-    def select(value = Proc.new)
+    def select(*fields)
       if block_given?
-        to_a.select {|*block_args| value.call(*block_args) }
+        to_a.select {|*block_args| yield(*block_args) }
       else
+        railse ArgumentError, 'Call this with at least one field' if fields.empty?
         clone.tap do |r|
-          r.select_values += Array.wrap(value)
+          r.select_values += fields
         end
       end
     end
@@ -256,10 +268,10 @@ module DatastaxRails
     # Reverses the order of the results. The following are equivalent:
     # 
     #   Model.order(:name).reverse_order
-    #   Model.order(:name => :desc) 
+    #   Model.order(name: :desc) 
     #
     #   Model.order(:name).reverse_order.reverse_order
-    #   Model.order(:name => :asc)
+    #   Model.order(name: :asc)
     #
     # @return [DatastaxRails::Relation] a new Relation object
     def reverse_order
@@ -268,9 +280,8 @@ module DatastaxRails
       end
     end
     
-    # By default, DatastaxRails uses the LuceneQueryParser.  Note that this
-    # is a change from the underlying Sunspot gem.  Sunspot defaults to the
-    # +disMax+ query parser.  If you want to use that, then pass that in here.
+    # By default, DatastaxRails uses the LuceneQueryParser. disMax
+    # is also supported. eDisMax probably works as well.
     #
     # *This only applies to fulltext queries*
     #
@@ -287,12 +298,12 @@ module DatastaxRails
     end
     
     # Have SOLR compute stats for a given numeric field.  Status computed include:
-    #  * min
-    #  * max
-    #  * sum
-    #  * sum of squares
-    #  * mean
-    #  * standard deviation
+    # * min
+    # * max
+    # * sum
+    # * sum of squares
+    # * mean
+    # * standard deviation
     #
     #   Model.compute_stats(:price)
     #   Model.compute_stats(:price, :quantity)
@@ -340,14 +351,14 @@ module DatastaxRails
     end
     
     # Specifies restrictions (scoping) on the result set. Expects a hash
-    # in the form +attribute => value+ for equality comparisons.
+    # in the form +attribute: value+ for equality comparisons.
     #
-    #   Model.where(:group_id => '1234', :active => 'Y')
+    #   Model.where(group_id: '1234', active: true)
     #
     # The value of the comparison does not need to be a scalar.  For example:
     #
-    #   Model.where(:name => ["Bob", "Tom", "Sally"])
-    #   Model.where(:age => 18..65)
+    #   Model.where(name: ["Bob", "Tom", "Sally"]) # Finds where name is any of the three names
+    #   Model.where(age: 18..65) # Finds where age is anywhere in the range
     #
     # Inequality comparisons such as greater_than and less_than are
     # specified via chaining:
@@ -359,8 +370,8 @@ module DatastaxRails
     # that can be done with a single call.  This is useful for remote APIs and
     # such.
     #
-    #   Model.where(:created_at => {:greater_than => 1.day.ago})
-    #   Model.where(:age => {:less_than => 65})
+    #   Model.where(:created_at => {greater_than: 1.day.ago})
+    #   Model.where(:age => {less_than: 65})
     #
     # NOTE: Due to the way SOLR handles range queries, all greater/less than
     # queries are actually greater/less than or equal to queries.
@@ -371,7 +382,7 @@ module DatastaxRails
     # @return [DatastaxRails::Relation] a new Relation object
     def where(attribute)
       return self if attribute.blank?
-      if attribute.is_a?(Symbol)
+      if attribute.is_a?(Symbol) || attribute.is_a?(String)
         WhereProxy.new(self, attribute)
       else
         clone.tap do |r|
@@ -397,14 +408,14 @@ module DatastaxRails
     end
     
     # Specifies restrictions (scoping) that should not match the result set.
-    # Expects a hash in the form +attribute => value+.
+    # Expects a hash in the form +attribute: value+.
     #
-    #   Model.where_not(:group_id => '1234', :active => 'N')
+    #   Model.where_not(group_id: '1234', active: false)
     #
     # Passing an array will search for records where none of the array entries
     # are present
     #
-    #   Model.where_not(:group_id => ['1234', '5678'])
+    #   Model.where_not(group_id: ['1234', '5678'])
     #
     # The above would find all models where group id is neither 1234 or 5678.
     #
@@ -446,9 +457,9 @@ module DatastaxRails
     #
     # You can also pass in an options hash with the following options:
     #
-    #  * :fields => list of fields to search instead of the default of all fields
+    # * :fields => list of fields to search instead of the default of all fields
     #
-    #   Model.fulltext("john smith", :fields => [:title])
+    #   Model.fulltext("john smith", fields: [:title])
     #
     # @param query [String] a fulltext query to pass to solr
     # @param opts [Hash] an optional options hash to modify the fulltext query
@@ -474,12 +485,12 @@ module DatastaxRails
     # In addition to the array of field names to highlight, you can pass in an
     # options hash with the following options:
     #
-    #  * :snippets => number of highlight snippets to return
-    #  * :fragsize => number of characters for each snippet length
-    #  * :pre_tag => text which appears before a highlighted term
-    #  * :post_tag => text which appears after a highlighted term
-    #  * :merge_contiguous => collapse contiguous fragments into a single fragment
-    #  * :use_fast_vector => enables the Solr FastVectorHighlighter
+    # * :snippets => number of highlight snippets to return
+    # * :fragsize => number of characters for each snippet length
+    # * :pre_tag => text which appears before a highlighted term
+    # * :post_tag => text which appears after a highlighted term
+    # * :merge_contiguous => collapse contiguous fragments into a single fragment
+    # * :use_fast_vector => enables the Solr FastVectorHighlighter
     # 
     # Note: When enabling +:use_fast_vector+, the highlighted fields must be also have
     # +:term_vectors+, +:term_positions+, and +:term_offsets+ enabled. 
@@ -529,11 +540,11 @@ module DatastaxRails
       return value unless use_solr_value
       case
         when value.is_a?(Time)
-          value.utc.strftime(DatastaxRails::Types::TimeType::FORMAT)
+          value.utc.strftime(DatastaxRails::Column::Format::SOLR_TIME_FORMAT)
         when value.is_a?(DateTime)
-          value.to_time.utc.strftime(DatastaxRails::Types::TimeType::FORMAT)
+          value.to_time.utc.strftime(DatastaxRails::Column::Format::SOLR_TIME_FORMAT)
         when value.is_a?(Date)
-          value.strftime(DatastaxRails::Types::TimeType::FORMAT)
+          value.strftime(DatastaxRails::Column::Format::SOLR_TIME_FORMAT)
         when value.is_a?(Array)
           value.collect {|v| v.to_s.gsub(/ /,"\\ ") }.join(" OR ")
         when value.is_a?(Fixnum)
@@ -549,6 +560,9 @@ module DatastaxRails
       end
     end
     
+    # WhereProxy objects act as a placeholder for queries in which #where does not include a value.
+    # In this case, #where must be chained with #greater_than, #less_than, or #equal_to to return
+    # a new relation.
     class WhereProxy #:nodoc:
       def initialize(relation, attribute, invert = false) #:nodoc:
         @relation, @attribute, @invert = relation, attribute, invert

data/lib/datastax_rails/rsolr_client_wrapper.rb

@@ -15,7 +15,7 @@ module DatastaxRails
       end
       @rsolr.__send__(sym, *args, &block)
     rescue Errno::ECONNREFUSED
-      tries ||= DatastaxRails::Base.thrift_client.options[:retries] + 1
+      tries ||= 3
       tries -= 1
       if tries > 0
         # Force cassandra connection to roll

data/lib/datastax_rails/schema/cassandra.rb

@@ -8,9 +8,9 @@ module DatastaxRails
           unless column_exists?(model.column_family.to_s, attribute.to_s)
             count += 1
             say "Adding column '#{attribute}'", :subitem
-            DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add(attribute => :text).execute
+            DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add(attribute => definition.cql_type).execute
           end
-          if(definition.indexed == :cassandra)
+          if(definition.options[:cql_index] && !definition.options[:solr_index])
             unless index_exists?(model.column_family.to_s, attribute.to_s)
               if index_exists?(model.column_family.to_s, attribute.to_s)
                 count += 1
@@ -21,12 +21,12 @@ module DatastaxRails
               say "Creating cassandra index on #{attribute.to_s}", :subitem
               DatastaxRails::Cql::CreateIndex.new(cassandra_index_cql_name(model.column_family.to_s, attribute.to_s)).on(model.column_family.to_s).column(attribute.to_s).execute
             end
-          elsif(definition.indexed == :both)
+          elsif(definition.options[:cql_index])
             unless column_exists?(model.column_family.to_s, "__#{attribute.to_s}")
               # Create and populate the new column
               count += 1
               say "Adding column '__#{attribute}'", :subitem
-              DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add("__#{attribute.to_s}" => model.attribute_definitions[attribute].coder.options[:cassandra_type]).execute
+              DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add("__#{attribute.to_s}" => definition.cql_type).execute
               say "Populating column '__#{attribute}' (this might take a while)", :subitem
               export = "echo \"copy #{model.column_family.to_s} (key, #{attribute.to_s}) TO 'dsr_export.csv';\" | cqlsh #{model.current_server}"
               import = "echo \"copy #{model.column_family.to_s} (key, __#{attribute.to_s}) FROM 'dsr_export.csv';\" | cqlsh #{model.current_server}"
@@ -44,34 +44,18 @@ module DatastaxRails
         count
       end
       
-      # Creates a payload column family via CQL
-      def create_payload_column_family(model)
-        say "Creating Payload Column Family", :subitem
-        columns = {:chunk => :int, :payload => :text}
-        DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(:digest).key_columns("digest, chunk").key_type(:text).columns(columns).with("COMPACT STORAGE").execute
-      end
-      
-      # Creates a wide-storage column family via CQL
-      def create_wide_storage_column_family(model)
-        say "Creating Wide-Storage Column Family", :subitem
-        key_name = model.key_factory.attributes.join
-        cluster_by = model.cluster_by.keys.first
-        cluster_dir = model.cluster_by.values.first
-        key_columns = "#{key_name}, #{cluster_by}"
-        columns = {}
-        model.attribute_definitions.each {|k,v| columns[k] = v.coder.options[:cassandra_type] unless k.to_s == key_name}
-        DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(key_name).key_columns(key_columns).key_type(:text).columns(columns).
-          with("CLUSTERING ORDER BY (#{cluster_by} #{cluster_dir.to_s.upcase})").execute
-      end
-      
-      # Creates a regular cassandra-only column family via CQL
-      def create_cassandra_column_family(model)
-        say "Creating Cassandra-Only Column Family", :subitem
-        key_name = "key"
-        key_columns = "#{key_name}"
+      # Creates a CQL3 backed column family
+      def create_cql3_column_family(model)
+        say "Creating Column Family via CQL3", :subitem
         columns = {}
-        model.attribute_definitions.each {|k,v| columns[k] = v.coder.options[:cassandra_type]}
-        DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(key_name).key_columns(key_columns).key_type(:text).columns(columns).execute
+        model.attribute_definitions.each {|k,col| columns[k] = col.cql_type}
+        pk = model.primary_key.to_s
+        if(model.respond_to?(:cluster_by) && model.cluster_by.present?)
+          pk += ", #{model.cluster_by.to_s}"
+        end
+        cql = DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).primary_key(pk).columns(columns)
+        cql.with(model.create_options) if model.create_options
+        cql.execute
       end
       
       # Creates the named keyspace
@@ -79,7 +63,7 @@ module DatastaxRails
         opts = { :name => keyspace.to_s,
                  :strategy_class => 'org.apache.cassandra.locator.NetworkTopologyStrategy'}.with_indifferent_access.merge(options)
 
-        if(connection.keyspaces.collect(&:name).include?(keyspace.to_s))
+        if(keyspace_exists?(keyspace.to_s))
           say "Keyspace #{keyspace.to_s} already exists"
           return false
         else
@@ -98,26 +82,6 @@ module DatastaxRails
         DatastaxRails::Cql::DropKeyspace.new(@keyspace.to_s).execute
       end
       
-      # Checks the Cassandra system tables to see if the key column is named properly.  This is
-      # a migration method to handle the fact that Solr used to create column families with "KEY"
-      # instead of the now default "key".
-      def check_key_name(cf)
-        count = 0
-        if(cf.respond_to?(:column_family))
-          cf = cf.column_family
-        end
-        klass = OpenStruct.new(:column_family => 'system.schema_columnfamilies', :default_consistency => 'QUORUM')
-        cql = DatastaxRails::Cql::ColumnFamily.new(klass)
-        results = CassandraCQL::Result.new(cql.select("key_alias, key_aliases").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute)
-        result = results.fetch
-        if(result && (result['key_alias'] == 'KEY' || result['key_aliases'].include?('KEY')) && (result['key_aliases'].blank? || !result['key_aliases'].include?('key')))
-          count += 1
-          say "Renaming KEY column", :subitem
-          DatastaxRails::Cql::AlterColumnFamily.new(cf).rename("KEY",'key').execute
-        end
-        count
-      end
-      
       # Computes the expected solr index name as reported by CQL.
       def solr_index_cql_name(cf, column)
         "#{@keyspace}_#{cf.to_s}_#{column.to_s}_index"
@@ -128,27 +92,35 @@ module DatastaxRails
         "#{cf.to_s}_#{column.to_s}_idx"
       end
       
+      # Checks the Cassandra system tables to see if a keyspace exists
+      def keyspace_exists?(keyspace)
+        klass = OpenStruct.new(:column_family => 'system.schema_keyspaces', :default_consistency => 'QUORUM')
+        cql = DatastaxRails::Cql::ColumnFamily.new(klass)
+        results = cql.select("count(*)").conditions('keyspace_name' => keyspace).execute
+        results.first['count'].to_i > 0
+      end
+      
       # Checks the Cassandra system tables to see if a column family exists
       def column_family_exists?(cf)
         klass = OpenStruct.new(:column_family => 'system.schema_columnfamilies', :default_consistency => 'QUORUM')
         cql = DatastaxRails::Cql::ColumnFamily.new(klass)
-        results = CassandraCQL::Result.new(cql.select("count(*)").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute)
-        results.fetch['count'] > 0
+        results = cql.select("count(*)").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute
+        results.first['count'] > 0
       end
       
       # Checks the Cassandra system tables to see if a column exists on a column family
       def column_exists?(cf, col)
         klass = OpenStruct.new(:column_family => 'system.schema_columns', :default_consistency => 'QUORUM')
         cql = DatastaxRails::Cql::ColumnFamily.new(klass)
-        results = CassandraCQL::Result.new(cql.select("count(*)").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute)
-        exists = results.fetch['count'] > 0
+        results = cql.select("count(*)").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute
+        exists = results.first['count'] > 0
         unless exists
-          # We need to check if it's part of the primary key (ugh)
+          # We need to check if it's part of an alias (ugh)
           klass = OpenStruct.new(:column_family => 'system.schema_columnfamilies', :default_consistency => 'QUORUM')
           cql = DatastaxRails::Cql::ColumnFamily.new(klass)
-          results = CassandraCQL::Result.new(cql.select("column_aliases, key_aliases").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute)
-          row = results.fetch
-          exists = row['key_aliases'].include?(col.to_s) || row['column_aliases'].include?(col.to_s)
+          results = cql.select("column_aliases, key_aliases, value_alias").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf).execute
+          row = results.first
+          exists = row['key_aliases'].include?(col.to_s) || row['column_aliases'].include?(col.to_s) || (row['value_alias'] && row['value_alias'].include?(col.to_s))
         end
         exists
       end
@@ -157,8 +129,8 @@ module DatastaxRails
       def index_exists?(cf, col)
         klass = OpenStruct.new(:column_family => 'system.schema_columns', :default_consistency => 'QUORUM')
         cql = DatastaxRails::Cql::ColumnFamily.new(klass)
-        results = CassandraCQL::Result.new(cql.select("index_name").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute)
-        results.fetch['index_name'] != nil
+        results = cql.select("index_name").conditions('keyspace_name' => @keyspace, 'columnfamily_name' => cf, 'column_name' => col).execute
+        results.first['index_name'] != nil
       end
     end
   end

data/lib/datastax_rails/schema/migrator.rb

@@ -33,28 +33,15 @@ module DatastaxRails
       def migrate_one(model, force = false)
         count = 0
         say_with_time("Migrating #{model.name} to latest version") do
-          if model.payload_model?
-            unless column_family_exists?(model.column_family.to_s)
-              create_payload_column_family(model)
-              count += 1
-            end
-          elsif model.wide_storage_model?
-            unless column_family_exists?(model.column_family.to_s)
-              create_wide_storage_column_family(model)
-              count += 1
-            end
-            count += check_missing_schema(model)
-          elsif model <= DatastaxRails::CassandraOnlyModel
-            unless column_family_exists?(model.column_family.to_s)
-              create_cassandra_column_family(model)
-              count += 1
-            end
-            count += check_key_name(model)
-            count += check_missing_schema(model)
-          else
-            count += check_key_name(model)
+          unless column_family_exists?(model.column_family.to_s)
+            create_cql3_column_family(model)
+            count += 1
+          end
+          
+          count += check_missing_schema(model)
+          
+          unless model <= DatastaxRails::CassandraOnlyModel
             count += upload_solr_configuration(model, force)
-            count += check_missing_schema(model)
           end
         end
         count
@@ -70,10 +57,8 @@ module DatastaxRails
         def check_schema_migrations
           unless column_family_exists?('schema_migrations')
             say "Creating schema_migrations column family"
-            connection.execute_cql_query(DatastaxRails::Cql::CreateColumnFamily.new('schema_migrations').key_type(:text).columns(:digest => :text, :solrconfig => :text, :stopwords => :text).to_cql)
+            DatastaxRails::Cql::CreateColumnFamily.new('schema_migrations').primary_key('cf').columns(:cf => :text, :digest => :text, :solrconfig => :text, :stopwords => :text).execute
           end
-          
-          check_key_name('schema_migrations')
         end
         
         def write(text="")