datastax_rails 1.1.0.3 → 1.2.3

data/lib/datastax_rails/relation/search_methods.rb

@@ -1,6 +1,23 @@
 module DatastaxRails
   module SearchMethods
     
+    # By default, Cassandra will throw an error if you try to set a where condition
+    # on either a column with no index or on more than one column that isn't part
+    # of the primary key. If you are confident that the number of records that need
+    # to be searched is low, then you can instruct it to ignore the warning.  Generally
+    # you only want to do this when either the number of records in the table is very
+    # small or when one of the other where conditions that has an index will reduce the
+    # number of records to a small number.
+    #
+    #   Model.where(:name => 'johndoe', :active => true).allow_filtering
+    #
+    # @return [DatastaxRails::Relation] a new Relation object
+    def allow_filtering
+      clone.tap do |r|
+        r.allow_filtering_value = true
+      end
+    end
+    
     # The default consistency level for DSR is QUORUM when searching by ID.
     # For all searches using SOLR, the default consistency is ONE.  Use this
     # to override it in either case.
@@ -157,15 +174,41 @@ module DatastaxRails
     #   Model.order(:name)
     #   Model.order(:name => :desc)
     #
+    # WARNING: If this call is combined with #with_cassandra, you can only
+    # order on the clustering column of the primary key.  If this doesn't
+    # mean anything to you, then you probably don't want to use these together.
+    #
     # @param attribute [Symbol, String, Hash] the attribute to sort by and optionally the direction to sort in
     # @return [DatastaxRails::Relation] a new Relation object
     def order(attribute)
       return self if attribute.blank?
 
       clone.tap do |r|
-        order_by = attribute.is_a?(Hash) ? attribute.dup : {attribute.to_sym => :asc}
-        
-        r.order_values << order_by 
+        r.order_values << (attribute.is_a?(Hash) ? attribute : {attribute.to_sym => :asc})
+      end
+    end
+    
+    # Orders the result set in memory after all matching records have been
+    # retrieved.
+    #
+    # This means that limit is ignored until the end.  ALL matching records WILL
+    # be retrieved and sorted before taking #limit records and returning them to
+    # the caller.
+    #
+    # Why would you do this?  If you are retrieving records from a cassandra index
+    # but don't have the appropriate clustering order you can use this, but you should
+    # only do so if you are confident that the number of records returned will be low.
+    #
+    # A warning will be printed to the log if this results in a very inefficient operation.
+    #
+    # USE WITH CARE!!!!!!
+    #
+    # @param attribute [Symbol, String, Hash] the attribute to sort by and optionally the direction to sort in
+    # @return [DatastaxRails::Relation] a new Relation object
+    def slow_order(attribute)
+      return self if attribute.blank?
+      clone.tap do |r|
+        r.slow_order_values << (attribute.is_a?(Hash) ? attribute : {attribute.to_sym => :asc}) 
       end
     end
     
@@ -404,15 +447,12 @@ module DatastaxRails
     # You can also pass in an options hash with the following options:
     #
     #  * :fields => list of fields to search instead of the default of all fields
-    #  * :highlight => List of fields to retrieve highlights for.  Note that highlighted fields *must* be +:stored+
     #
     #   Model.fulltext("john smith", :fields => [:title])
-    #   Model.fulltext("john smith", :hightlight => [:body])
     #
     # @param query [String] a fulltext query to pass to solr
     # @param opts [Hash] an optional options hash to modify the fulltext query
     # @option opts [Array] :fields list of fields to search instead of the default of all text fields (not-implemented)
-    # @option opts [Array] :highlight list of fields to retrieve highlights for (not-implemented)
     # @return [DatastaxRails::Relation] a new Relation object
     def fulltext(query, opts = {})
       return self if query.blank?
@@ -424,6 +464,56 @@ module DatastaxRails
       end
     end
     
+    # Enables highlighting on specific fields when used with full 
+    # text searching. In order for highlighting to work, the highlighted 
+    # field(s) *must* be +:stored+
+    # 
+    #   Model.fulltext("ruby on rails").highlight(:tags, :body)
+    #   Model.fulltext("pizza").highlight(:description, snippets: 3, fragsize: 150)
+    # 
+    # In addition to the array of field names to highlight, you can pass in an
+    # options hash with the following options:
+    #
+    #  * :snippets => number of highlight snippets to return
+    #  * :fragsize => number of characters for each snippet length
+    #  * :pre_tag => text which appears before a highlighted term
+    #  * :post_tag => text which appears after a highlighted term
+    #  * :merge_contiguous => collapse contiguous fragments into a single fragment
+    #  * :use_fast_vector => enables the Solr FastVectorHighlighter
+    # 
+    # Note: When enabling +:use_fast_vector+, the highlighted fields must be also have
+    # +:term_vectors+, +:term_positions+, and +:term_offsets+ enabled. 
+    # For more information about these options, refer to Solr's wiki 
+    # on HighlightingParameters[http://http://wiki.apache.org/solr/HighlightingParameters].
+    #
+    # @overload highlight(*args, opts)
+    #   Highlights the full text search terms for the specified fields with the
+    #   given options
+    #   @param [Array] args list of field names to be highlighted
+    #   @param [Hash] opts an options hash to configure the Solr highlighter
+    #   @option opts [Integer] :snippets number of highlighted snippets to return
+    #   @option opts [Integer] :fragsize number of characters for each snippet length
+    #   @option opts [String] :pre_tag text which appears before a highlighted term
+    #   @option opts [String] :post_tag text which appears after a highlighted term
+    #   @option opts [true, false] :merge_contiguous collapse contiguous fragments into a single fragment
+    #   @option opts [true, false] :use_fast_vector enables the Solr FastVectorHighlighter
+    #   @return [DatastaxRails::Relation] a new Relation object
+    # @overload highlight(*args)
+    #   Highlights the full text search terms for the specified fields
+    #   @param [Array] args list of field names to be highlighted
+    #   @return [DatastaxRails::Relation] a new Relation object
+    def highlight(*args)
+      return self if args.blank?
+      
+      opts = args.last.is_a?(Hash) ? args.pop : {}
+      
+      clone.tap do |r|
+        opts[:fields] = r.highlight_options[:fields] || []
+        opts[:fields] |= args # Union unique field names
+        r.highlight_options.merge! opts
+      end
+    end
+    
     # @see where
     def less_than(value)
       raise ArgumentError, "#less_than can only be called after an appropriate where call. e.g. where(:created_at).less_than(1.day.ago)"
@@ -438,10 +528,14 @@ module DatastaxRails
     def solr_format(value)
       return value unless use_solr_value
       case
-        when value.is_a?(Date), value.is_a?(Time)
-          value.strftime('%Y-%m-%dT%H:%M:%SZ')
+        when value.is_a?(Time)
+          value.utc.strftime(DatastaxRails::Types::TimeType::FORMAT)
+        when value.is_a?(DateTime)
+          value.to_time.utc.strftime(DatastaxRails::Types::TimeType::FORMAT)
+        when value.is_a?(Date)
+          value.strftime(DatastaxRails::Types::TimeType::FORMAT)
         when value.is_a?(Array)
-          value.collect {|v| v.gsub(/ /,"\\ ") }.join(" OR ")
+          value.collect {|v| v.to_s.gsub(/ /,"\\ ") }.join(" OR ")
         when value.is_a?(Fixnum)
           value < 0 ? "\\#{value}" : value
         when value.is_a?(Range)
@@ -455,29 +549,6 @@ module DatastaxRails
       end
     end
     
-    protected
-      def find_by_attributes(match, attributes, *args) #:nodoc:
-        conditions = {}
-        Hash[attributes.map {|a| [a, args[attributes.index(a)]]}].each do |k,v|
-          if(v.is_a?(String))
-            conditions[k] = v.gsub(/(\W)/, '\\\\\1')
-          else
-            conditions[k] = v
-          end
-        end
-        
-        self.where_values << conditions
-        result = self.send(match.finder)
-        #result = where(conditions).send(match.finder)
-        
-        if match.blank? && result.blank?
-          raise RecordNotFound, "Couldn't find #{klass.name} with #{conditions.to_a.collect {|p| p.join('=')}.join(', ')}"
-        else
-          yield(result) if block_given?
-          result
-        end
-      end
-    
     class WhereProxy #:nodoc:
       def initialize(relation, attribute, invert = false) #:nodoc:
         @relation, @attribute, @invert = relation, attribute, invert

data/lib/datastax_rails/relation/spawn_methods.rb

@@ -10,6 +10,8 @@ module DatastaxRails
       
       merged_relation = clone
       
+      r = r.with_default_scope if r.default_scoped? && r.klass != klass
+      
       (Relation::MULTI_VALUE_METHODS - [:where, :where_not]).each do |method|
         value = r.send(:"#{method}_values")
         merged_relation.send(:"#{method}_values=", merged_relation.send(:"#{method}_values") + value) if value.present?
@@ -31,7 +33,7 @@ module DatastaxRails
       
       (Relation::SINGLE_VALUE_METHODS).each do |method|
         value = r.send(:"#{method}_value")
-        merged_relation.send(:"#{method}_value=", value) unless value.nil?
+        merged_relation.send(:"#{method}_value=", value) unless value.nil? || value == :default
       end
       
       merged_relation

data/lib/datastax_rails/relation/stats_methods.rb

@@ -62,7 +62,7 @@ module DatastaxRails
     private
     def calculate_stats(field)
       unless @stats[field]
-        @stats[field] = limit(0).compute_stats(field).stats[field]
+        @stats[field] = limit(1).compute_stats(field).stats[field]
       end
     end
   end 

data/lib/datastax_rails/relation.rb

@@ -1,9 +1,10 @@
 require 'rsolr'
+require 'pp' if ENV['DEBUG_SOLR'] == 'true'
 
 module DatastaxRails
   class Relation
-    MULTI_VALUE_METHODS = [:order, :where, :where_not, :fulltext, :greater_than, :less_than, :select, :stats]
-    SINGLE_VALUE_METHODS = [:page, :per_page, :reverse_order, :query_parser, :consistency, :ttl, :use_solr, :escape, :group]
+    MULTI_VALUE_METHODS = [:order, :where, :where_not, :fulltext, :greater_than, :less_than, :select, :stats, :field_facet, :range_facet, :slow_order]
+    SINGLE_VALUE_METHODS = [:page, :per_page, :reverse_order, :query_parser, :consistency, :ttl, :use_solr, :escape, :group, :allow_filtering]
     
     SOLR_CHAR_RX = /([\+\!\(\)\[\]\^\"\~\:\'\=\/]+)/
     
@@ -14,6 +15,7 @@ module DatastaxRails
       attr_accessor :"#{m}_value"
     end
     attr_accessor :create_with_value, :default_scoped
+    attr_accessor :highlight_options
     
     include SearchMethods
     include ModificationMethods
@@ -21,6 +23,7 @@ module DatastaxRails
     include SpawnMethods
     include StatsMethods
     include Batches
+    include FacetMethods
     
     attr_reader :klass, :column_family, :loaded, :cql
     alias :loaded? :loaded
@@ -41,14 +44,14 @@ module DatastaxRails
       
       SINGLE_VALUE_METHODS.each {|v| instance_variable_set(:"@#{v}_value", nil)}
       MULTI_VALUE_METHODS.each {|v| instance_variable_set(:"@#{v}_values", [])}
+      @highlight_options = {}
       @per_page_value = @klass.default_page_size
       @page_value = 1
-      @use_solr_value = true
+      @use_solr_value = :default
       @extensions = []
       @create_with_value = {}
-      @escape_value = true
+      @escape_value = :default
       @stats = {}
-      apply_default_scope
     end
     
     # Returns true if the two relations have the same query parameters
@@ -87,11 +90,8 @@ module DatastaxRails
     # matching documents
     #
     # Compare with #size.
-    #
-    # XXX: Count via CQL is useless unless criteria has been applied.
-    # Otherwise you get everything that has ever been in the CF.
     def count
-      @count ||= self.use_solr_value ? count_via_solr : count_via_cql
+      @count ||= (with_default_scope.path_decision == :solr) ? with_default_scope.count_via_solr : with_default_scope.count_via_cql
     end
     
     def stats
@@ -118,10 +118,16 @@ module DatastaxRails
     
     # Gets a default scope with no conditions or search attributes set.
     def default_scope
-      clone.tap do |r|
-        SINGLE_VALUE_METHODS.each {|v| r.instance_variable_set(:"@#{v}_value", nil)}
-        MULTI_VALUE_METHODS.each {|v| r.instance_variable_set(:"@#{v}_values", [])}
-        apply_default_scope
+      klass.scoped.with_default_scope
+    end
+    
+    def with_default_scope #:nodoc:
+      if default_scoped? && default_scope = klass.send(:build_default_scope)
+        default_scope = default_scope.merge(self)
+        default_scope.default_scoped = false
+        default_scope
+      else
+        self
       end
     end
     
@@ -207,11 +213,11 @@ module DatastaxRails
     # Returns a standard array thus no more methods may be chained.
     def to_a
       return @results if loaded?
-      if use_solr_value
-        @results = query_via_solr
+      if with_default_scope.path_decision == :solr
+        @results = with_default_scope.query_via_solr
         @count = @group_value ? @results.total_for_all : @results.total_entries
       else
-        @results = query_via_cql
+        @results = with_default_scope.query_via_cql
       end
       @loaded = true
       @results
@@ -236,16 +242,56 @@ module DatastaxRails
       super
     end
     
-    # NOTE: This method does not actually run a count via CQL because it only
-    # works if you run against a secondary index. So this currently just
-    # delegates to the count_via_solr method.
+    def path_decision
+      return :cassandra if klass <= DatastaxRails::CassandraOnlyModel
+      case use_solr_value
+      when false
+        return :cassandra
+      when true
+        return :solr
+      else
+        # If we've already decided to use cassandra, just go with it.
+        return :cassandra unless use_solr_value
+        [order_values, where_not_values, fulltext_values, greater_than_values, less_than_values, field_facet_values,
+         range_facet_values, group_value].each do |solr_only_stuff|
+           return :solr unless solr_only_stuff.blank? 
+         end
+        return :solr unless group_value.blank?
+        return :solr unless page_value == 1
+        @where_values.each do |wv|
+          wv.each do |k,v|
+            next if k.to_sym == :id
+            if(klass.attribute_definitions[k].indexed == :solr || !klass.attribute_definitions[k].indexed)
+              return :solr
+            end
+          end
+        end
+        # If we get here, we can safely run this query via Cassandra
+        return :cassandra
+      end
+    end
+    
+    # If we index something into both cassandra and solr, we rename the cassandra
+    # column.  This method maps the column names as necessary
+    def map_cassandra_columns(conditions)
+      {}.tap do |mapped|
+        conditions.each do |k,v|
+          if(klass.attribute_definitions[k].indexed == :both)
+            mapped["__#{k}"] = v
+          else
+            mapped[k] = v
+          end
+        end
+      end
+    end
+    
     def count_via_cql
-      select_columns = ['count(*)']
-      cql = @cql.select(select_columns)
+      cql = @cql.select(['count(*)'])
       cql.using(@consistency_value) if @consistency_value
       @where_values.each do |wv|
         cql.conditions(wv)
       end
+      cql.allow_filtering if @allow_filtering_value
       CassandraCQL::Result.new(cql.execute).fetch['count']
     end
     
@@ -254,10 +300,10 @@ module DatastaxRails
     # For ad-hoc queries, you will have to use Solr.
     def query_via_cql
       select_columns = select_values.empty? ? (@klass.attribute_definitions.keys - @klass.lazy_attributes) : select_values.flatten
-      cql = @cql.select(select_columns + ['key'])
+      cql = @cql.select((select_columns + @klass.key_factory.key_columns).uniq)
       cql.using(@consistency_value) if @consistency_value
       @where_values.each do |wv|
-        cql.conditions(wv)
+        cql.conditions(Hash[wv.map {|k,v| [(k.to_sym == :id ? :key : k), v]}])
       end
       @greater_than_values.each do |gtv|
         gtv.each do |k,v|
@@ -270,13 +316,47 @@ module DatastaxRails
       if(@per_page_value)
         cql.limit(@per_page_value)
       end
+      cql.allow_filtering if @allow_filtering_value
       results = []
-      CassandraCQL::Result.new(cql.execute).fetch do |row|
-        results << @klass.instantiate(row['key'], row.to_hash, select_columns)
+      begin
+        CassandraCQL::Result.new(cql.execute).fetch do |row|
+          results << @klass.instantiate(row['key'], row.to_hash, select_columns)
+        end
+      rescue CassandraCQL::Error::InvalidRequestException => e
+        # If we get an exception about an empty key, ignore it.  We'll return an empty set.
+        if e.message =~ /Key may not be empty/
+          # No-Op
+        else
+          raise
+        end
+      end
+      if(@slow_order_values.any?)
+        results.sort! do |a,b| 
+          values = slow_ordering(a,b)
+          values[0] <=> values[1]
+        end
       end
       results
     end
     
+    def slow_ordering(obj1, obj2)
+      [[],[]].tap do |values|
+        i=0
+        @slow_order_values.each do |ordering|
+          ordering.each do |k,v|
+            if v == :asc
+              values[0][i] = obj1.send(k)
+              values[1][i] = obj2.send(k)
+            else
+              values[1][i] = obj1.send(k)
+              values[0][i] = obj2.send(k)
+            end
+          end
+          i += 1
+        end
+      end
+    end
+    
     # Runs the query with a limit of 1 just to grab the total results attribute off
     # the result set. 
     def count_via_solr
@@ -354,10 +434,9 @@ module DatastaxRails
         q = "*:*"
       else
         q = @fulltext_values.collect {|ftv| "(" + ftv[:query] + ")"}.join(' AND ')
+        hl_fields = @fulltext_values.collect { |ftv| ftv[:highlight].join(",") if ftv[:highlight].present? }.join(",")
       end
       
-      #TODO highlighting and fielded queries of fulltext
-      
       params = {:q => q}
       unless sort.empty?
         params[:sort] = sort
@@ -366,6 +445,54 @@ module DatastaxRails
       unless filter_queries.empty?
         params[:fq] = filter_queries
       end
+
+      # Facets
+      # facet=true to enable faceting,  facet.field=<field_name> (can appear more than once for multiple fields)
+      # Additional options: f.<field_name>.facet.<option> [e.g. f.author.facet.sort=index]
+      
+      # Facet Fields
+      unless field_facet_values.empty?
+        params['facet'] = 'true'
+        facet_fields = []
+        field_facet_values.each do |facet|
+          facet_field = facet[:field]
+          facet_fields << facet_field
+          facet[:options].each do |key,value|
+            params["f.#{facet_field}.facet.#{key}"] = value.to_s
+          end
+        end
+        params['facet.field'] = facet_fields
+      end
+
+      # Facet Ranges
+      unless range_facet_values.empty?
+        params['facet'] = 'true'
+        facet_fields = []
+        range_facet_values.each do |facet|
+          facet_field = facet[:field]
+          facet_fields << facet_field
+          facet[:options].each do |key,value|
+            params["f.#{facet_field}.facet.range.#{key}"] = value.to_s
+          end
+        end
+        params['facet.range'] = facet_fields
+      end
+      
+      if @highlight_options[:fields].present?
+        params[:hl] = true
+        params['hl.fl'] = @highlight_options[:fields]
+        params['hl.snippets'] = @highlight_options[:snippets] if @highlight_options[:snippets]
+        params['hl.fragsize'] = @highlight_options[:fragsize] if @highlight_options[:fragsize]
+        if @highlight_options[:use_fast_vector]
+          params['hl.useFastVectorHighlighter'] = true
+          params['hl.tag.pre'] = @highlight_options[:pre_tag] if @highlight_options[:pre_tag].present?
+          params['hl.tag.post'] = @highlight_options[:post_tag] if @highlight_options[:post_tag].present?
+        else
+          params['hl.mergeContiguous'] = !!@highlight_options[:merge_contiguous]
+          params['hl.simple.pre'] = @highlight_options[:pre_tag] if @highlight_options[:pre_tag].present?
+          params['hl.simple.post'] = @highlight_options[:post_tag] if @highlight_options[:post_tag].present?
+        end
+      end
       
       select_columns = select_values.empty? ? (@klass.attribute_definitions.keys - @klass.lazy_attributes) : select_values.flatten
       select_columns << "id"
@@ -387,10 +514,10 @@ module DatastaxRails
         params['group.field'] = @group_value
         params['group.limit'] = @per_page_value
         params['group.offset'] = (@page_value - 1) * @per_page_value
-        params['group.ngroups'] = 'true'
+        params['group.ngroups'] = 'false' # must be false due to issues with solr sharding
         solr_response = rsolr.post('select', :data => params)
         response = solr_response["grouped"][@group_value.to_s]
-        results.total_groups = response['ngroups'].to_i
+        results.total_groups = response['groups'].size
         results.total_for_all = response['matches'].to_i
         results.total_entries = 0
         response['groups'].each do |group|
@@ -400,11 +527,19 @@ module DatastaxRails
       else
         solr_response = rsolr.paginate(@page_value, @per_page_value, 'select', :data => params, :method => :post)
         response = solr_response["response"]
+        pp solr_response if ENV['DEBUG_SOLR'] == 'true'
         results = parse_docs(response, select_columns)
+        results.highlights = solr_response['highlighting']
       end
       if solr_response["stats"]
         @stats = solr_response["stats"]["stats_fields"].with_indifferent_access
       end
+      # Apply Facets if they exist
+      if solr_response['facet_counts']
+        results.facets = {}
+        results.facets = results.facets.merge(solr_response['facet_counts']['facet_fields'].to_h)
+        results.facets = results.facets.merge(solr_response['facet_counts']['facet_ranges'].to_h)
+      end
       pp params if ENV['DEBUG_SOLR'] == 'true'
       results
     end
@@ -426,6 +561,7 @@ module DatastaxRails
           obj = @klass.with_cassandra.consistency(@consistency_value).find_by_id(id)
           results << obj if obj
         else
+          #byebug
           results << @klass.instantiate(id, doc, select_columns)
         end
       end
@@ -493,7 +629,7 @@ module DatastaxRails
     protected
       
       def method_missing(method, *args, &block) #:nodoc:
-        if Array.method_defined?(method)
+        if DatastaxRails::Collection.method_defined?(method)
           to_a.send(method, *args, &block)
         elsif @klass.respond_to?(method, true)
           scoping { @klass.send(method, *args, &block) }