sunspot 1.0.5 → 1.1.0

data/README.rdoc

@@ -230,6 +230,9 @@ Sunspot repository at `upstream`:
 * Tom Coleman (tom@thesnail.org)
 * Matt Mitchell (goodieboy@gmail.com)
 * Nathan Beyer (nbeyer@gmail.com)
+* Kieran Topping
+* Nicolas Braem (nicolas.braem@gmail.com)
+* Jeremy Ashkenas (jashkenas@gmail.com)
 
 == License
 

data/TODO

@@ -1,12 +1,13 @@
-=== Sunspot 1.0 ===
-* new_search should accept block
+=== Sunspot 1.1.x ===
+* commitWithin (needs support from RSolr, currently available in master)
+* commit options (non-blocking, etc.)
 
-=== Future ===
+=== Someday ===
 * Shorthand arguments to Sunspot#search
 * Rudimentary fulltext prefixing support
-* commitWithin (need solr-ruby support for this)
-* Support Solr functions (e.g. dismax :bf)
 
 === Solr 1.5 ===
 * Field Collapsing (SOLR-236)
 * Support for official spatial support (SOLR-773)
+* Support sorting by function
+* Support string constants in functions

data/bin/sunspot-solr

@@ -30,6 +30,10 @@ OptionParser.new do |opts|
     server.solr_home = File.expand_path(s)
   end
 
+  opts.on '-j', '--solr-jar=JAR', 'Solr start jar' do |j|
+    server.solr_jar = File.expand_path(j)
+  end
+
   opts.on '--pid-dir=PID_DIR', 'Directory for pid files' do |pd|
     server.pid_dir = File.expand_path(pd)
   end

data/installer/config/schema.yml

@@ -12,37 +12,60 @@ types: !omap
   - boolean:
       class: BoolField
       suffix: b
+      invariants:
+        termVectors: valse
   - date:
       class: DateField
       suffix: d
+      invariants:
+        termVectors: valse
   - rand:
       class: RandomSortField
   - sdouble:
       class: SortableDoubleField
       suffix: e
+      invariants:
+        termVectors: valse
   - sfloat:
       class: SortableFloatField
       suffix: f
+      invariants:
+        termVectors: valse
   - sint:
       class: SortableIntField
       suffix: i
+      invariants:
+        termVectors: valse
   - slong:
       class: SortableLongField
       suffix: l
+      invariants:
+        termVectors: valse
   - string:
       class: StrField
       suffix: s
+      invariants:
+        termVectors: valse
   - tint:
       class: TrieIntField
       suffix: it
+      invariants:
+        termVectors: valse
   - tfloat:
       class: TrieFloatField
       suffix: ft
+      invariants:
+        termVectors: valse
   - tdouble:
       class: TrieDoubleField
+      suffix: et
+      invariants:
+        termVectors: valse
   - tdate:
       class: TrieDateField
       suffix: dt
+      invariants:
+        termVectors: valse
 fixed: !omap
   - id:
       type: string
@@ -69,3 +92,4 @@ fixed: !omap
 variants: !omap
   - multiValued: m
   - stored: s
+  - termVectors: v

data/lib/sunspot/composite_setup.rb

@@ -112,6 +112,10 @@ module Sunspot
       @text_fields ||= text_fields_hash.values.map { |set| set.to_a }.flatten
     end
 
+    def all_more_like_this_fields
+      @more_like_this_fields ||= more_like_this_fields_hash.values.map { |set| set.to_a }.flatten
+    end
+
     private
 
     # 
@@ -132,6 +136,16 @@ module Sunspot
         end
     end
 
+    def more_like_this_fields_hash
+      @more_like_this_fields_hash ||=
+        setups.inject({}) do |hash, setup|
+          setup.all_more_like_this_fields.each do |more_like_this_field|
+            (hash[more_like_this_field.name] ||= Set.new) << more_like_this_field
+          end
+          hash
+        end
+    end
+
     # 
     # Return a hash of field names to field objects, containing all fields
     # that are common to all of the classes enclosed. In order for fields

data/lib/sunspot/dsl/adjustable.rb

@@ -0,0 +1,47 @@
+module Sunspot
+  module DSL #:nodoc:
+    module Adjustable #:nodoc
+      # <strong>Expert:</strong> Adjust or reset the parameters passed to Solr.
+      # The adjustment will take place just before sending the params to solr,
+      # after Sunspot builds the Solr params based on the methods called in the
+      # DSL.
+      #
+      # Under normal circumstances, using this method should not be necessary;
+      # if you find that it is, please consider submitting a feature request.
+      # Using this method requires knowledge of Sunspot's internal Solr schema
+      # and Solr query representations, which are not part of Sunspot's public
+      # API; they could change at any time. <strong>This method is unsupported
+      # and your mileage may vary.</strong>
+      #
+      # ==== Examples
+      #
+      #   Sunspot.search(Post) do
+      #     adjust_solr_params do |params|
+      #       params[:q] += ' AND something_s:more'
+      #     end
+      #   end
+      #
+      #   Sunspot.more_like_this(my_post) do
+      #     adjust_solr_params do |params|
+      #       params["mlt.match.include"] = true
+      #     end
+      #   end
+      # 
+      def adjust_solr_params( &block )
+        @query.solr_parameter_adjustment = block
+      end
+
+      #
+      # <strong>Expert:</strong> Use a custom request handler for this search.
+      # The general use case for this would be a request handler configuration
+      # you've defined in solrconfig that has different search components,
+      # defaults, etc. Using this to point at an entirely different type of
+      # request handler that Sunspot doesn't support probably won't get you very
+      # far.
+      #
+      def request_handler(request_handler)
+        @search.request_handler = request_handler
+      end
+    end
+  end
+end

data/lib/sunspot/dsl/fulltext.rb

@@ -7,6 +7,9 @@ module Sunspot
     class Fulltext
       attr_reader :exclude_fields #:nodoc:
 
+      # accept function in boost
+      include Functional
+
       def initialize(query, setup) #:nodoc:
         @query, @setup = query, setup
         @fields_added = false
@@ -139,10 +142,15 @@ module Sunspot
 
       # 
       # Boost queries allow specification of an arbitrary scope for which
-      # matching documents should receive an extra boost. The block is evaluated
+      # matching documents should receive an extra boost. You can either specify 
+      # a boost factor and a block, or a boost function. The block is evaluated
       # in the usual scope DSL, and field names are attribute fields, not text
       # fields, as in other scope.
       #
+      # The boost function can be a constant (numeric or string literal), 
+      # a field name or another function. You can build arbitrarily complex 
+      # functions, which are passed transparently to solr.
+      #
       # This method can be called more than once for different boost queries
       # with different boosts.
       #
@@ -153,16 +161,23 @@ module Sunspot
       #       boost(2.0) do
       #         with(:featured, true)
       #       end
+      #
+      #       boost(function { sum(:average_rating, product(:popularity, 10)) })
       #     end
       #   end
       #
-      # In the above search, featured posts will receive a boost of 2.0.
-      #
-      def boost(factor, &block)
-        Sunspot::Util.instance_eval_or_call(
-          Scope.new(@query.create_boost_query(factor), @setup),
-          &block
-        )
+      # In the above search, featured posts will receive a boost of 2.0 and all posts 
+      # will be boosted by (average_rating + popularity * 10).
+      #
+      def boost(factor_or_function, &block)
+        if factor_or_function.is_a?(Sunspot::Query::FunctionQuery)
+          @query.add_boost_function(factor_or_function)
+        else
+          Sunspot::Util.instance_eval_or_call(
+            Scope.new(@query.create_boost_query(factor_or_function), @setup),
+            &block
+          )
+        end
       end
 
       #

data/lib/sunspot/dsl/function.rb

@@ -0,0 +1,14 @@
+module Sunspot
+  module DSL
+    class Function #:nodoc:
+      def initialize(functional) #:nodoc:
+        @functional = functional
+      end
+
+      def method_missing(method, *args, &block)
+        function_args = args.map { |arg| @functional.create_function_query(arg) }
+        Sunspot::Query::FunctionalFunctionQuery.new(method, function_args)
+      end
+    end
+  end
+end

data/lib/sunspot/dsl/functional.rb

@@ -0,0 +1,41 @@
+module Sunspot
+  module DSL
+    # 
+    # Mixin DSL to accept functions.
+    #
+    module Functional
+
+      #
+      # Specify a function query with a block that returns an expression.
+      #
+      # === Examples
+      #
+      #   function { 10 }
+      #   function { :average_rating }
+      #   function { sum(:average_rating, 10) }
+      #
+      def function(&block)
+        expression = Sunspot::Util.instance_eval_or_call(
+          Function.new(self),
+          &block
+        )
+        create_function_query(expression)
+      end
+
+      #
+      # Creates an AbstractFunctionQuery from an expression, also called by
+      # Sunspot::DSL::Function
+      #
+      def create_function_query(expression) #:nodoc:
+        if expression.is_a?(Sunspot::Query::FunctionQuery)
+          expression
+        elsif expression.is_a?(Symbol)
+          Sunspot::Query::FieldFunctionQuery.new(@setup.field(expression))
+        else
+          Sunspot::Query::ConstantFunctionQuery.new(expression)
+        end
+      end
+    end
+  end
+end
+

data/lib/sunspot/dsl/more_like_this_query.rb

@@ -0,0 +1,56 @@
+module Sunspot
+  module DSL #:nodoc:
+    #
+    # This class provides the DSL for MoreLikeThis queries.
+    #
+    class MoreLikeThisQuery < FieldQuery
+      include Paginatable, Adjustable
+
+      def fields(*field_names)
+        boosted_fields = field_names.pop if field_names.last.is_a?(Hash)
+        field_names.each do |name|
+          mlt_fields = @setup.more_like_this_fields(name)
+          raise(ArgumentError, "Field #{name} is not setup for more_like_this") if mlt_fields.empty?
+          mlt_fields.each { |field| @query.more_like_this.add_field(field) }
+        end
+        if boosted_fields
+          boosted_fields.each_pair do |field_name, boost|
+            @setup.more_like_this_fields(field_name).each do |field|
+              @query.more_like_this.add_field(field, boost)
+            end
+          end
+        end
+      end
+
+      def minimum_term_frequency(value)
+        @query.more_like_this.minimum_term_frequency = value
+      end
+      alias_method :mintf, :minimum_term_frequency
+      
+      def minimum_document_frequency(value)
+        @query.more_like_this.minimum_document_frequency = value
+      end
+      alias_method :mindf, :minimum_document_frequency
+
+      def minimum_word_length(value)
+        @query.more_like_this.minimum_word_length = value
+      end
+      alias_method :minwl, :minimum_word_length
+
+      def maximum_word_length(value)
+        @query.more_like_this.maximum_word_length = value
+      end
+      alias_method :maxwl, :maximum_word_length
+      
+      def maximum_query_terms(value)
+        @query.more_like_this.maximum_query_terms = value
+      end
+      alias_method :maxqt, :maximum_query_terms
+
+      def boost_by_relevance(should_boost)
+        @query.more_like_this.boost_by_relevance = should_boost
+      end
+      alias_method :boost, :boost_by_relevance
+    end
+  end
+end

data/lib/sunspot/dsl/paginatable.rb

@@ -0,0 +1,28 @@
+module Sunspot
+  module DSL #:nodoc
+    module Paginatable
+      # Paginate your search. This works the same way as WillPaginate's
+      # paginate().
+      #
+      # Note that Solr searches are _always_ paginated. Not calling #paginate is
+      # the equivalent of calling:
+      #
+      #   paginate(:page => 1, :per_page => Sunspot.config.pagination.default_per_page)
+      #
+      # ==== Options (options)
+      #
+      # :page<Integer,String>:: The requested page. The default is 1.
+      #
+      # :per_page<Integer,String>::
+      #   How many results to return per page. The default is the value in
+      #   +Sunspot.config.pagination.default_per_page+
+      #
+      def paginate(options = {})
+        page = options.delete(:page)
+        per_page = options.delete(:per_page)
+        raise ArgumentError, "unknown argument #{options.keys.first.inspect} passed to paginate" unless options.empty?
+        @query.paginate(page, per_page)
+      end
+    end
+  end
+end

data/lib/sunspot/dsl/search.rb

@@ -5,7 +5,7 @@ module Sunspot
     # Sunspot.query. See Sunspot::DSL::Query, Sunspot::DSL::FieldQuery, and
     # Sunspot::DSL::Scope for the full API presented.
     #
-    class Search < Query
+    class Search < StandardQuery
       def initialize(search, setup) #:nodoc:
         @search = search
         super(search, search.query, setup)

data/lib/sunspot/dsl/{query.rb → standard_query.rb}

@@ -8,7 +8,9 @@ module Sunspot
     #
     # See Sunspot.search for usage examples
     #
-    class Query < FieldQuery
+    class StandardQuery < FieldQuery
+      include Paginatable, Adjustable
+
       # Specify a phrase that should be searched as fulltext. Only +text+
       # fields are searched - see DSL::Fields.text
       #
@@ -53,7 +55,7 @@ module Sunspot
       #
       def fulltext(keywords, options = {}, &block)
         if keywords && !(keywords.to_s =~ /^\s*$/)
-          fulltext_query = @query.set_fulltext(keywords)
+          fulltext_query = @query.add_fulltext(keywords)
           if field_names = options.delete(:fields)
             Util.Array(field_names).each do |field_name|
               @setup.text_fields(field_name).each do |field|
@@ -101,54 +103,7 @@ module Sunspot
       end
       alias_method :keywords, :fulltext
 
-      # Paginate your search. This works the same way as WillPaginate's
-      # paginate().
-      #
-      # Note that Solr searches are _always_ paginated. Not calling #paginate is
-      # the equivalent of calling:
-      #
-      #   paginate(:page => 1, :per_page => Sunspot.config.pagination.default_per_page)
-      #
-      # ==== Options (options)
       #
-      # :page<Integer,String>:: The requested page. The default is 1.
-      #
-      # :per_page<Integer,String>::
-      #   How many results to return per page. The default is the value in
-      #   +Sunspot.config.pagination.default_per_page+
-      #
-      def paginate(options = {})
-        page = options.delete(:page)
-        per_page = options.delete(:per_page)
-        raise ArgumentError, "unknown argument #{options.keys.first.inspect} passed to paginate" unless options.empty?
-        @query.paginate(page, per_page)
-      end
-
-      # <strong>Expert:</strong> Adjust or reset the parameters passed to Solr.
-      # The adjustment will take place just before sending the params to solr,
-      # after Sunspot builds the Solr params based on the methods called in the
-      # DSL.
-      #
-      # Under normal circumstances, using this method should not be necessary;
-      # if you find that it is, please consider submitting a feature request.
-      # Using this method requires knowledge of Sunspot's internal Solr schema
-      # and Solr query representations, which are not part of Sunspot's public
-      # API; they could change at any time. <strong>This method is unsupported
-      # and your mileage may vary.</strong>
-      #
-      # ==== Example
-      #
-      #   Sunspot.search(Post) do
-      #     adjust_solr_params do |params|
-      #       params[:q] += ' AND something_s:more'
-      #     end
-      #   end
-      # 
-      def adjust_solr_params( &block )
-        @query.set_solr_parameter_adjustment( block )
-      end
-
-      # 
       # Scope the search by geographical distance from a given point.
       # +coordinates+ should either respond to #first and #last (e.g. a
       # two-element array), or to #lat and one of #lng, #lon, or #long.

data/lib/sunspot/dsl.rb

@@ -1,4 +1,5 @@
-%w(fields scope field_query query query_facet fulltext restriction
-   search).each do |file|
+%w(fields scope paginatable adjustable field_query standard_query query_facet
+   functional fulltext restriction search more_like_this_query
+   function).each do |file|
   require File.join(File.dirname(__FILE__), 'dsl', file)
 end

data/lib/sunspot/field.rb

@@ -10,6 +10,8 @@ module Sunspot
     def initialize(name, type, options = {}) #:nodoc
       @name, @type = name.to_sym, type
       @stored = !!options.delete(:stored)
+      @more_like_this = !!options.delete(:more_like_this)
+      raise ArgumentError, "Field of type #{type} cannot be used for more_like_this" unless type.accepts_more_like_this? or !@more_like_this
     end
 
     # Convert a value to its representation for Solr indexing. This delegates
@@ -77,6 +79,18 @@ module Sunspot
       !!@multiple
     end
 
+    # 
+    # Whether this field can be used for more_like_this queries.
+    # If true, the field is configured to store termVectors.
+    #
+    # ==== Returns
+    #
+    # Boolean:: True if this field can be used for more_like_this queries.
+    #
+    def more_like_this?
+      !!@more_like_this
+    end
+
     def hash
       indexed_name.hash
     end
@@ -107,7 +121,7 @@ module Sunspot
     end
 
     def indexed_name
-      "#{super}#{'s' if @stored}"
+      "#{super}#{'s' if @stored}#{'v' if more_like_this?}"
     end
   end
 
@@ -140,7 +154,7 @@ module Sunspot
     # String:: The field's indexed name
     #
     def indexed_name
-      "#{super}#{'m' if @multiple}#{'s' if @stored}"
+      "#{super}#{'m' if @multiple}#{'s' if @stored}#{'v' if more_like_this?}"
     end
   end
 

data/lib/sunspot/indexer.rb

@@ -54,7 +54,7 @@ module Sunspot
       if clazz
         @connection.delete_by_query("type:#{escape(clazz.name)}")
       else
-        @connection.delete_by_query("type:[* TO *]")
+        @connection.delete_by_query("*:*")
       end
     end
 

data/lib/sunspot/installer/schema_builder.rb

@@ -157,7 +157,7 @@ module Sunspot
 
       def add_comment(node)
         comment_message = " *** This #{node.name} is used by Sunspot! *** "
-        unless (comment = previous_non_text_sibling(node)) && comment.node_type == :comment && comment.content =~ /Sunspot/
+        unless (comment = previous_non_text_sibling(node)) && comment.comment? && comment.content =~ /Sunspot/
           node.add_previous_sibling(Nokogiri::XML::Comment.new(@document, comment_message))
         end
       end

data/lib/sunspot/installer/solrconfig_updater.rb

@@ -46,6 +46,7 @@ module Sunspot
           @root = @document.root
           maybe_create_spatial_component
           maybe_add_spatial_component_to_standard_handler
+          maybe_add_more_like_this_handler
           original_path = "#{@solrconfig_path}.orig"
           FileUtils.cp(@solrconfig_path, original_path)
           say("Saved backup of original to #{original_path}")
@@ -88,6 +89,18 @@ module Sunspot
           add_element(last_components_node, 'str').content = 'spatial'
         end
       end
+
+      def maybe_add_more_like_this_handler
+        unless @root.xpath('requestHandler[@name="/mlt"]').first
+          mlt_node = add_element(
+            @root, 'requestHandler',
+            :name => '/mlt', :class => 'solr.MoreLikeThisHandler'
+          )
+          defaults_node = add_element(mlt_node, 'lst', :name => 'defaults')
+          add_element(defaults_node, 'str', :name => 'mlt.mintf').content = '1'
+          add_element(defaults_node, 'str', :name => 'mlt.mindf').content = '2'
+        end
+      end
     end
   end
 end

data/lib/sunspot/installer/task_helper.rb

@@ -9,7 +9,7 @@ module Sunspot
 
       def add_element(node, name, attributes = {})
         new_node = Nokogiri::XML::Node.new(name, @document)
-        attributes.each_pair { |name, value| new_node[name] = value }
+        attributes.each_pair { |name, value| new_node[name.to_s] = value }
         node << new_node 
         new_node
       end

data/lib/sunspot/query/abstract_field_facet.rb

@@ -1,6 +1,8 @@
 module Sunspot
   module Query
     class AbstractFieldFacet
+      include RSolr::Char
+
       def initialize(field, options)
         @field, @options = field, options
       end
@@ -24,6 +26,9 @@ module Sunspot
         if @options[:limit]
           params[qualified_param('limit')] = @options[:limit].to_i
         end
+        if @options[:prefix]
+          params[qualified_param('prefix')] = escape(@options[:prefix].to_s)
+        end
         params[qualified_param('mincount')] = 
           case
           when @options[:minimum_count] then @options[:minimum_count].to_i

data/lib/sunspot/query/boost_query.rb

@@ -13,7 +13,11 @@ module Sunspot
       end
 
       def to_boolean_phrase
-        "#{super}^#{@boost}"
+        if @boost.is_a?(FunctionQuery)
+          "#{@boost}"
+        else
+          "#{super}^#{@boost}"
+        end
       end
     end
   end