sunspot 0.10.5 → 0.10.6

data/History.txt

@@ -1,3 +1,16 @@
+== 0.10.6 2009-11-05
+* Support more dismax parameters
+* Support multiple boost queries
+* Allow "extra" facet rows
+* Allow exclusion of fulltext fields
+* Allow specification of per-field highlighting params
+* Specify coordinates using block extraction
+* Return empty array if no highlights available
+* Get stored text fields from hits
+* Update docs to reflect a requirement of at least one search type
+* added --max-memory and --min-memory parameters to sunspot-solr
+* LocalLucene and LocalSolr compatible with Java 1.5
+
 == 0.10.5 2009-10-22
 * Fix highlighting for multiple-model search
 

data/README.rdoc

@@ -125,9 +125,9 @@ me so I can rectify the situation!
 
 == Dependencies
 
-1. RSolr
-2. Daemons
-4. Java
+1. RSolr 0.9.6
+2. Daemons 1.x
+4. Java 1.5+
 
 Sunspot has been tested with MRI 1.8.6 and 1.8.7, REE 1.8.6, YARV 1.9.1, and
 JRuby 1.2.0

data/TODO

@@ -1,11 +1,10 @@
-=== 0.10 ===
-* Wrap everything into :q parameter when local search performed
-* Allow boosting without field constraints
-* Allow coordinates to be specified with block in setup
-=== 0.11 ===
+=== 0.10.x ===
+* Assumed inconsistency
 * Support all operations in batches. Make it smart.
   * Don't use more than one commits when one is equivalent
   * Preserve adds/deletes that are done after last commit
   * Don't do adds and deletes for the same document out of order
   * Don't do more than one add for the same document
   * Do use as few requests as possible within those constraints
+=== Future ===
+* Support Solr functions (e.g. dismax :bf)

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :minor: 10
-:patch: 5
+:patch: 6
 :major: 0

data/bin/sunspot-solr

@@ -19,12 +19,14 @@ end
 working_directory = FileUtils.pwd
 solr_install = File.expand_path(File.join(File.dirname(__FILE__), '..', 'solr'))
 
-port      = '8983'
-data_dir  = File.expand_path(File.join(Dir.tmpdir, 'solr_data'))
-solr_home = File.join(solr_install, 'solr')
-pid_dir   = working_directory
-log_file  = nil
-log_level = 'OFF'
+port        = '8983'
+data_dir    = File.expand_path(File.join(Dir.tmpdir, 'solr_data'))
+solr_home   = File.join(solr_install, 'solr')
+pid_dir     = working_directory
+log_file    = nil
+log_level   = 'OFF'
+min_memory  = '128m'
+max_memory  = '512m'
 
 OptionParser.new do |opts|
   opts.banner = "Usage: sunspot-solr start [options]"
@@ -52,6 +54,14 @@ OptionParser.new do |opts|
   opts.on '--log-file=LOG_FILE', 'Path to Solr log file' do |lf|
     log_file = File.expand_path(lf)
   end
+  
+  opts.on '--max-memory=MEMORY', 'Specify the maximum size of the memory allocation pool' do |mm|
+    max_memory = mm
+  end
+  
+  opts.on '--min-memory=MEMORY', 'Specify the initial size of the memory allocation pool' do |mm|
+    min_memory = mm
+  end
 end.parse!
 
 options = { :dir_mode => :normal, :dir => pid_dir }
@@ -72,6 +82,8 @@ end
 Daemons.run_proc('sunspot-solr', options) do
   FileUtils.cd(solr_install) do
     args = ['java']
+    args << "-Xms#{min_memory}"
+    args << "-Xmx#{max_memory}"
     args << "-Djetty.port=#{port}" if port
     args << "-Dsolr.data.dir=#{data_dir}" if data_dir
     args << "-Dsolr.solr.home=#{solr_home}" if solr_home

data/lib/sunspot/dsl/field_query.rb

@@ -6,8 +6,8 @@ module Sunspot
     # query DSL.
     #
     class FieldQuery < Scope
-      def initialize(query, setup)
-        @query = query
+      def initialize(search, query, setup) #:nodoc:
+        @search, @query = search, query
         super(query.scope, setup)
       end
 
@@ -58,39 +58,90 @@ module Sunspot
       # :zeros<Boolean>::
       #   Return facet rows for which there are no matches (equivalent to
       #   :minimum_count => 0). Default is false.
+      # :extra<Symbol,Array>::
+      #   One or more of :any and :none. :any returns a facet row with a count
+      #   of all matching documents that have some value for this field. :none
+      #   returns a facet row with a count of all matching documents that have
+      #   no value for this field. The facet row(s) corresponding to the extras
+      #   have a value of the symbol passed.
       #
       def facet(*field_names, &block)
+        options = Sunspot::Util.extract_options_from(field_names)
+
         if block
-          options =
-            if field_names.last.is_a?(Hash)
-              field_names.pop
-            else
-              {}
-            end
           if field_names.length != 1
             raise(
               ArgumentError,
               "wrong number of arguments (#{field_names.length} for 1)"
             )
           end
-          name = field_names.first
-          DSL::QueryFacet.new(@query.add_query_facet(name, options), @setup).instance_eval(&block)
+          search_facet = @search.add_query_facet(field_names.first, options)
+          Sunspot::Util.instance_eval_or_call(
+            QueryFacet.new(@query, @setup, search_facet),
+            &block
+          )
+        elsif options[:only]
+          field_names.each do |field_name|
+            field = @setup.field(field_name)
+            search_facet = @search.add_field_facet(field, options)
+            Array(options[:only]).each do |value|
+              facet = Sunspot::Query::QueryFacet.new
+              facet.add_restriction(field, Sunspot::Query::Restriction::EqualTo, value)
+              @query.add_query_facet(facet)
+              search_facet.add_row(value, facet.to_boolean_phrase)
+            end
+          end
         else
-          options = 
-            if field_names.last.is_a?(Hash)
-              field_names.pop
-            else
-              {}
+          field_names.each do |field_name|
+            search_facet = nil
+            field = @setup.field(field_name)
+            facet =
+              if options[:time_range]
+                unless field.type == Sunspot::Type::TimeType
+                  raise(
+                    ArgumentError,
+                    ':time_range can only be specified for Date or Time fields'
+                  )
+                end
+                search_facet = @search.add_date_facet(field, options)
+                Sunspot::Query::DateFieldFacet.new(field, options)
+              else
+                search_facet = @search.add_field_facet(field)
+                Sunspot::Query::FieldFacet.new(field, options)
+              end
+            @query.add_field_facet(facet)
+            Array(options[:extra]).each do |extra|
+              extra_facet = Sunspot::Query::QueryFacet.new
+              case extra
+              when :any
+                extra_facet.add_negated_restriction(
+                  field,
+                  Sunspot::Query::Restriction::EqualTo,
+                  nil
+                )
+              when :none
+                extra_facet.add_restriction(
+                  field,
+                  Sunspot::Query::Restriction::EqualTo,
+                  nil
+                )
+              else
+                raise(
+                  ArgumentError,
+                  "Allowed values for :extra are :any and :none"
+                )
+              end
+              search_facet.add_row(extra, extra_facet.to_boolean_phrase)
+              @query.add_query_facet(extra_facet)
             end
-          for field_name in field_names
-            @query.add_field_facet(@setup.field(field_name), options)
           end
         end
       end
 
       def dynamic(base_name, &block)
+        dynamic_field_factory = @setup.dynamic_field_factory(base_name)
         Sunspot::Util.instance_eval_or_call(
-          FieldQuery.new(@query, @setup.dynamic_field_factory(base_name)),
+          FieldQuery.new(@search, @query, dynamic_field_factory),
           &block
         )
       end

data/lib/sunspot/dsl/fields.rb

@@ -44,12 +44,13 @@ module Sunspot
       end
 
       # 
-      # Specify a method that returns the geographical coordinates associated
-      # with the document. The object returned must respond to #first and #last
-      # (e.g., a two-element Array); or to #lat and one of #lng, #lon, or #long
+      # Specify a method or block that returns the geographical coordinates
+      # associated with the document. The object returned must respond to #first
+      # and #last (e.g., a two-element Array); or to #lat and one of #lng, #lon,
+      # or #long
       #
-      def coordinates(name)
-        @setup.set_coordinates_field(name)
+      def coordinates(name = nil, &block)
+        @setup.set_coordinates_field(name, &block)
       end
 
       # 

data/lib/sunspot/dsl/fulltext.rb

@@ -5,9 +5,12 @@ module Sunspot
     # handler.
     #
     class Fulltext
+      attr_reader :exclude_fields #:nodoc:
+
       def initialize(query, setup) #:nodoc:
         @query, @setup = query, setup
         @fields_added = false
+        @exclude_fields = []
       end
 
       # 
@@ -46,6 +49,14 @@ module Sunspot
         end
       end
 
+      # 
+      # Exclude the given fields from the search. All fields that are configured
+      # for the types under search and not listed here will be searched.
+      #
+      def exclude_fields(*field_names)
+        @exclude_fields.concat(field_names)
+      end
+
       # 
       # Enable keyword highlighting for this search. By default, the fields 
       # under search will be highlighted; you may also may pass one or more
@@ -86,7 +97,7 @@ module Sunspot
         fields = []
         args.each { |field_name| fields.concat(@setup.text_fields(field_name)) }
 
-        @query.set_highlight(fields, options)
+        @query.add_highlight(fields, options)
       end
 
       # 
@@ -115,12 +126,26 @@ module Sunspot
         end
       end
 
+      # 
+      # The maximum number of words that can appear between search terms for a
+      # field to qualify for phrase field boost. See #query_phrase_slop for
+      # examples. Phrase slop is only meaningful if phrase fields are specified
+      # (see #phrase_fields), and it does not have an effect on which results
+      # are returned; only on what their respective boosts are.
+      #
+      def phrase_slop(slop)
+        @query.phrase_slop = slop
+      end
+
       # 
       # Boost queries allow specification of an arbitrary scope for which
       # matching documents should receive an extra boost. The block is evaluated
       # in the usual scope DSL, and field names are attribute fields, not text
       # fields, as in other scope.
       #
+      # This method can be called more than once for different boost queries
+      # with different boosts.
+      #
       # === Example
       #
       #   Sunspot.search(Post) do
@@ -159,6 +184,37 @@ module Sunspot
           end
         end
       end
+      
+      #
+      # The minimum number of search terms that a result must match. By
+      # default, all search terms must match; if the number of search terms
+      # is less than this number, the default behavior applies.
+      #
+      def minimum_match(minimum_match)
+        @query.minimum_match = minimum_match
+      end
+
+      #
+      # The number of words that can appear between the words in a
+      # user-entered phrase (i.e., keywords in quotes) and still match. For
+      # instance, in a search for "\"great pizza\"" with a query phrase slop of
+      # 1, "great pizza" and "great big pizza" will match, but "great monster of
+      # a pizza" will not. Default behavior is a query phrase slop of zero.
+      #
+      def query_phrase_slop(slop)
+        @query.query_phrase_slop = slop
+      end
+
+      #
+      # A tiebreaker coefficient for scores derived from subqueries that are
+      # lower-scoring than the maximum score subquery. Typically a near-zero
+      # value is useful. See
+      # http://wiki.apache.org/solr/DisMaxRequestHandler#tie_.28Tie_breaker.29
+      # for more information.
+      #
+      def tie(tie)
+        @query.tie = tie
+      end
 
       def fields_added? #:nodoc:
         @fields_added

data/lib/sunspot/dsl/query.rb

@@ -34,6 +34,22 @@ module Sunspot
       #   If true, perform keyword highlighting on all searched fields. If an
       #   array of field names, perform highlighting on the specified fields.
       #   This can also be called from within the fulltext block.
+      # :minimum_match<Integer>::
+      #   The minimum number of search terms that a result must match. By
+      #   default, all search terms must match; if the number of search terms
+      #   is less than this number, the default behavior applies.
+      # :tie<Float>::
+      #   A tiebreaker coefficient for scores derived from subqueries that are
+      #   lower-scoring than the maximum score subquery. Typically a near-zero
+      #   value is useful. See
+      #   http://wiki.apache.org/solr/DisMaxRequestHandler#tie_.28Tie_breaker.29
+      #   for more information.
+      # :query_phrase_slop<Integer>::
+      #   The number of words that can appear between the words in a
+      #   user-entered phrase (i.e., keywords in quotes) and still match. For
+      #   instance, in a search for "\"great pizza\"" with a phrase slop of 1,
+      #   "great pizza" and "great big pizza" will match, but "great monster of
+      #   a pizza" will not. Default behavior is a query phrase slop of zero.
       #
       def fulltext(keywords, options = {}, &block)
         if keywords && !(keywords.to_s =~ /^\s*$/)
@@ -45,15 +61,24 @@ module Sunspot
               end
             end
           end
+          if minimum_match = options.delete(:minimum_match)
+            fulltext_query.minimum_match = minimum_match.to_i
+          end
+          if tie = options.delete(:tie)
+            fulltext_query.tie = tie.to_f
+          end
+          if query_phrase_slop = options.delete(:query_phrase_slop)
+            fulltext_query.query_phrase_slop = query_phrase_slop.to_i
+          end
           if highlight_field_names = options.delete(:highlight)
             if highlight_field_names == true
-              fulltext_query.set_highlight
+              fulltext_query.add_highlight
             else
               highlight_fields = []
               Array(highlight_field_names).each do |field_name|
                 highlight_fields.concat(@setup.text_fields(field_name))
               end
-              fulltext_query.set_highlight(highlight_fields)
+              fulltext_query.add_highlight(highlight_fields)
             end
           end
           if block && fulltext_query
@@ -66,7 +91,9 @@ module Sunspot
           if !field_names && (!fulltext_dsl || !fulltext_dsl.fields_added?)
             @setup.all_text_fields.each do |field|
               unless fulltext_query.has_fulltext_field?(field)
-                fulltext_query.add_fulltext_field(field, field.default_boost)
+                unless fulltext_dsl && fulltext_dsl.exclude_fields.include?(field.name)
+                  fulltext_query.add_fulltext_field(field, field.default_boost)
+                end
               end
             end
           end

data/lib/sunspot/dsl/query_facet.rb

@@ -5,8 +5,8 @@ module Sunspot
     # method.
     #
     class QueryFacet
-      def initialize(query_facet, setup) #:nodoc:
-        @query_facet, @setup = query_facet, setup
+      def initialize(query, setup, facet) #:nodoc:
+        @query, @setup, @facet = query, setup, facet
       end
 
       # 
@@ -24,7 +24,12 @@ module Sunspot
       #   An object used to identify this facet row in the results.
       #
       def row(label, &block)
-        Scope.new(@query_facet.add_row(label), @setup).instance_eval(&block)
+        query_facet = Sunspot::Query::QueryFacet.new
+        Sunspot::Util.instance_eval_or_call(
+          Scope.new(@query.add_query_facet(query_facet), @setup),
+          &block
+        )
+        @facet.add_row(label, query_facet.to_boolean_phrase)
       end
     end
   end

data/lib/sunspot/dsl/search.rb

@@ -8,7 +8,7 @@ module Sunspot
     class Search < Query
       def initialize(search, setup) #:nodoc:
         @search = search
-        super(search.query, setup)
+        super(search, search.query, setup)
       end
 
       # 

data/lib/sunspot/dsl.rb

@@ -1,4 +1,4 @@
-%w(fields scope field_query query fulltext query_facet restriction
+%w(fields scope field_query query query_facet fulltext restriction
    search).each do |file|
   require File.join(File.dirname(__FILE__), 'dsl', file)
 end

data/lib/sunspot/field_factory.rb

@@ -123,10 +123,13 @@ module Sunspot
       end
     end
 
-    #XXX Right now this doubles as a Field and a FieldFactory - good idea?
     class Coordinates
-      def initialize(name)
-        @data_extractor = DataExtractor::AttributeExtractor.new(name)
+      def initialize(name = nil, &block)
+        if block
+          @data_extractor = DataExtractor::BlockExtractor.new(&block)
+        else
+          @data_extractor = DataExtractor::AttributeExtractor.new(name)
+        end
       end
 
       def populate_document(document, model)

data/lib/sunspot/query/abstract_field_facet.rb

@@ -0,0 +1,43 @@
+module Sunspot
+  module Query
+    class AbstractFieldFacet
+      def initialize(field, options)
+        @field, @options = field, options
+      end
+
+      def to_params
+        params = {
+          :facet => 'true',
+        }
+        case @options[:sort]
+        when :count
+          params[qualified_param('sort')] = 'true'
+        when :index
+          params[qualified_param('sort')] = 'false'
+        when nil
+        else
+          raise(
+            ArgumentError,
+            "#{@options[:sort].inspect} is not an allowed value for :sort. Allowed options are :count and :index"
+          )
+        end
+        if @options[:limit]
+          params[qualified_param('limit')] = @options[:limit].to_i
+        end
+        params[qualified_param('mincount')] = 
+          case
+          when @options[:minimum_count] then @options[:minimum_count].to_i
+          when @options[:zeros] then 0
+          else 1
+          end
+        params
+      end
+
+      private
+
+      def qualified_param(param)
+        :"f.#{@field.indexed_name}.facet.#{param}"
+      end
+    end
+  end
+end

data/lib/sunspot/query/date_field_facet.rb

@@ -0,0 +1,14 @@
+module Sunspot
+  module Query
+    class DateFieldFacet < AbstractFieldFacet
+      def to_params
+        params = super
+        params[:"facet.date"] = [@field.indexed_name]
+        params[qualified_param('date.start')] = @field.to_indexed(@options[:time_range].first)
+        params[qualified_param('date.end')] = @field.to_indexed(@options[:time_range].last)
+        params[qualified_param('date.gap')] = "+#{@options[:time_interval] || 86400}SECONDS"
+        params
+      end
+    end
+  end
+end

data/lib/sunspot/query/dismax.rb

@@ -1,9 +1,13 @@
 module Sunspot
   module Query
     class Dismax
+      attr_writer :minimum_match, :phrase_slop, :query_phrase_slop, :tie
+
       def initialize(keywords)
         @keywords = keywords
         @fulltext_fields = {}
+        @boost_queries = []
+        @highlights = []
       end
 
       # 
@@ -17,11 +21,25 @@ module Sunspot
         if @phrase_fields
           params[:pf] = @phrase_fields.map { |field| field.to_boosted_field }.join(' ')
         end
-        if @boost_query
-          params[:bq] = @boost_query.to_boolean_phrase
+        unless @boost_queries.empty?
+          params[:bq] = @boost_queries.map do |boost_query|
+            boost_query.to_boolean_phrase
+          end
+        end
+        if @minimum_match
+          params[:mm] = @minimum_match
+        end
+        if @phrase_slop
+          params[:ps] = @phrase_slop
+        end
+        if @query_phrase_slop
+          params[:qs] = @query_phrase_slop
+        end
+        if @tie
+          params[:tie] = @tie
         end
-        if @highlight
-          Sunspot::Util.deep_merge!(params, @highlight.to_params)
+        @highlights.each do |highlight|
+          Sunspot::Util.deep_merge!(params, highlight.to_params)
         end
         params
       end
@@ -30,7 +48,8 @@ module Sunspot
       # Assign a new boost query and return it.
       #
       def create_boost_query(factor)
-        @boost_query = BoostQuery.new(factor)
+        @boost_queries << boost_query = BoostQuery.new(factor)
+        boost_query
       end
 
       # 
@@ -53,8 +72,8 @@ module Sunspot
       # Highlighting object won't pass field names at all, which means
       # the dismax's :qf parameter will be used by Solr.
       #
-      def set_highlight(fields=[], options={})
-        @highlight = Highlighting.new(fields, options)
+      def add_highlight(fields=[], options={})
+        @highlights << Highlighting.new(fields, options)
       end
 
       #