DrMark-thinking-sphinx 0.9.9 → 1.1.6

data/lib/thinking_sphinx/index/builder.rb

@@ -18,9 +18,14 @@ module ThinkingSphinx
         # rails documentation. It's not needed though, so it gets undef'd.
         # Hopefully the list of methods that get in the way doesn't get too
         # long.
-        undef_method :parent
+        HiddenMethods = [:parent, :name, :id, :type].each { |method|
+          define_method(method) {
+            caller.grep(/irb.completion/).empty? ? method_missing(method) : super
+          }
+        }
         
-        attr_accessor :fields, :attributes, :properties, :conditions
+        attr_accessor :fields, :attributes, :properties, :conditions,
+          :groupings
         
         # Set up all the collections. Consider this the equivalent of an
         # instance's initialize method.
@@ -30,6 +35,7 @@ module ThinkingSphinx
           @attributes = []
           @properties = {}
           @conditions = []
+          @groupings  = []
         end
         
         # This is how you add fields - the strings Sphinx looks at - to your
@@ -81,22 +87,16 @@ module ThinkingSphinx
         def indexes(*args)
           options = args.extract_options!
           args.each do |columns|
-            fields << Field.new(FauxColumn.coerce(columns), options)
+            field = Field.new(FauxColumn.coerce(columns), options)
+            fields << field
             
-            if fields.last.sortable
-              attributes << Attribute.new(
-                fields.last.columns.collect { |col| col.clone },
-                options.merge(
-                  :type => :string,
-                  :as => fields.last.unique_name.to_s.concat("_sort").to_sym
-                )
-              )
-            end
+            add_sort_attribute field, options   if field.sortable
+            add_facet_attribute field, options  if field.faceted
           end
         end
         alias_method :field,    :indexes
         alias_method :includes, :indexes
-        
+                
         # This is the method to add attributes to your index (hence why it is
         # aliased as 'attribute'). The syntax is the same as #indexes, so use
         # that as starting point, but keep in mind the following points.
@@ -128,7 +128,7 @@ module ThinkingSphinx
         # when you would like to index a calculated value. Don't forget to set
         # the type of the attribute though:
         #
-        #   indexes "age < 18", :as => :minor, :type => :boolean
+        #   has "age < 18", :as => :minor, :type => :boolean
         # 
         # If you're creating attributes for latitude and longitude, don't
         # forget that Sphinx expects these values to be in radians.
@@ -136,11 +136,26 @@ module ThinkingSphinx
         def has(*args)
           options = args.extract_options!
           args.each do |columns|
-            attributes << Attribute.new(FauxColumn.coerce(columns), options)
+            attribute = Attribute.new(FauxColumn.coerce(columns), options)
+            attributes << attribute
+            
+            add_facet_attribute attribute, options if attribute.faceted
           end
         end
         alias_method :attribute, :has
         
+        def facet(*args)
+          options = args.extract_options!
+          options[:facet] = true
+          
+          args.each do |columns|
+            attribute = Attribute.new(FauxColumn.coerce(columns), options)
+            attributes << attribute
+            
+            add_facet_attribute attribute, options
+          end
+        end
+        
         # Use this method to add some manual SQL conditions for your index
         # request. You can pass in as many strings as you like, they'll get
         # joined together with ANDs later on.
@@ -152,6 +167,16 @@ module ThinkingSphinx
           @conditions += args
         end
         
+        # Use this method to add some manual SQL strings to the GROUP BY
+        # clause. You can pass in as many strings as you'd like, they'll get
+        # joined together with commas later on.
+        # 
+        #   group_by "lat", "lng"
+        # 
+        def group_by(*args)
+          @groupings += args
+        end
+        
         # This is what to use to set properties on the index. Chief amongst
         # those is the delta property - to allow automatic updates to your
         # indexes as new models are added and edited - but also you can
@@ -160,6 +185,9 @@ module ThinkingSphinx
         # 
         #   set_property :delta => true
         #   set_property :field_weights => {"name" => 100}
+        #   set_property :order => "name ASC"
+        #   set_property :include => :picture
+        #   set_property :select => 'name'
         # 
         # Also, the following two properties are particularly relevant for
         # geo-location searching - latitude_attr and longitude_attr. If your
@@ -168,11 +196,21 @@ module ThinkingSphinx
         # when defining the index, so you don't need to specify them for every
         # geo-related search.
         #
-        #   set_property :latitude_attr => "lt", :longitude => "lg"
+        #   set_property :latitude_attr => "lt", :longitude_attr => "lg"
         # 
         # Please don't forget to add a boolean field named 'delta' to your
         # model's database table if enabling the delta index for it.
+        # Valid options for the delta property are:
+        # 
+        # true
+        # false
+        # :default
+        # :delayed
+        # :datetime
         # 
+        # You can also extend ThinkingSphinx::Deltas::DefaultDelta to implement 
+        # your own handling for delta indexing.
+        
         def set_property(*args)
           options = args.extract_options!
           if options.empty?
@@ -189,6 +227,37 @@ module ThinkingSphinx
         def method_missing(method, *args)
           FauxColumn.new(method, *args)
         end
+        
+        # A method to allow adding fields from associations which have names
+        # that clash with method names in the Builder class (ie: properties,
+        # fields, attributes).
+        # 
+        # Example: indexes assoc(:properties).column
+        # 
+        def assoc(assoc, *args)
+          FauxColumn.new(assoc, *args)
+        end
+        
+        private
+        
+        def add_sort_attribute(field, options)
+          add_internal_attribute field, options, "_sort"
+        end
+        
+        def add_facet_attribute(resource, options)
+          add_internal_attribute resource, options, "_facet", true
+        end
+        
+        def add_internal_attribute(resource, options, suffix, crc = false)
+          @attributes << Attribute.new(
+            resource.columns.collect { |col| col.clone },
+            options.merge(
+              :type => resource.is_a?(Field) ? :string : nil,
+              :as   => resource.unique_name.to_s.concat(suffix).to_sym,
+              :crc  => crc
+            ).except(:facet)
+          )
+        end
       end
     end
   end

data/lib/thinking_sphinx/rails_additions.rb

@@ -29,6 +29,18 @@ Array.send(
   :include, ThinkingSphinx::ArrayExtractOptions
 ) unless Array.instance_methods.include?("extract_options!")
 
+module ThinkingSphinx
+  module AbstractQuotedTableName
+    def quote_table_name(name)
+      quote_column_name(name)
+    end
+  end
+end
+
+ActiveRecord::ConnectionAdapters::AbstractAdapter.send(
+  :include, ThinkingSphinx::AbstractQuotedTableName
+) unless ActiveRecord::ConnectionAdapters::AbstractAdapter.instance_methods.include?("quote_table_name")
+
 module ThinkingSphinx
   module MysqlQuotedTableName
     def quote_table_name(name) #:nodoc:
@@ -37,10 +49,13 @@ module ThinkingSphinx
   end
 end
 
-if ActiveRecord::ConnectionAdapters.constants.include?("MysqlAdapter")
-  ActiveRecord::ConnectionAdapters::MysqlAdapter.send(
-    :include, ThinkingSphinx::MysqlQuotedTableName
-  ) unless ActiveRecord::ConnectionAdapters::MysqlAdapter.instance_methods.include?("quote_table_name")
+if ActiveRecord::ConnectionAdapters.constants.include?("MysqlAdapter") or ActiveRecord::Base.respond_to?(:jdbcmysql_connection)
+  adapter = ActiveRecord::ConnectionAdapters.const_get(
+    defined?(JRUBY_VERSION) ? :JdbcAdapter : :MysqlAdapter
+  )
+  unless adapter.instance_methods.include?("quote_table_name")
+    adapter.send(:include, ThinkingSphinx::MysqlQuotedTableName)
+  end
 end
 
 module ThinkingSphinx
@@ -53,4 +68,69 @@ end
 
 ActiveRecord::Base.extend(
   ThinkingSphinx::ActiveRecordQuotedName
-) unless ActiveRecord::Base.respond_to?("quoted_table_name")
+) unless ActiveRecord::Base.respond_to?("quoted_table_name")
+
+module ThinkingSphinx
+  module ActiveRecordStoreFullSTIClass
+    def store_full_sti_class
+      false
+    end
+  end
+end
+
+ActiveRecord::Base.extend(
+  ThinkingSphinx::ActiveRecordStoreFullSTIClass
+) unless ActiveRecord::Base.respond_to?(:store_full_sti_class)
+
+module ThinkingSphinx
+  module ClassAttributeMethods
+    def cattr_reader(*syms)
+      syms.flatten.each do |sym|
+        next if sym.is_a?(Hash)
+        class_eval(<<-EOS, __FILE__, __LINE__)
+          unless defined? @@#{sym}
+            @@#{sym} = nil
+          end
+
+          def self.#{sym}
+            @@#{sym}
+          end
+
+          def #{sym}
+            @@#{sym}
+          end
+        EOS
+      end
+    end
+
+    def cattr_writer(*syms)
+      options = syms.extract_options!
+      syms.flatten.each do |sym|
+        class_eval(<<-EOS, __FILE__, __LINE__)
+          unless defined? @@#{sym}
+            @@#{sym} = nil
+          end
+
+          def self.#{sym}=(obj)
+            @@#{sym} = obj
+          end
+
+          #{"
+          def #{sym}=(obj)
+            @@#{sym} = obj
+          end
+          " unless options[:instance_writer] == false }
+        EOS
+      end
+    end
+
+    def cattr_accessor(*syms)
+      cattr_reader(*syms)
+      cattr_writer(*syms)
+    end
+  end
+end
+
+Class.extend(
+  ThinkingSphinx::ClassAttributeMethods
+) unless Class.respond_to?(:cattr_reader)

data/lib/thinking_sphinx/search.rb

@@ -5,35 +5,29 @@ module ThinkingSphinx
   # Most times, you will just want a specific model's results - to search and
   # search_for_ids methods will do the job in exactly the same manner when
   # called from a model.
-  #
+  # 
   class Search
+    GlobalFacetOptions = {
+      :all_attributes => false,
+      :class_facet    => true
+    }
+    
     class << self
       # Searches for results that match the parameters provided. Will only
       # return the ids for the matching objects. See #search for syntax
       # examples.
       #
+      # Note that this only searches the Sphinx index, with no ActiveRecord
+      # queries. Thus, if your index is not in sync with the database, this
+      # method may return ids that no longer exist there.
+      #
       def search_for_ids(*args)
         results, client = search_results(*args.clone)
-
+        
         options = args.extract_options!
         page    = options[:page] ? options[:page].to_i : 1
-
-        begin
-          pager = WillPaginate::Collection.create(page,
-            client.limit, results[:total_found] || 0) do |collection|
-            collection.replace results[:matches].collect { |match|
-              match[:attributes]["sphinx_internal_id"]
-            }
-            collection.instance_variable_set :@total_entries, results[:total_found]
-          end
-          return (options[:include_raw] ? [pager, results] : pager)
-        rescue
-          if options[:include_raw]
-            results[:matches].collect { |match| match[:attributes]["sphinx_internal_id"] }, results
-          else
-            results[:matches].collect { |match| match[:attributes]["sphinx_internal_id"] }
-          end
-        end
+        
+        ThinkingSphinx::Collection.ids_from_results(results, page, client.limit, options)
       end
 
       # Searches through the Sphinx indexes for relevant matches. There's
@@ -44,11 +38,11 @@ module ThinkingSphinx
       # just like paginate. The same parameters - :page and :per_page - work as
       # expected, and the returned result set can be used by the will_paginate
       # helper.
-      #
+      # 
       # == Basic Searching
       #
       # The simplest way of searching is straight text.
-      #
+      # 
       #   ThinkingSphinx::Search.search "pat"
       #   ThinkingSphinx::Search.search "google"
       #   User.search "pat", :page => (params[:page] || 1)
@@ -56,10 +50,10 @@ module ThinkingSphinx
       #
       # If you specify :include, like in an #find call, this will be respected
       # when loading the relevant models from the search results.
-      #
+      # 
       #   User.search "pat", :include => :posts
       #
-      # == Advanced Searching
+      # == Match Modes
       #
       # Sphinx supports 5 different matching modes. By default Thinking Sphinx
       # uses :all, which unsurprisingly requires all the supplied search terms
@@ -77,11 +71,25 @@ module ThinkingSphinx
       # for more complex query syntax, refer to the sphinx documentation for further
       # details.
       #
-      # == Searching by Fields
+      # == Weighting
+      #
+      # Sphinx has support for weighting, where matches in one field can be considered
+      # more important than in another. Weights are integers, with 1 as the default.
+      # They can be set per-search like this:
+      #
+      #   User.search "pat allan", :field_weights => { :alias => 4, :aka => 2 }
+      #
+      # If you're searching multiple models, you can set per-index weights:
+      #
+      #   ThinkingSphinx::Search.search "pat", :index_weights => { User => 10 }
+      #
+      # See http://sphinxsearch.com/doc.html#weighting for further details.
       #
+      # == Searching by Fields
+      # 
       # If you want to step it up a level, you can limit your search terms to
       # specific fields:
-      #
+      # 
       #   User.search :conditions => {:name => "pat"}
       #
       # This uses Sphinx's extended match mode, unless you specify a different
@@ -91,21 +99,29 @@ module ThinkingSphinx
       # == Searching by Attributes
       #
       # Also known as filters, you can limit your searches to documents that
-      # have specific values for their attributes. There are two ways to do
-      # this. The first is one that works in all scenarios - using the :with
-      # option.
-      #
-      #   ThinkingSphinx::Search.search :with => {:parent_id => 10}
-      #
-      # The second is only viable if you're searching with a specific model
-      # (not multi-model searching). With a single model, Thinking Sphinx
-      # can figure out what attributes and fields are available, so you can
-      # put it all in the :conditions hash, and it will sort it out.
-      #
+      # have specific values for their attributes. There are three ways to do
+      # this. The first two techniques work in all scenarios - using the :with
+      # or :with_all options.
+      #
+      #   ThinkingSphinx::Search.search :with => {:tag_ids => 10}
+      #   ThinkingSphinx::Search.search :with => {:tag_ids => [10,12]}
+      #   ThinkingSphinx::Search.search :with_all => {:tag_ids => [10,12]}
+      #
+      # The first :with search will match records with a tag_id attribute of 10.
+      # The second :with will match records with a tag_id attribute of 10 OR 12.
+      # If you need to find records that are tagged with ids 10 AND 12, you
+      # will need to use the :with_all search parameter. This is particuarly
+      # useful in conjunction with Multi Value Attributes (MVAs).
+      #
+      # The third filtering technique is only viable if you're searching with a
+      # specific model (not multi-model searching). With a single model,
+      # Thinking Sphinx can figure out what attributes and fields are available,
+      # so you can put it all in the :conditions hash, and it will sort it out.
+      # 
       #   Node.search :conditions => {:parent_id => 10}
-      #
+      # 
       # Filters can be single values, arrays of values, or ranges.
-      #
+      # 
       #   Article.search "East Timor", :conditions => {:rating => 3..5}
       #
       # == Excluding by Attributes
@@ -115,6 +131,51 @@ module ThinkingSphinx
       #
       #   User.search :without => {:role_id => 1}
       #
+      # == Excluding by Primary Key
+      #
+      # There is a shortcut to exclude records by their ActiveRecord primary key:
+      #
+      #   User.search :without_ids => 1
+      #
+      # Pass an array or a single value.
+      #
+      # The primary key must be an integer as a negative filter is used. Note
+      # that for multi-model search, an id may occur in more than one model.
+      #
+      # == Infix (Star) Searching
+      #
+      # By default, Sphinx uses English stemming, e.g. matching "shoes" if you
+      # search for "shoe". It won't find "Melbourne" if you search for
+      # "elbourn", though.
+      #
+      # Enable infix searching by something like this in config/sphinx.yml:
+      #
+      #   development:
+      #     enable_star: 1
+      #     min_infix_length: 2
+      #
+      # Note that this will make indexing take longer.
+      #
+      # With those settings (and after reindexing), wildcard asterisks can be used
+      # in queries:
+      #
+      #   Location.search "*elbourn*"
+      #
+      # To automatically add asterisks around every token (but not operators),
+      # pass the :star option:
+      #
+      #   Location.search "elbourn -ustrali", :star => true, :match_mode => :boolean
+      #
+      # This would become "*elbourn* -*ustrali*". The :star option only adds the
+      # asterisks. You need to make the config/sphinx.yml changes yourself.
+      #
+      # By default, the tokens are assumed to match the regular expression /\w+/u.
+      # If you've modified the charset_table, pass another regular expression, e.g.
+      #
+      #   User.search("oo@bar.c", :star => /[\w@.]+/u)
+      #
+      # to search for "*oo@bar.c*" and not "*oo*@*bar*.*c*".
+      #
       # == Sorting
       #
       # Sphinx can only sort by attributes, so generally you will need to avoid
@@ -138,15 +199,80 @@ module ThinkingSphinx
       # documentation[http://sphinxsearch.com/doc.html] for that level of
       # detail though.
       #
-      # == Grouping
+      # If desired, you can sort by a column in your model instead of a sphinx
+      # field or attribute. This sort only applies to the current page, so is
+      # most useful when performing a search with a single page of results.
       #
+      #   User.search("pat", :sql_order => "name")
+      #
+      # == Grouping
+      # 
       # For this you can use the group_by, group_clause and group_function
       # options - which are all directly linked to Sphinx's expectations. No
       # magic from Thinking Sphinx. It can get a little tricky, so make sure
       # you read all the relevant
       # documentation[http://sphinxsearch.com/doc.html#clustering] first.
+      # 
+      # Grouping is done via three parameters within the options hash
+      # * <tt>:group_function</tt> determines the way grouping is done
+      # * <tt>:group_by</tt> determines the field which is used for grouping
+      # * <tt>:group_clause</tt> determines the sorting order 
+      #
+      # === group_function
+      #  
+      # Valid values for :group_function are
+      # * <tt>:day</tt>, <tt>:week</tt>, <tt>:month</tt>, <tt>:year</tt> - Grouping is done by the respective timeframes. 
+      # * <tt>:attr</tt>, <tt>:attrpair</tt> - Grouping is done by the specified attributes(s)
+      # 
+      # === group_by
       #
-      # Yes this section will be expanded, but this is a start.
+      # This parameter denotes the field by which grouping is done. Note that the
+      # specified field must be a sphinx attribute or index.
+      #
+      # === group_clause
+      #
+      # This determines the sorting order of the groups. In a grouping search,
+      # the matches within a group will sorted by the <tt>:sort_mode</tt> and <tt>:order</tt> parameters.
+      # The group matches themselves however, will be sorted by <tt>:group_clause</tt>. 
+      # 
+      # The syntax for this is the same as an order parameter in extended sort mode.
+      # Namely, you can specify an SQL-like sort expression with up to 5 attributes 
+      # (including internal attributes), eg: "@relevance DESC, price ASC, @id DESC"
+      #
+      # === Grouping by timestamp
+      # 
+      # Timestamp grouping groups off items by the day, week, month or year of the
+      # attribute given. In order to do this you need to define a timestamp attribute,
+      # which pretty much looks like the standard defintion for any attribute.
+      #
+      #   define_index do
+      #     #
+      #     # All your other stuff
+      #     #
+      #     has :created_at
+      #   end
+      #
+      # When you need to fire off your search, it'll go something to the tune of
+      #   
+      #   Fruit.search "apricot", :group_function => :day, :group_by => 'created_at'
+      #
+      # The <tt>@groupby</tt> special attribute will contain the date for that group.
+      # Depending on the <tt>:group_function</tt> parameter, the date format will be
+      #
+      # * <tt>:day</tt> - YYYYMMDD
+      # * <tt>:week</tt> - YYYYNNN (NNN is the first day of the week in question, 
+      #   counting from the start of the year )
+      # * <tt>:month</tt> - YYYYMM
+      # * <tt>:year</tt> - YYYY
+      #
+      #
+      # === Grouping by attribute
+      #
+      # The syntax is the same as grouping by timestamp, except for the fact that the 
+      # <tt>:group_function</tt> parameter is changed
+      #
+      #   Fruit.search "apricot", :group_function => :attr, :group_by => 'size'
+      # 
       #
       # == Geo/Location Searching
       #
@@ -155,11 +281,11 @@ module ThinkingSphinx
       # take advantage of this, you will need to have both of those values in
       # attributes. To search with that point, you can then use one of the
       # following syntax examples:
-      #
-      #   Address.search "Melbourne", :geo => [1.4, -2.217]
-      #   Address.search "Australia", :geo => [-0.55, 3.108],
+      # 
+      #   Address.search "Melbourne", :geo => [1.4, -2.217], :order => "@geodist asc"
+      #   Address.search "Australia", :geo => [-0.55, 3.108], :order => "@geodist asc"
       #     :latitude_attr => "latit", :longitude_attr => "longit"
-      #
+      # 
       # The first example applies when your latitude and longitude attributes
       # are named any of lat, latitude, lon, long or longitude. If that's not
       # the case, you will need to explicitly state them in your search, _or_
@@ -168,17 +294,17 @@ module ThinkingSphinx
       #   define_index do
       #     has :latit  # Float column, stored in radians
       #     has :longit # Float column, stored in radians
-      #
+      #     
       #     set_property :latitude_attr   => "latit"
       #     set_property :longitude_attr  => "longit"
       #   end
-      #
+      # 
       # Now, geo-location searching really only has an affect if you have a
       # filter, sort or grouping clause related to it - otherwise it's just a
-      # normal search. To make use of the positioning difference, use the
-      # special attribute "@geodist" in any of your filters or sorting or grouping
-      # clauses.
-      #
+      # normal search, and _will not_ return a distance value otherwise. To
+      # make use of the positioning difference, use the special attribute
+      # "@geodist" in any of your filters or sorting or grouping clauses.
+      # 
       # And don't forget - both the latitude and longitude you use in your
       # search, and the values in your indexes, need to be stored as a float in radians,
       # _not_ degrees. Keep in mind that if you do this conversion in SQL
@@ -188,40 +314,93 @@ module ThinkingSphinx
       #     has 'RADIANS(lat)', :as => :lat,  :type => :float
       #     # ...
       #   end
+      # 
+      # Once you've got your results set, you can access the distances as
+      # follows:
+      # 
+      # @results.each_with_geodist do |result, distance|
+      #   # ...
+      # end
+      # 
+      # The distance value is returned as a float, representing the distance in
+      # metres.
+      # 
+      # == Handling a Stale Index
+      #
+      # Especially if you don't use delta indexing, you risk having records in the
+      # Sphinx index that are no longer in the database. By default, those will simply
+      # come back as nils:
       #
+      #   >> pat_user.delete
+      #   >> User.search("pat")
+      #   Sphinx Result: [1,2]
+      #   => [nil, <#User id: 2>]
+      #
+      # (If you search across multiple models, you'll get ActiveRecord::RecordNotFound.)
+      #
+      # You can simply Array#compact these results or handle the nils in some other way, but
+      # Sphinx will still report two results, and the missing records may upset your layout.
+      #
+      # If you pass :retry_stale => true to a single-model search, missing records will
+      # cause Thinking Sphinx to retry the query but excluding those records. Since search
+      # is paginated, the new search could potentially include missing records as well, so by
+      # default Thinking Sphinx will retry three times. Pass :retry_stale => 5 to retry five
+      # times, and so on. If there are still missing ids on the last retry, they are
+      # shown as nils.
+      # 
       def search(*args)
-        results, client = search_results(*args.clone)
-
-        ::ActiveRecord::Base.logger.error(
-          "Sphinx Error: #{results[:error]}"
-        ) if results[:error]
-
-        options = args.extract_options!
-        klass   = options[:class]
-        page    = options[:page] ? options[:page].to_i : 1
-
-        # begin
-          pager = ThinkingSphinx::Collection.new(page, client.limit,
-            results[:total] || 0, results[:total_found] || 0)
-          pager.replace instances_from_results(results[:matches], options, klass)
-          # pager = WillPaginate::Collection.create(page,
-          #   client.limit, results[:total] || 0) do |collection|
-          #   collection.replace instances_from_results(results[:matches], options, klass)
-          #   collection.instance_variable_set :@total_entries, results[:total_found]
-          # end
-          return (options[:include_raw] ? [pager, results] : pager)
-        # rescue StandardError => err
-        #   if options[:include_raw]
-        #     return instances_from_results(results[:matches], options, klass), results
-        #   else
-        #     return instances_from_results(results[:matches], options, klass)
-        #   end
+        query = args.clone  # an array
+        options = query.extract_options!
+        
+        retry_search_on_stale_index(query, options) do
+          results, client = search_results(*(query + [options]))
+        
+          ::ActiveRecord::Base.logger.error(
+            "Sphinx Error: #{results[:error]}"
+          ) if results[:error]
+        
+          klass   = options[:class]
+          page    = options[:page] ? options[:page].to_i : 1
+        
+          ThinkingSphinx::Collection.create_from_results(results, page, client.limit, options)
+        end
+      end
+      
+      def retry_search_on_stale_index(query, options, &block)
+        stale_ids = []
+        stale_retries_left = case options[:retry_stale]
+                              when true
+                                3  # default to three retries
+                              when nil, false
+                                0  # no retries
+                              else             options[:retry_stale].to_i
+                              end
+        begin
+          # Passing this in an option so Collection.create_from_results can see it.
+          # It should only raise on stale records if there are any retries left.
+          options[:raise_on_stale] = stale_retries_left > 0
+          block.call
+        # If ThinkingSphinx::Collection.create_from_results found records in Sphinx but not
+        # in the DB and the :raise_on_stale option is set, this exception is raised. We retry
+        # a limited number of times, excluding the stale ids from the search.
+        rescue StaleIdsException => e
+          stale_retries_left -= 1
+
+          stale_ids |= e.ids  # For logging
+          options[:without_ids] = Array(options[:without_ids]) | e.ids  # Actual exclusion
+
+          tries = stale_retries_left
+          ::ActiveRecord::Base.logger.debug("Sphinx Stale Ids (%s %s left): %s" % [
+              tries, (tries==1 ? 'try' : 'tries'), stale_ids.join(', ')
+          ])
+          
+          retry
         end
       end
 
       def count(*args)
         results, client = search_results(*args.clone)
-        results[:total] || 0
+        results[:total_found] || 0
       end
 
       # Checks if a document with the given id exists within a specific index.
@@ -230,50 +409,70 @@ module ThinkingSphinx
       # - ID of the document
       # - Index to check within
       # - Options hash (defaults to {})
-      #
+      # 
       # Example:
-      #
+      # 
       #   ThinkingSphinx::Search.search_for_id(10, "user_core", :class => User)
-      #
+      # 
       def search_for_id(*args)
         options = args.extract_options!
         client  = client_from_options options
-
+        
         query, filters    = search_conditions(
           options[:class], options[:conditions] || {}
         )
         client.filters   += filters
         client.match_mode = :extended unless query.empty?
         client.id_range   = args.first..args.first
-
+        
         begin
           return client.query(query, args[1])[:matches].length > 0
         rescue Errno::ECONNREFUSED => err
           raise ThinkingSphinx::ConnectionError, "Connection to Sphinx Daemon (searchd) failed."
         end
       end
-
+      
+      # Model.facets *args
+      # ThinkingSphinx::Search.facets *args
+      # ThinkingSphinx::Search.facets *args, :all_attributes  => true
+      # ThinkingSphinx::Search.facets *args, :class_facet     => false
+      # 
+      def facets(*args)
+        options = args.extract_options!
+        
+        if options[:class]
+          facets_for_model options[:class], args, options
+        else
+          facets_for_all_models args, options
+        end
+      end
+      
       private
-
+      
       # This method handles the common search functionality, and returns both
       # the result hash and the client. Not super elegant, but it'll do for
       # the moment.
-      #
+      # 
       def search_results(*args)
         options = args.extract_options!
+        query   = args.join(' ')
         client  = client_from_options options
-
-        query, filters    = search_conditions(
+        
+        query = star_query(query, options[:star]) if options[:star]
+        
+        extra_query, filters = search_conditions(
           options[:class], options[:conditions] || {}
         )
         client.filters   += filters
-        client.match_mode = :extended unless query.empty?
-        query             = args.join(" ") + query
-
+        client.match_mode = :extended unless extra_query.empty?
+        query             = [query, extra_query].join(' ')
+        query.strip!  # Because "" and " " are not equivalent
+                
         set_sort_options! client, options
-
+        
         client.limit  = options[:per_page].to_i if options[:per_page]
         page          = options[:page] ? options[:page].to_i : 1
+        page          = 1 if page <= 0
         client.offset = (page - 1) * client.limit
 
         begin
@@ -283,74 +482,42 @@ module ThinkingSphinx
         rescue Errno::ECONNREFUSED => err
           raise ThinkingSphinx::ConnectionError, "Connection to Sphinx Daemon (searchd) failed."
         end
-
+        
         return results, client
       end
-
-      # This function loops over the records and appends a 'distance' variable to each one with
-      # the value from Sphinx
-      def append_distances(instances, results, distance_name)
-        instances.each_with_index do |record, index|
-          if record
-            distance = (results[index][:attributes]['@geodist'] or nil)
-            record.instance_variable_get('@attributes')["#{distance_name}"] = distance
-          end
-        end
-      end
-
-      def instances_from_results(results, options = {}, klass = nil)
-        if klass.nil?
-          results.collect { |result| instance_from_result result, options }
-        else
-          ids = results.collect { |result| result[:attributes]["sphinx_internal_id"] }
-          instances = ids.length > 0 ? klass.find(
-            :all,
-            :conditions => {klass.primary_key.to_sym => ids},
-            :include    => options[:include],
-            :select     => options[:select]
-          ) : []
-          final_instances = ids.collect { |obj_id| instances.detect { |obj| obj.id == obj_id } }
-
-          final_instances = append_distances(final_instances, results, options[:distance_name]) if options[:distance_name] && (results.collect { |result| result[:attributes]['@geodist'] }.length > 0)
-
-          return final_instances
-        end
-      end
-
-      # Either use the provided class to instantiate a result from a model, or
-      # get the result's CRC value and determine the class from that.
-      #
-      def instance_from_result(result, options)
-        class_from_crc(result[:attributes]["class_crc"]).find(
-          result[:attributes]["sphinx_internal_id"],
-          :include => options[:include], :select => options[:select]
-        )
-      end
-
-      # Convert a CRC value to the corresponding class.
-      #
-      def class_from_crc(crc)
-        unless @models_by_crc
-          Configuration.new.load_models
-
-          @models_by_crc = ThinkingSphinx.indexed_models.inject({}) do |hash, model|
-            hash[model.constantize.to_crc32] = model
-            hash
-          end
-        end
-
-        @models_by_crc[crc].constantize
-      end
-
+      
       # Set all the appropriate settings for the client, using the provided
       # options hash.
       # 
       def client_from_options(options = {})
-        config = ThinkingSphinx::Configuration.new
+        config = ThinkingSphinx::Configuration.instance
         client = Riddle::Client.new config.address, config.port
         klass  = options[:class]
-        index_options = klass ? klass.indexes.last.options : {}
-
+        index_options = klass ? klass.sphinx_index_options : {}
+
+        # The Riddle default is per-query max_matches=1000. If we set the
+        # per-server max to a smaller value in sphinx.yml, we need to override
+        # the Riddle default or else we get search errors like
+        # "per-query max_matches=1000 out of bounds (per-server max_matches=200)"
+        if per_server_max_matches = config.configuration.searchd.max_matches
+          options[:max_matches] ||= per_server_max_matches
+        end
+        
+        # Turn :index_weights => { "foo" => 2, User => 1 }
+        # into :index_weights => { "foo" => 2, "user_core" => 1, "user_delta" => 1 }
+        if iw = options[:index_weights]
+          options[:index_weights] = iw.inject({}) do |hash, (index,weight)|
+            if index.is_a?(Class)
+              name = ThinkingSphinx::Index.name(index)
+              hash["#{name}_core"]  = weight
+              hash["#{name}_delta"] = weight
+            else
+              hash[index] = weight
+            end
+            hash
+          end
+        end
+        
         [
           :max_matches, :match_mode, :sort_mode, :sort_by, :id_range,
           :group_by, :group_function, :group_clause, :group_distinct, :cut_off,
@@ -366,116 +533,160 @@ module ThinkingSphinx
         options[:classes] = [klass] if klass
         
         client.anchor = anchor_conditions(klass, options) || {} if client.anchor.empty?
-
+        
         client.filters << Riddle::Client::Filter.new(
           "sphinx_deleted", [0]
         )
         
         # class filters
         client.filters << Riddle::Client::Filter.new(
-          "subclass_crcs", options[:classes].collect { |k| k.to_crc32s }.flatten
+          "class_crc", options[:classes].collect { |k| k.to_crc32s }.flatten
         ) if options[:classes]
-
+        
         # normal attribute filters
         client.filters += options[:with].collect { |attr,val|
           Riddle::Client::Filter.new attr.to_s, filter_value(val)
         } if options[:with]
-
+        
         # exclusive attribute filters
         client.filters += options[:without].collect { |attr,val|
           Riddle::Client::Filter.new attr.to_s, filter_value(val), true
         } if options[:without]
-
+        
+        # every-match attribute filters
+        client.filters += options[:with_all].collect { |attr,vals|
+          Array(vals).collect { |val|
+            Riddle::Client::Filter.new attr.to_s, filter_value(val)
+          }
+        }.flatten if options[:with_all]
+        
+        # exclusive attribute filter on primary key
+        client.filters += Array(options[:without_ids]).collect { |id|
+          Riddle::Client::Filter.new 'sphinx_internal_id', filter_value(id), true
+        } if options[:without_ids]
+        
         client
       end
-
+      
+      def star_query(query, custom_token = nil)
+        token = custom_token.is_a?(Regexp) ? custom_token : /\w+/u
+
+        query.gsub(/("#{token}(.*?#{token})?"|(?![!-])#{token})/u) do
+          pre, proper, post = $`, $&, $'
+          is_operator = pre.match(%r{(\W|^)[@~/]\Z})  # E.g. "@foo", "/2", "~3", but not as part of a token
+          is_quote    = proper.starts_with?('"') && proper.ends_with?('"')  # E.g. "foo bar", with quotes
+          has_star    = pre.ends_with?("*") || post.starts_with?("*")
+          if is_operator || is_quote || has_star
+            proper
+          else
+            "*#{proper}*"
+          end
+        end
+      end
+      
       def filter_value(value)
         case value
         when Range
-          value.first.is_a?(Time) ? value.first.to_i..value.last.to_i : value
+          value.first.is_a?(Time) ? timestamp(value.first)..timestamp(value.last) : value
         when Array
-          value.collect { |val| val.is_a?(Time) ? val.to_i : val }
+          value.collect { |val| val.is_a?(Time) ? timestamp(val) : val }
         else
           Array(value)
         end
       end
-
+      
+      # Returns the integer timestamp for a Time object.
+      # 
+      # If using Rails 2.1+, need to handle timezones to translate them back to
+      # UTC, as that's what datetimes will be stored as by MySQL.
+      # 
+      # in_time_zone is a method that was added for the timezone support in
+      # Rails 2.1, which is why it's used for testing. I'm sure there's better
+      # ways, but this does the job.
+      # 
+      def timestamp(value)
+        value.respond_to?(:in_time_zone) ? value.utc.to_i : value.to_i
+      end
+      
       # Translate field and attribute conditions to the relevant search string
       # and filters.
-      #
+      # 
       def search_conditions(klass, conditions={})
-        attributes = klass ? klass.indexes.collect { |index|
+        attributes = klass ? klass.sphinx_indexes.collect { |index|
           index.attributes.collect { |attrib| attrib.unique_name }
         }.flatten : []
-
-        search_string = ""
+        
+        search_string = []
         filters       = []
-
+        
         conditions.each do |key,val|
           if attributes.include?(key.to_sym)
             filters << Riddle::Client::Filter.new(
               key.to_s, filter_value(val)
             )
           else
-            search_string << "@#{key} #{val} "
+            search_string << "@#{key} #{val}"
           end
         end
         
-        return search_string, filters
+        return search_string.join(' '), filters
       end
-
+      
       # Return the appropriate latitude and longitude values, depending on
       # whether the relevant attributes have been defined, and also whether
       # there's actually any values.
-      #
+      # 
       def anchor_conditions(klass, options)
-        attributes = klass ? klass.indexes.collect { |index|
+        attributes = klass ? klass.sphinx_indexes.collect { |index|
           index.attributes.collect { |attrib| attrib.unique_name }
         }.flatten : []
-
-        lat_attr = klass ? klass.indexes.collect { |index|
+        
+        lat_attr = klass ? klass.sphinx_indexes.collect { |index|
           index.options[:latitude_attr]
         }.compact.first : nil
-
-        lon_attr = klass ? klass.indexes.collect { |index|
+        
+        lon_attr = klass ? klass.sphinx_indexes.collect { |index|
           index.options[:longitude_attr]
         }.compact.first : nil
-
+        
         lat_attr = options[:latitude_attr] if options[:latitude_attr]
         lat_attr ||= :lat       if attributes.include?(:lat)
         lat_attr ||= :latitude  if attributes.include?(:latitude)
-
+        
         lon_attr = options[:longitude_attr] if options[:longitude_attr]
+        lon_attr ||= :lng       if attributes.include?(:lng)
         lon_attr ||= :lon       if attributes.include?(:lon)
         lon_attr ||= :long      if attributes.include?(:long)
         lon_attr ||= :longitude if attributes.include?(:longitude)
-
+        
         lat = options[:lat]
         lon = options[:lon]
-
+        
         if options[:geo]
           lat = options[:geo].first
           lon = options[:geo].last
         end
-
+        
         lat && lon ? {
-          :latitude_attribute   => lat_attr,
+          :latitude_attribute   => lat_attr.to_s,
           :latitude             => lat,
-          :longitude_attribute  => lon_attr,
+          :longitude_attribute  => lon_attr.to_s,
           :longitude            => lon
         } : nil
       end
-
+      
       # Set the sort options using the :order key as well as the appropriate
       # Riddle settings.
-      #
+      # 
       def set_sort_options!(client, options)
         klass = options[:class]
-        fields = klass ? klass.indexes.collect { |index|
+        fields = klass ? klass.sphinx_indexes.collect { |index|
           index.fields.collect { |field| field.unique_name }
         }.flatten : []
+        index_options = klass ? klass.sphinx_index_options : {}
 
-        case order = options[:order]
+        order = options[:order] || index_options[:order]        
+        case order
         when Symbol
           client.sort_mode = :attr_asc if client.sort_mode == :relevance || client.sort_mode.nil?
           if fields.include?(order)
@@ -489,23 +700,81 @@ module ThinkingSphinx
         else
           # do nothing
         end
-
+        
         client.sort_mode = :attr_asc  if client.sort_mode == :asc
         client.sort_mode = :attr_desc if client.sort_mode == :desc
       end
-
+      
       # Search through a collection of fields and translate any appearances
       # of them in a string to their attribute equivalent for sorting.
-      #
+      # 
       def sorted_fields_to_attributes(string, fields)
         fields.each { |field|
           string.gsub!(/(^|\s)#{field}(,?\s|$)/) { |match|
             match.gsub field.to_s, field.to_s.concat("_sort")
           }
         }
-
+        
         string
       end
+      
+      def facets_for_model(klass, args, options)
+        hash    = ThinkingSphinx::FacetCollection.new args + [options]
+        options = options.clone.merge! :group_function => :attr
+        
+        klass.sphinx_facets.inject(hash) do |hash, facet|
+          unless facet.name == :class && !options[:class_facet]
+            options[:group_by] = facet.attribute_name
+            hash.add_from_results facet, search(*(args + [options]))
+          end
+          
+          hash
+        end
+      end
+      
+      def facets_for_all_models(args, options)
+        options = GlobalFacetOptions.merge(options)
+        hash    = ThinkingSphinx::FacetCollection.new args + [options]
+        options = options.merge! :group_function => :attr
+        
+        facet_names(options).inject(hash) do |hash, name|
+          options[:group_by] = name
+          hash.add_from_results name, search(*(args + [options]))
+          hash
+        end
+      end
+      
+      def facet_classes(options)
+        options[:classes] || ThinkingSphinx.indexed_models.collect { |model|
+          model.constantize
+        }
+      end
+      
+      def facet_names(options)
+        classes = facet_classes(options)
+        names   = options[:all_attributes] ?
+          facet_names_for_all_classes(classes) :
+          facet_names_common_to_all_classes(classes)
+        
+        names.delete "class_crc" unless options[:class_facet]
+        names
+      end
+      
+      def facet_names_for_all_classes(classes)
+        classes.collect { |klass|
+          klass.sphinx_facets.collect { |facet| facet.attribute_name }
+        }.flatten.uniq
+      end
+      
+      def facet_names_common_to_all_classes(classes)
+        facet_names_for_all_classes(classes).select { |name|
+          classes.all? { |klass|
+            klass.sphinx_facets.detect { |facet|
+              facet.attribute_name == name
+            }
+          }
+        }
+      end
     end
   end
-end
+end