ultrasphinx 1 → 1.5

data/lib/ultrasphinx/core_extensions.rb

@@ -1,6 +1,4 @@
 
-require 'chronic'
-
 class Array
   def _flatten_once
     self.inject([]) do |set, element| 
@@ -15,6 +13,11 @@ class Object
       self
     end
   end
+  
+  def _deep_dup
+    # Cause Ruby's dup sucks.
+    Marshal.load(Marshal.dump(self))
+  end
 end
 
 class String
@@ -36,16 +39,45 @@ end
 class Hash
   def _coerce_basic_types
     Hash[*self.map do |key, value|
-      [key.to_sym,
+      [key.to_s,
         if value.respond_to?(:to_i) && value.to_i.to_s == value
           value.to_i
         elsif value == ""
           nil
-        elsif value.is_a? String
-          value.to_sym
         else
           value
         end]
       end._flatten_once]
   end
+  
+  def _to_conf_string(section = nil)
+    inner = self.map do |key, value|
+      "  #{key} = #{value}"
+    end.join("\n")
+    section ? "#{section} {\n#{inner}\n}\n" : inner
+  end
+  
+  def _deep_stringify_keys
+    Hash[*(self.map do |key, value|
+#      puts "#{key.inspect}, #{value.inspect}"
+      z = [key.to_s,
+        case value
+          when Hash
+            value._deep_stringify_keys
+          when Array
+            value.map do |subvalue|
+              if subvalue.is_a? Hash or subvalue.is_a? Array
+                subvalue._deep_stringify_keys
+              else
+                subvalue
+              end
+            end
+          else
+            value
+        end
+      ]
+#      p z
+#      z
+    end._flatten_once)]
+  end
 end

data/lib/ultrasphinx/fields.rb

@@ -3,26 +3,58 @@ require 'singleton'
 
 module Ultrasphinx
 
-  class Fields < Hash
+  class Fields
     include Singleton
     
+    TYPE_MAP = {
+      'string' => 'text', 
+      'text' => 'text', 
+      'integer' => 'numeric', 
+      'date' => 'date', 
+      'datetime' => 'date'
+    }    
+    
+    attr_accessor :classes, :types
+    
     def initialize
-      self["class_id"] = "numeric"
-      self["class"] = "text"
+      @types = {}
+      @classes = Hash.new([])
+      @groups = []
+    end
+    
+    def groups
+      @groups.compact.sort_by do |string| 
+        string[/= (.*)/, 1]
+      end
     end
   
-    def check_type_match(field, new_type)
-      # tries to smoosh fields together by name in the sphinx query schema; raises if their types don't match
-      field, new_type = field.to_s, COLUMN_TYPES[new_type.to_sym]
-      if self[field]
-        raise ConfigurationError, "Column type mismatch for #{field.inspect}; was already #{self[field].inspect}, but is now #{new_type.inspect}." unless self[field] == new_type
+    def save_and_verify_type(field, new_type, string_sortable, klass)
+      # Smoosh fields together based on their name in the Sphinx query schema
+      field, new_type = field.to_s, TYPE_MAP[new_type.to_s]
+
+      if types[field]
+        # Existing field name; verify its type
+        raise ConfigurationError, "Column type mismatch for #{field.inspect}; was already #{types[field].inspect}, but is now #{new_type.inspect}." unless types[field] == new_type
+        classes[field] = (classes[field] + [klass]).uniq
+
       else
-        self[field] = new_type
+        # New field      
+        types[field] = new_type
+        classes[field] = [klass]
+
+        @groups << case new_type
+          when 'numeric'
+            "sql_group_column = #{field}"
+          when 'date'
+            "sql_date_column = #{field}"
+          when 'text' 
+            "sql_str2ordinal_column = #{field}" if string_sortable
+        end
       end
     end
     
     def cast(source_string, field)
-      if self[field] == "date"
+      if types[field] == "date"
         "UNIX_TIMESTAMP(#{source_string})"
       elsif source_string =~ /GROUP_CONCAT/
         "CAST(#{source_string} AS CHAR)"
@@ -32,38 +64,56 @@ module Ultrasphinx
     end    
       
     def null(field)
-      case self[field]
+      case types[field]
         when 'text'
           "''"
         when 'numeric'
           "0"
         when 'date'
           "UNIX_TIMESTAMP('1970-01-01 00:00:00')"
+        else
+          raise "Field #{field} does not have a valid type."
       end + " AS #{field}"
     end
     
     def configure(configuration)
 
       configuration.each do |model, options|        
-        klass = model.constantize
+
+        klass = model.constantize        
+        save_and_verify_type('class_id', 'integer', nil, klass)
+        save_and_verify_type('class', 'string', nil, klass)
                 
         begin
-          # fields are from the model
-          options[:fields].to_a.each do |entry|
-            entry = {:field => entry, :as => entry} unless entry.is_a? Hash
-            unless klass.columns_hash[entry[:field]]
-              ActiveRecord::Base.logger.warn "ultrasphinx: WARNING: field #{entry[:field]} is not present in #{model}"
+        
+          # Fields are from the model. We destructively canonicize them back onto the configuration hash.
+          options['fields'] = options['fields'].to_a.map do |entry|
+            
+            entry = {'field' => entry} unless entry.is_a? Hash
+            entry['as'] = entry['field'] unless entry['as']
+            
+            unless klass.columns_hash[entry['field']]
+              ActiveRecord::Base.logger.warn "ultrasphinx: WARNING: field #{entry['field']} is not present in #{model}"
             else
-              check_type_match(entry[:as], klass.columns_hash[entry[:field]].type)
+              save_and_verify_type(entry['as'], klass.columns_hash[entry['field']].type, entry['sortable'], klass)
+            end
+            
+            if entry['facet']
+              save_and_verify_type(entry['as'], 'text', nil, klass) # source must be a string
+              save_and_verify_type("#{entry['as']}_facet", 'integer', nil, klass)
             end
+            
+            entry
           end  
-          # joins are whatever they are in the target       
-          options[:includes].to_a.each do |join|
-            check_type_match(join[:as] || join[:field], join[:model].constantize.columns_hash[join[:field]].type)
+          
+          # Joins are whatever they are in the target       
+          options['include'].to_a.each do |entry|
+            save_and_verify_type(entry['as'] || entry['field'], entry['class_name'].constantize.columns_hash[entry['field']].type, entry['sortable'], klass)
           end  
-          # regular concats are CHAR (I think), group_concats are BLOB and need to be cast to CHAR, e.g. :text
-          options[:concats].to_a.each do |concats|
-            check_type_match(concats[:as], :text)          
+          
+          # Regular concats are CHAR (I think), group_concats are BLOB and need to be cast to CHAR, e.g. :text
+          options['concatenate'].to_a.each do |entry|
+            save_and_verify_type(entry['as'], 'text', entry['sortable'], klass)
           end          
         rescue ActiveRecord::StatementInvalid
           ActiveRecord::Base.logger.warn "ultrasphinx: WARNING: model #{model} does not exist in the database yet"

data/lib/ultrasphinx/is_indexed.rb

@@ -6,24 +6,41 @@ module ActiveRecord
 
 =begin rdoc
 
-The is_indexed macro configures a model for indexing. Its parameters are used to generate SQL queries for Sphinx.
+The is_indexed method configures a model for indexing. Its parameters help generate SQL queries for Sphinx.
 
-== Indexing single fields
+= Options
+
+== Including regular fields
 
 Use the <tt>:fields</tt> key.
 
-Accepts an array of field names. 
-  :fields => ["created_at", "title", "body"]
+Accepts an array of field names or field hashes. 
+  :fields => [
+    'created_at', 
+    'title', 
+    {:field => 'body', :as => 'description'},
+    {:field => 'user_category', :facet => true, :as => 'category' }
+  ]
+  
+To alias a field, pass a hash instead of a string and set the <tt>:as</tt> key. 
+
+To allow faceting support on a text field, also pass a hash and set the <tt>:facet</tt> key to <tt>true</tt>. Faceting is off by default for text fields because there is some indexing overhead associated with it. Faceting is always on for numeric or date fields.
 
-== Indexing fields from belongs_to associations
+To allow sorting by a text field, also pass a hash and set the <tt>:sortable</tt> key to true. This is turned off by default for the same reason as above. Sorting is always on for numeric or date fields.
 
-Use the <tt>:includes</tt> key.
+To apply an SQL function to a field before it is indexed, use the key <tt>:function_sql</tt>. Pass a string such as <tt>"REPLACE(?, '_', ' ')"</tt>. The table and column name for your field will be interpolated into the first <tt>?</tt> in the string.
+
+== Including a field from an association
+
+Use the <tt>:include</tt> key.
 
 Accepts an array of hashes. 
 
-Each should contain a <tt>:model</tt> key (the class name of the included model), a <tt>:field</tt> key (the name of the field to include), and an optional <tt>:as</tt> key (what to name the field in the parent). You can use the optional key <tt>:association_sql</tt> if you need to pass a custom JOIN string, in which case the default JOIN will not be generated.
+Each should contain a <tt>:class_name</tt> key (the class name of the included model), a <tt>:field</tt> key (the name of the field to include), and an optional <tt>:as</tt> key (what to name the field in the parent). You can use the optional key <tt>:association_sql</tt> if you need to pass a custom JOIN string, in which case the default JOIN for <tt>belongs_to</tt> will not be generated.
+
+The keys <tt>:facet</tt>, <tt>:sortable</tt>, and <tt>:function_sql</tt> are also recognized, just like for regular fields.
 
-== Scoping the searchable records
+== Requiring conditions
 
 Use the <tt>:conditions</tt> key.
 
@@ -31,56 +48,95 @@ SQL conditions, to scope which records are selected for indexing. Accepts a stri
   :conditions => "created_at < NOW() AND deleted IS NOT NULL"
 The <tt>:conditions</tt> key is especially useful if you delete records by marking them deleted rather than removing them from the database.
 
-== Concatenating multiple fields
+== Concatenating several fields within a record
 
-Use the <tt>:concats</tt> key (MySQL only).
+Use the <tt>:concatenate</tt> key (MySQL only).
 
-Accepts an array of option hashes, which can be of two types: 
+Accepts an array of option hashes. 
 
-1. To concatenate many fields within one record, use a regular (or horizontal) concatenation. Regular concatenations contain a <tt>:fields</tt> key (again, an array of field names), and a mandatory <tt>:as</tt> key (the name of the result of the concatenation). For example, to concatenate the <tt>title</tt> and <tt>body</tt> into one field called <tt>text</tt>: 
-  :concats => [{:fields => ["title", "body"], :as => "text"}]
+To concatenate several fields within one record as a combined field, use a regular (or horizontal) concatenation. Regular concatenations contain a <tt>:fields</tt> key (again, an array of field names), and a mandatory <tt>:as</tt> key (the name of the result of the concatenation). For example, to concatenate the <tt>title</tt> and <tt>body</tt> into one field called <tt>text</tt>: 
+  :concatenate => [{:fields => ['title', 'body'], :as => 'text'}]
+  
+The keys <tt>:facet</tt>, <tt>:sortable</tt>, and <tt>:function_sql</tt> are also recognized, just like for regular fields.
+
+== Concatenating one field from a set of associated records 
+
+Also use the <tt>:concatenate</tt> key.
 
-2. To group and concatenate a field from a set of associated records, use a group (or vertical) concatenation. Group concatenations join into another table, and can be used to index a number of associated models as one field in a parent model. Group concatenations contain a <tt>:model</tt> key (the class name of the included model), a <tt>:field</tt> key (the field on the included model to concatenate), and an optional <tt>:as</tt> key (also the name of the result of the concatenation). For example, to concatenate all <tt>Post#body</tt> contents into the parent's <tt>responses</tt> field:
-  :concats => {:model => "Post", :field => "body", :as => "responses"}
+To concatenate one field from a set of associated records as a combined field in the parent record, use a group (or vertical) concatenation. A group concatenation should contain a <tt>:class_name</tt> key (the class name of the included model), a <tt>:field</tt> key (the field on the included model to concatenate), and an optional <tt>:as</tt> key (also the name of the result of the concatenation). For example, to concatenate all <tt>Post#body</tt> contents into the parent's <tt>responses</tt> field:
+  :concatenate => [{:class_name => 'Post', :field => 'body', :as => 'responses'}]
 
 Optional group concatenation keys are <tt>:association_name</tt> (if your <tt>has_many</tt> association can't be derived from the model name), <tt>:association_sql</tt>, if you need to pass a custom JOIN string (for example, a double JOIN for a <tt>has_many :through</tt>), and <tt>:conditions</tt> (if you need custom WHERE conditions for this particular association).
 
-== Example
+The keys <tt>:facet</tt>, <tt>:sortable</tt>, and <tt>:function_sql</tt> are also recognized, just like for regular fields.
+
+Ultrasphinx is not an object-relational mapper, and the association generation is intended to stay minimal--don't be afraid of <tt>:association_sql</tt>.
+
+= Examples
+
+== Complex configuration
 
 Here's an example configuration using most of the options, taken from production code:
 
   class Story < ActiveRecord::Base  
     is_indexed :fields => [
-        "title", 
-        "published_at"
+        'title', 
+        'published_at',
+        {:field => 'author', :facet => true}
       ],
-      :includes => [
-        {:model => "Category", :field => "name", :as => "category"}
+      :include => [
+        {:class_name => 'Category', :field => 'name', :as => 'category'}
       ],      
-      :concats => [
-        {:fields => ["title", "long_description", "short_description"], :as => "editorial"},
-        {:model => "Page", :field => "body", :as => "body", :association_name => "pages"},
-        {:model => "Comment", :field => "body", :as => "comments", 
+      :concatenate => [
+        {:fields => ['title', 'long_description', 'short_description'], 
+          :as => 'editorial'},
+        {:class_name => 'Page', :field => 'body', :as => 'body', 
+          :association_name => 'pages'},
+        {:class_name => 'Comment', :field => 'body', :as => 'comments', 
           :conditions => "comments.item_type = '#{base_class}'"}
       ],
       :conditions => self.live_condition_string
   end  
 
+Note how setting the <tt>:conditions</tt> on Comment is enough to configure a polymorphic <tt>has_many</tt>.
+
+== Association scoping
+
+A common use case is to only search records that belong to a particular parent model. Ultrasphinx configures Sphinx to support a <tt>:filter</tt> element on any date or numeric field, so any <tt>*_id</tt> fields you have will be filterable.
+
+For example, say a Company <tt>has_many :users</tt> and each User <tt>has_many :articles</tt>. If you want to to filter Articles by Company, add <tt>company_id</tt> to the Article's <tt>is_indexed</tt> method. The best way is to grab it from the User association:
+
+  class Article < ActiveRecord::Base 
+     is_indexed :include => [{:class_name => 'User', :field => 'company_id'}]
+  end
+ 
+Now you can run:
+
+ @search = Ultrasphinx::Search.new('something', 
+   :filter => {'company_id' => 493})
+ 
+If the associations weren't just <tt>has_many</tt> and <tt>belongs_to</tt>, you would need to use the <tt>:association_sql</tt> key to set up a custom JOIN. 
+
 =end
   
-    def self.is_indexed opts = {}
-    
-      opts.assert_valid_keys [:fields, :concats, :conditions, :includes, :nulls]
+    def self.is_indexed opts = {}    
+      opts = opts._deep_stringify_keys
+          
+      opts.assert_valid_keys ['fields', 'concatenate', 'conditions', 'include']
+      
+      Array(opts[:fields]).each do |field|
+        field.assert_valid_keys ['field', 'as', 'facet', 'function_sql', 'sortable'] if field.is_a? Hash
+      end
       
-      Array(opts[:concats]).each do |concat|
-        concat.assert_valid_keys [:model, :conditions, :field, :as, :fields, :association_name, :association_sql]
-        raise Ultrasphinx::ConfigurationError, "You can't mix regular concat and group concats" if concat[:fields] and (concat[:field] or concat[:model] or concat[:association_name])
-        raise Ultrasphinx::ConfigurationError, "Group concats must not have multiple fields" if concat[:field].is_a? Array
-        raise Ultrasphinx::ConfigurationError, "Regular concats should have multiple fields" if concat[:fields] and !concat[:fields].is_a?(Array)
+      Array(opts[:concatenate]).each do |concat|
+        concat.assert_valid_keys ['class_name', 'conditions', 'field', 'as', 'fields', 'association_name', 'association_sql', 'facet', 'function_sql', 'sortable']
+        raise Ultrasphinx::ConfigurationError, "You can't mix regular concat and group concats" if concat['fields'] and (concat['field'] or concat['class_name'] or concat['association_name'])
+        raise Ultrasphinx::ConfigurationError, "Group concats must not have multiple fields" if concat['field'].is_a? Array
+        raise Ultrasphinx::ConfigurationError, "Regular concats should have multiple fields" if concat['fields'] and !concat['fields'].is_a?(Array)
       end
       
-      Array(opts[:joins]).each do |join|
-        join.assert_valid_keys [:model, :field, :as]
+      Array(opts[:include]).each do |inc|
+        inc.assert_valid_keys ['class_name', 'field', 'as', 'association_sql', 'facet', 'function_sql', 'sortable']
       end
       
       Ultrasphinx::MODEL_CONFIGURATION[self.name] = opts

data/lib/ultrasphinx/search.rb

@@ -4,40 +4,65 @@ module Ultrasphinx
 =begin rdoc
 Command-interface Search object.
 
-== Making a search
+== Basic usage
   
-To perform a search, instantiate an Ultrasphinx::Search object. Parameters are the query string, and an optional hash of query options.  
+To set up a search, instantiate an Ultrasphinx::Search object with a hash of parameters. Only the <tt>'query'</tt> key is mandatory.
   @search = Ultrasphinx::Search.new(
-    @query, 
-    :sort_mode => 'descending', 
-    :sort_by => 'created_at'
+    'query' => @query, 
+    'sort_mode' => 'descending', 
+    'sort_by' => 'created_at'
   )
     
-Now, to run the query, call its <tt>run()</tt> method. Your results will be available as ActiveRecord instances via <tt>results()</tt>. Example:  
+Now, to run the query, call its <tt>run</tt> method. Your results will be available as ActiveRecord instances via the <tt>results</tt> method. Example:  
   @search.run
   @search.results
 
-== Query options
-<tt>:per_page</tt>:: An integer.. How many results per page.
-<tt>:page</tt>:: An integer. Which page of the paginated results to return.
-<tt>:models</tt>:: An array or string. The class name of the model you want to search, an array of models names to search, or nil for all available models.    
-<tt>:sort_mode</tt>:: 'relevance' or 'ascending' or 'descending'. How to order the result set. Note that 'time' and 'extended' modes are available, but not tested.  
-<tt>:sort_by</tt>:: A field name. What field to order by for 'ascending' or 'descending' mode. Has no effect for 'relevance'.
-<tt>:weights</tt>:: A hash. Text-field names and associated query weighting. The default weight for every field is 1.0. Example: <tt>:weights => {"title" => 2.0}</tt>
-<tt>:raw_filters</tt>:: A hash. Field names and associated values. You can use a single value, an array of values, or a range. 
+= Options
+
+== Query format
+
+The query string supports boolean operation, parentheses, phrases, and field-specific search. Query words are stemmed and joined by an implicit <tt>AND</tt> by default.
+
+* Valid boolean operators are <tt>AND</tt>, <tt>OR</tt>, and <tt>NOT</tt>.
+* Field-specific searches should be formatted as <tt>fieldname:contents</tt>. (This will only work for text fields. For numeric and date fields, see the <tt>'filters</tt> parameter, below.)
+* Phrases must be enclosed in double quotes.
+    
+A Sphinx::SphinxInternalError will be raised on invalid queries. In general, queries can only be nested to one level. 
+  @query = 'dog OR cat OR "white tigers" NOT (lions OR bears) AND title:animals'
+
+== Hash parameters
+
+The hash lets you customize internal aspects of the search.
+
+<tt>'per_page'</tt>:: An integer. How many results per page.
+<tt>'page'</tt>:: An integer. Which page of the results to return.
+<tt>'class_name'</tt>:: An array or string. The class name of the model you want to search, an array of model names to search, or <tt>nil</tt> for all available models.    
+<tt>'sort_mode'</tt>:: 'relevance' or 'ascending' or 'descending'. How to order the result set. Note that 'time' and 'extended' modes are available, but not tested.  
+<tt>'sort_by'</tt>:: A field name. What field to order by for 'ascending' or 'descending' mode. Has no effect for 'relevance'.
+<tt>'weight'</tt>:: A hash. Text-field names and associated query weighting. The default weight for every field is 1.0. Example: <tt>'weight' => {'title' => 2.0}</tt>
+<tt>'filter'</tt>:: A hash. Names of numeric or date fields and associated values. You can use a single value, an array of values, or a range. (See the bottom of the ActiveRecord::Base page for an example.)
+<tt>'facets'</tt>:: An array of fields for grouping/faceting. You can access the returned facet values and their result counts with the <tt>facets</tt> method.
 
 Note that you can set up your own query defaults in <tt>environment.rb</tt>: 
   
   Ultrasphinx::Search.query_defaults = {
-    :per_page => 10,
-    :sort_mode => :relevance,
-    :weights => {"title" => 2.0}
+    'per_page' => 10,
+    'sort_mode' => 'relevance',
+    'weight' => {'title' => 2.0}
   }
 
+= Advanced features
+
 == Cache_fu integration
   
 The <tt>get_cache</tt> method will be used to instantiate records for models that respond to it. Otherwise, <tt>find</tt> is used.
 
+== Will_paginate integration
+
+The Search instance responds to the same methods as a WillPaginate::Collection object, so once you have called <tt>run</tt> or <tt>excerpt</tt> you can use it directly in your views:
+
+  will_paginate(@search)
+
 == Excerpt mode
 
 You can have Sphinx excerpt and highlight the matched sections in the associated fields. Instead of calling <tt>run</tt>, call <tt>excerpt</tt>. 
@@ -46,17 +71,17 @@ You can have Sphinx excerpt and highlight the matched sections in the associated
 
 The returned models will be frozen and have their field contents temporarily changed to the excerpted and highlighted results. 
   
-You need to set the <tt>content_methods</tt> key on Ultrasphinx::Search.excerpting_options to whatever methods you need the excerpter to try to excerpt. This way Ruby-only methods are supported (for example, a metadata method which combines various model fields, or an aliased field so that the original record contents are still available).
+You need to set the <tt>content_methods</tt> key on Ultrasphinx::Search.excerpting_options to whatever groups of methods you need the excerpter to try to excerpt. The first responding method in each group for each record will be excerpted. This way Ruby-only methods are supported (for example, a metadata method which combines various model fields, or an aliased field so that the original record contents are still available).
   
 There are some other keys you can set, such as excerpt size, HTML tags to highlight with, and number of words on either side of each excerpt chunk. Example (in <tt>environment.rb</tt>):
   
   Ultrasphinx::Search.excerpting_options = {
-    'before_match' => "<strong>", 
-    'after_match' => "</strong>",
+    'before_match' => '<strong>', 
+    'after_match' => '</strong>',
     'chunk_separator' => "...",
     'limit' => 256,
     'around' => 3,
-    'content_methods' => [[:title], [:body, :description, :content], [:metadata]] 
+    'content_methods' => [['title'], ['body', 'description', 'content'], ['metadata']] 
   }
   
 Note that your database is never changed by anything Ultrasphinx does.
@@ -64,16 +89,22 @@ Note that your database is never changed by anything Ultrasphinx does.
 =end    
 
   class Search  
-    unloadable if RAILS_ENV == "development"
+  
+    include Internals
+    include Parser
     
     cattr_accessor :query_defaults  
-    self.query_defaults ||= {:page => 1,
-      :models => nil,
-      :per_page => 20,
-      :sort_by => 'created_at',
-      :sort_mode => :relevance,
-      :weights => nil,
-      :raw_filters => nil}
+    self.query_defaults ||= {
+      'query' => nil,
+      'page' => 1,
+      'class_name' => nil,
+      'per_page' => 20,
+      'sort_by' => 'created_at',
+      'sort_mode' => 'relevance',
+      'weight' => nil,
+      'filter' => nil,
+      'facets' => nil
+    }
     
     cattr_accessor :excerpting_options
     self.excerpting_options ||= {
@@ -81,29 +112,35 @@ Note that your database is never changed by anything Ultrasphinx does.
       'chunk_separator' => "...",
       'limit' => 256,
       'around' => 3,
-      # results should respond to one in each group of these, in precedence order, in order for the excerpting to fire
-      'content_methods' => [[:title, :name], [:body, :description, :content], [:metadata]] 
+      # results should respond to one in each group of these, in precedence order, for the excerpting to fire
+      'content_methods' => [['title', 'name'], ['body', 'description', 'content'], ['metadata']] 
     }
     
     cattr_accessor :client_options
     self.client_options ||= { 
-      :with_subtotals => true, 
-      :max_retries => 4,
-      :retry_sleep_time => 3
+      'with_subtotals' => false, 
+      'max_retries' => 4,
+      'retry_sleep_time' => 3,
+      'max_facets' => 100,
+      'finder_methods' => ['get_cache', 'find']
     }
     
     # mode to integer mappings    
     SPHINX_CLIENT_PARAMS = { 
-      :sort_mode => {
-        :relevance => Sphinx::Client::SPH_SORT_RELEVANCE, 
-        :descending => Sphinx::Client::SPH_SORT_ATTR_DESC, 
-        :ascending => Sphinx::Client::SPH_SORT_ATTR_ASC, 
-        :time => Sphinx::Client::SPH_SORT_TIME_SEGMENTS,
-        :extended => Sphinx::Client::SPH_SORT_EXTENDED,
-        :desc => Sphinx::Client::SPH_SORT_ATTR_DESC, # legacy compatibility
-        :asc => Sphinx::Client::SPH_SORT_ATTR_ASC
+      'sort_mode' => {
+        'relevance' => Sphinx::Client::SPH_SORT_RELEVANCE, 
+        'descending' => Sphinx::Client::SPH_SORT_ATTR_DESC, 
+        'ascending' => Sphinx::Client::SPH_SORT_ATTR_ASC, 
+        'time' => Sphinx::Client::SPH_SORT_TIME_SEGMENTS,
+        'extended' => Sphinx::Client::SPH_SORT_EXTENDED,
+        'desc' => Sphinx::Client::SPH_SORT_ATTR_DESC, # legacy compatibility
+        'asc' => Sphinx::Client::SPH_SORT_ATTR_ASC
       }
     }
+    
+    LEGACY_QUERY_KEYS = ['raw_filters'] #:nodoc:
+    
+    INTERNAL_KEYS = ['parsed_query'] #:nodoc:
 
     def self.get_models_to_class_ids #:nodoc:
       # reading the conf file makes sure that we are in sync with the actual sphinx index,
@@ -132,30 +169,52 @@ Note that your database is never changed by anything Ultrasphinx does.
       
     MAX_MATCHES = DAEMON_SETTINGS["max_matches"].to_i 
     
-    # Returns the options hash you used.
-    def options; @options; end
+    FACET_CACHE = {} #:nodoc: 
+    
+    # Returns the options hash.
+    def options
+      @options
+    end
     
     #  Returns the query string used.
-    def query; @query; end
+    def query
+      # redundant with method_missing
+      @options['query']
+    end
+    
+    def parsed_query #:nodoc:
+      # redundant with method_missing
+      @options['parsed_query']
+    end
     
     # Returns an array of result objects.
-    def results; @results; end
+    def results
+      run?(true)
+      @results
+    end
+    
+    def facets
+      raise UsageError, "No facet field was configured" unless @options['facets']
+      run?(true)
+      @facets
+    end      
+      
     
     # Returns the raw response from the Sphinx client.
-    def response; @response; end
+    def response
+      @response
+    end
     
-    # Returns a hash of total result counts, scoped to each available model.
-    def subtotals; @subtotals; end
+    # Returns a hash of total result counts, scoped to each available model. This requires extra queries against the search daemon right now. Set <tt>Ultrasphinx::Search.client_options['with_subtotals'] = true</tt> to enable the extra queries. Most of the overhead is in instantiating the AR result sets, so the performance hit is not usually significant.
+    def subtotals
+      raise UsageError, "Subtotals are not enabled" unless self.class.client_options['with_subtotals']
+      @subtotals
+    end
 
     # Returns the total result count.
-    def total
+    def total_entries
       [response['total_found'] || 0, MAX_MATCHES].min
-    end
-  
-    # Returns the number of results on this particular page, and may range from 0 up to per_page().
-    def found
-      results.size
-    end
+    end  
   
     # Returns the response time of the query, in milliseconds.
     def time
@@ -163,60 +222,103 @@ Note that your database is never changed by anything Ultrasphinx does.
     end
 
     # Returns whether the query has been run.  
-    def run?
-      !response.blank?
+    def run?(should_raise = false)
+      if response.blank? and should_raise
+        raise UsageError, "Search has not yet been run" unless run?
+      else
+        !response.blank?
+      end
     end
  
     # Returns the current page number of the result set. (Page indexes begin at 1.) 
-    def page
-      options[:page]
+    def current_page
+      @options['page']
     end
   
     # Returns the number of records per page.
     def per_page
-      options[:per_page]
+      @options['per_page']
+    end
+    
+    # Clear the associated facet caches. They will be rebuilt on your next <tt>run</tt> or <tt>excerpt</tt>.
+    def clear_facet_caches
+      Array(@options['facets']).each do |facet|
+        FACET_CACHE.delete(facet)
+      end
     end
     
     # Returns the last available page number in the result set.  
-    def last_page
-      (total / per_page) + (total % per_page == 0 ? 0 : 1)
+    def page_count
+      (total_entries / per_page) + (total_entries % per_page == 0 ? 0 : 1)
+    end
+            
+    # Returns the previous page number.
+    def previous_page 
+      current_page > 1 ? (current_page - 1) : nil
+    end
+
+    # Returns the next page number.
+    def next_page
+      current_page < page_count ? (current_page + 1) : nil
     end
     
+    # Returns the global index position of the first result on this page.
+    def offset 
+      (current_page - 1) * per_page
+    end    
+    
     # Builds a new command-interface Search object.
-    def initialize query, opts = {}                
-      @query = query || ""
-      @parsed_query = parse_google_to_sphinx(@query)
-        
-      @options = self.class.query_defaults.merge(opts._coerce_basic_types)        
-      @options[:raw_filters] ||= {}
-      @options[:models] = Array(@options[:models])
+    def initialize opts = {}
+      
+      opts = opts._deep_stringify_keys
+            
+      @options = self.class.query_defaults.merge(opts._deep_dup._coerce_basic_types)
+
+      @options['filter'] ||= @options['raw_filters'] || {} # XXX legacy name
+
+      @options['query'] = @options['query'].to_s
+      @options['class_name'] = Array(@options['class_name'])
+      
+      @options['parsed_query'] = if query.blank? 
+         "@empty_searchable #{EMPTY_SEARCHABLE}"
+        else
+          parse(query)
+        end
   
-      @results, @subtotals, @response = [], {}, {}
-              
-      raise Sphinx::SphinxArgumentError, "Invalid options: #{@extra * ', '}" if (@extra = (@options.keys - (SPHINX_CLIENT_PARAMS.merge(self.class.query_defaults).keys))).size > 0      
+      @results, @subtotals, @facets, @response = [], {}, {}, {}
+        
+      extra_keys = @options.keys - (SPHINX_CLIENT_PARAMS.merge(self.class.query_defaults).keys + LEGACY_QUERY_KEYS + INTERNAL_KEYS)
+      logger.warn "Discarded invalid keys: #{extra_keys * ', '}" if extra_keys.any?
     end
     
-    # Run the search, filling results with an array of ActiveRecord objects.
-    def run(reify = true)      
+    # Run the search, filling results with an array of ActiveRecord objects. Set the parameter to false if you only want the ids returned.
+    def run(reify = true)
       @request = build_request_with_options(@options)
+      @paginate = nil # clear cache
       tries = 0
 
-      logger.info "** ultrasphinx: searching for #{query.inspect} (parsed as #{@parsed_query.inspect}), options #{@options.inspect}"
+      logger.info "** ultrasphinx: searching for #{@options.inspect}"
 
       begin
-        @response = @request.Query(@parsed_query)
-        logger.info "** ultrasphinx: search returned, error #{@request.GetLastError.inspect}, warning #{@request.GetLastWarning.inspect}, returned #{total}/#{response['total_found']} in #{time} seconds."  
+              
+        @response = @request.Query(parsed_query)
+        logger.info "** ultrasphinx: search returned, error #{@request.GetLastError.inspect}, warning #{@request.GetLastWarning.inspect}, returned #{total_entries}/#{response['total_found']} in #{time} seconds."  
 
-        @subtotals = get_subtotals(@request, @parsed_query) if self.class.client_options[:with_subtotals]
+        @subtotals = get_subtotals(@request, parsed_query) if self.class.client_options['with_subtotals']
+        
+        Array(@options['facets']).each do |facet|
+          @facets[facet] = get_facets(@request, parsed_query, facet)
+        end
+        
         @results = response['matches']
         
         # if you don't reify, you'll have to do the modulus reversal yourself to get record ids
         @results = reify_results(@results) if reify
-                  
+                                
       rescue Sphinx::SphinxResponseError, Sphinx::SphinxTemporaryError, Errno::EPIPE => e
-        if (tries += 1) <= self.class.client_options[:max_retries]
+        if (tries += 1) <= self.class.client_options['max_retries']
           logger.warn "** ultrasphinx: restarting query (#{tries} attempts already) (#{e})"
-          sleep(self.class.client_options[:retry_sleep_time]) if tries == self.class.client_options[:max_retries]
+          sleep(self.class.client_options['retry_sleep_time']) if tries == self.class.client_options['max_retries']
           retry
         else
           logger.warn "** ultrasphinx: query failed"
@@ -253,7 +355,7 @@ Note that your database is never changed by anything Ultrasphinx does.
       responses = @request.BuildExcerpts(
         texts, 
         UNIFIED_INDEX_NAME, 
-        strip_query_commands(@parsed_query),
+        strip_query_commands(parsed_query),
         self.class.excerpting_options.except('content_methods')
       ).in_groups_of(self.class.excerpting_options['content_methods'].size)
       
@@ -261,179 +363,30 @@ Note that your database is never changed by anything Ultrasphinx does.
         # override the individual model accessors with the excerpted data
         result, methods = result_and_methods
         methods.each_with_index do |method, j|
-          result._metaclass.send(:define_method, method) { responses[i][j] } if method
+          result._metaclass.send('define_method', method) { responses[i][j] } if method
         end
       end
   
-      @results = results_with_content_methods.map(&:first).map(&:freeze)
+      @results = results_with_content_methods.map do |result_and_content_method| 
+        result_and_content_method.first.freeze
+      end
       
       self
     end  
-  
-  
-    private
     
-    def build_request_with_options opts
-
-      request = Sphinx::Client.new
-
-      request.SetServer(PLUGIN_SETTINGS['server_host'], PLUGIN_SETTINGS['server_port'])
-      request.SetMatchMode Sphinx::Client::SPH_MATCH_EXTENDED # force extended query mode
-
-      offset, limit = opts[:per_page] * (opts[:page] - 1), opts[:per_page]
-      
-      request.SetLimits offset, limit, [offset + limit, MAX_MATCHES].min
-      request.SetSortMode SPHINX_CLIENT_PARAMS[:sort_mode][opts[:sort_mode]], opts[:sort_by].to_s
-
-      if weights = opts[:weights]
-        # order the weights hash according to the field order for sphinx, and set the missing fields to 1.0
-        # XXX we shouldn't really have to hit Fields.instance from within Ultrasphinx::Search
-        request.SetWeights(Fields.instance.select{|n,t| t == 'text'}.map(&:first).sort.inject([]) do |array, field|
-          array << (weights[field] || 1.0)
-        end)
-      end
-
-      unless opts[:models].compact.empty?
-        request.SetFilter 'class_id', opts[:models].map{|m| MODELS_TO_IDS[m.to_s]}
-      end        
-
-      # extract ranged raw filters 
-      # XXX some of this mangling might not be necessary
-      opts[:raw_filters].each do |field, value|
-        begin
-          unless value.is_a? Range
-            request.SetFilter field, Array(value)
-          else
-            min, max = [value.first, value.last].map do |x|
-              x._to_numeric if x.is_a? String
-            end
-            unless min.class != max.class
-              min, max = max, min if min > max
-              request.SetFilterRange field, min, max
-            end
-          end
-        rescue NoMethodError => e
-          raise Sphinx::SphinxArgumentError, "filter: #{field.inspect}:#{value.inspect} is invalid"
-        end
-      end
-      
-      # request.SetIdRange # never useful
-      # request.SetGroup # never useful
-      
-      request
-    end    
-  
-    def get_subtotals(request, query)
-      # XXX andrew says there's a better way to do this
-      subtotals, filtered_request = {}, request.dup
-      
-      MODELS_TO_IDS.each do |name, class_id|
-        filtered_request.instance_eval { @filters.delete_if {|f| f['attr'] == 'class_id'} }
-        filtered_request.SetFilter 'class_id', [class_id]
-        subtotals[name] = request.Query(query)['total_found']
-      end
-      
-      subtotals
-    end
-
-    def strip_bogus_characters(s)
-      # used to remove some garbage before highlighting
-      s.gsub(/<.*?>|\.\.\.|\342\200\246|\n|\r/, " ").gsub(/http.*?( |$)/, ' ') if s
-    end
-    
-    def strip_query_commands(s)
-      # XXX dumb hack for query commands, since sphinx doesn't intelligently parse the query in excerpt mode
-      s.gsub(/AND|OR|NOT|\@\w+/, "")
-    end 
-  
-    def parse_google_to_sphinx query
-      # alters google-style querystring into sphinx-style
-      return if query.blank?
-
-      # remove AND's, always
-      query = " #{query} ".gsub(" AND ", " ")
-
-      # split query on spaces that are not inside sets of quotes or parens
-      query = query.scan(/[^"() ]*["(][^")]*[")]|[^"() ]+/) 
-
-      query.each_with_index do |token, index|
-      
-        # recurse for parens, if necessary
-        if token =~ /^(.*?)\((.*)\)(.*?$)/
-          token = query[index] = "#{$1}(#{parse_google_to_sphinx $2})#{$3}"
-        end       
-        
-        # translate to sphinx-language
-        case token
-          when "OR"
-            query[index] = "|"
-          when "NOT"
-            query[index] = "-#{query[index+1]}"
-            query[index+1] = ""
-          when "AND"
-            query[index] = ""
-          when /:/
-            query[query.size] = "@" + query[index].sub(":", " ")
-            query[index] = ""
-        end
-        
+            
+    # Delegates enumerable methods to @results, if possible. This allows us to behave directly like a WillPaginate::Collection. Failing that, we delegate to the options hash if a key is set. This lets us use the <tt>self</tt> directly in view helpers.
+    def method_missing(*args, &block)
+      if @results.respond_to? args.first
+        @results.send(*args, &block)
+      elsif options.has_key? args.first.to_s
+        @options[args.first.to_s]
+      else
+        super
       end
-      query.join(" ").squeeze(" ")
     end
   
-    def reify_results(sphinx_ids)
-  
-      # order by position and then toss the rest of the data
-      # make sure you are using the bundled Sphinx client, which has a patch
-      sphinx_ids = sphinx_ids.sort_by do |key, value| 
-        value['index'] or raise ConfigurationError, "Your Sphinx client is not properly patched."
-      end.map(&:first)
-  
-      # inverse-modulus map the sphinx ids to the table-specific ids
-      ids = Hash.new([])
-      sphinx_ids.each do |id|
-        ids[MODELS_TO_IDS.invert[id % MODELS_TO_IDS.size]] += [id / MODELS_TO_IDS.size] # yay math
-      end
-      raise Sphinx::SphinxResponseError, "impossible document id in query result" unless ids.values.flatten.size == sphinx_ids.size
-  
-      # fetch them for real
-      results = []
-      ids.each do |model, id_set|
-        klass = model.constantize
-        finder = klass.respond_to?(:get_cache) ? :get_cache : :find
-        logger.debug "** ultrasphinx: using #{klass.name}\##{finder} as finder method"
-  
-        begin
-          results += case instances = id_set.map {|id| klass.send(finder, id)} # XXX temporary until we update cache_fu
-            when Hash
-              instances.values
-            when Array
-              instances
-            else
-              Array(instances)
-          end
-        rescue ActiveRecord:: ActiveRecordError => e
-          raise Sphinx::SphinxResponseError, e.inspect
-        end
-      end
-  
-      # put them back in order
-      results.sort_by do |r| 
-        raise Sphinx::SphinxResponseError, "Bogus ActiveRecord id for #{r.class}:#{r.id}" unless r.id
-        index = (sphinx_ids.index(sphinx_id = r.id * MODELS_TO_IDS.size + MODELS_TO_IDS[r.class.base_class.name]))
-        raise Sphinx::SphinxResponseError, "Bogus reverse id for #{r.class}:#{r.id} (Sphinx:#{sphinx_id})" unless index
-        index / sphinx_ids.size.to_f
-      end
-      
-      # add an accessor for absolute search rank for each record
-      results.each_with_index do |r, index|
-        i = per_page * page + index
-        r._metaclass.send(:define_method, "result_index") { i }
-      end
-      
-    end  
-  
-    def logger
+    def logger #:nodoc:
       RAILS_DEFAULT_LOGGER
     end