ultrasphinx 1.9 → 1.11

data/lib/ultrasphinx/core_extensions.rb

@@ -51,6 +51,15 @@ class Hash
       end._flatten_once]
   end
   
+  def _stringify_all!(*skip)
+    # Stringifies all keys, and stringifies all values except those slotted for keys in 'skip'
+    stringify_keys!
+    self.except(*skip).each do |key, value|
+      self[key] = value.to_s
+    end
+    self
+  end
+  
   # Delete by multiple keys
   def _delete(*args)
     args.map do |key|
@@ -97,9 +106,15 @@ class String
     end
   end
   
-  def _interpolate(value)
-    self.gsub('?', value)
+  # Interpolate SQL field names into functions
+  def _interpolate(*values)
+    if values.size == 1
+      self.gsub('?', values.first)
+    else
+      values.inject(self) { |result, value| result.sub('?', value.to_s) }
+    end
   end
+  
 end
 
 module Ultrasphinx::NumericSelf

data/lib/ultrasphinx/fields.rb

@@ -6,6 +6,8 @@ This is a special singleton configuration class that stores the index field conf
 =end
 
   class Fields
+    # XXX Class needs a big refactoring; one of the worst parts of Ultrasphinx
+    
     include Singleton
     include Associations
     
@@ -17,6 +19,7 @@ This is a special singleton configuration class that stores the index field conf
       'datetime' => 'date',
       'timestamp' => 'date',
       'float' => 'float',
+      'decimal' => 'integer', # this ain't right, but float doesn't work.
       'boolean' => 'bool'
     }
         
@@ -28,19 +31,22 @@ This is a special singleton configuration class that stores the index field conf
       @groups = []
     end
     
+    
     def groups
       @groups.compact.sort_by do |string| 
         string[/= (.*)/, 1]
       end
     end
   
-    def save_and_verify_type(field, new_type, string_sortable, klass)
+  
+    def save_and_verify_type(field, new_type, string_sortable, klass, msg = nil)
       # Smoosh fields together based on their name in the Sphinx query schema
       field, new_type = field.to_s, TYPE_MAP[new_type.to_s]
 
       if types[field]
         # Existing field name; verify its type
-        raise ConfigurationError, "Column type mismatch for #{field.inspect}; was already #{types[field].inspect}, but is now #{new_type.inspect}." unless types[field] == new_type
+        msg ||= "Column type mismatch for #{field.inspect}; was already #{types[field].inspect}, but is now #{new_type.inspect}."
+        raise ConfigurationError, msg unless types[field] == new_type
         classes[field] = (classes[field] + [klass]).uniq
 
       else
@@ -63,6 +69,7 @@ This is a special singleton configuration class that stores the index field conf
       end
     end
     
+    
     def cast(source_string, field)
       if types[field] == "date"
         "UNIX_TIMESTAMP(#{source_string})"
@@ -73,6 +80,7 @@ This is a special singleton configuration class that stores the index field conf
       end + " AS #{field}"
     end    
       
+      
     def null(field)      
       case types[field]
         when 'text'
@@ -88,6 +96,7 @@ This is a special singleton configuration class that stores the index field conf
       end + " AS #{field}"
     end
     
+    
     def configure(configuration)
 
       configuration.each do |model, options|        
@@ -99,10 +108,7 @@ This is a special singleton configuration class that stores the index field conf
         begin
         
           # Fields are from the model
-          # We destructively canonicize them back onto the configuration hash
           options['fields'] = options['fields'].to_a.map do |entry|
-            entry = {'field' => entry} unless entry.is_a? Hash
-
             extract_table_alias!(entry, klass)
             extract_field_alias!(entry, klass)
             
@@ -110,8 +116,8 @@ This is a special singleton configuration class that stores the index field conf
               # XXX I think this is here for migrations
               Ultrasphinx.say "warning: field #{entry['field']} is not present in #{model}"
             else
-              save_and_verify_type(entry['as'], klass.columns_hash[entry['field']].type, entry['sortable'], klass)
-              install_facets!(entry, klass)
+              save_and_verify_type(entry['as'], klass.columns_hash[entry['field']].type, nil, klass)
+              install_duplicate_fields!(entry, klass)
             end            
           end  
           
@@ -122,15 +128,15 @@ This is a special singleton configuration class that stores the index field conf
             
             association_model = get_association_model(klass, entry)
             
-            save_and_verify_type(entry['as'] || entry['field'], association_model.columns_hash[entry['field']].type, entry['sortable'], klass)
-            install_facets!(entry, klass)
+            save_and_verify_type(entry['as'] || entry['field'], association_model.columns_hash[entry['field']].type, nil, klass)
+            install_duplicate_fields!(entry, klass)
           end  
           
           # Regular concats are CHAR, group_concats are BLOB and need to be cast to CHAR
           options['concatenate'].to_a.each do |entry|
             extract_table_alias!(entry, klass)
-            save_and_verify_type(entry['as'], 'text', entry['sortable'], klass) 
-            install_facets!(entry, klass)
+            save_and_verify_type(entry['as'], 'text', nil, klass) 
+            install_duplicate_fields!(entry, klass)
           end          
           
         rescue ActiveRecord::StatementInvalid
@@ -141,20 +147,34 @@ This is a special singleton configuration class that stores the index field conf
       self
     end
     
-    def install_facets!(entry, klass)
+    
+    def install_duplicate_fields!(entry, klass)
       if entry['facet']
-        save_and_verify_type(entry['as'], 'text', nil, klass) # source must be a string
+        # Source must be a string
+        save_and_verify_type(entry['as'], 'text', nil, klass, 
+          "#{klass}##{entry['as']}: 'facet' option is only valid for text fields; numeric fields are enabled by default")
+        # Install facet column                
         save_and_verify_type("#{entry['as']}_facet", 'integer', nil, klass)
       end
+
+      if entry['sortable']
+        # Source must be a string
+        save_and_verify_type(entry['as'], 'text', nil, klass, 
+          "#{klass}##{entry['as']}: 'sortable' option is only valid for text columns; numeric fields are enabled by default")
+        # Install sortable column        
+        save_and_verify_type("#{entry['as']}_sortable", 'text', true, klass)      
+      end
       entry
     end
     
+    
     def extract_field_alias!(entry, klass)
       unless entry['as']    
         entry['as'] = entry['field'] 
       end
     end
     
+    
     def extract_table_alias!(entry, klass)
       unless entry['table_alias']
         entry['table_alias'] = if entry['field'] and entry['field'].include? "." and entry['association_sql']

data/lib/ultrasphinx/is_indexed.rb

@@ -42,6 +42,16 @@ SQL conditions, to scope which records are selected for indexing. Accepts a stri
   
 The <tt>:conditions</tt> key is especially useful if you delete records by marking them deleted rather than removing them from the database.
 
+== Ordering subgroups
+ 
+Use the <tt>:order</tt> key.
+
+An SQL order string.
+
+  :order => 'posts.id ASC'
+  
+
+
 == Including a field from an association
 
 Use the <tt>:include</tt> key.
@@ -76,9 +86,9 @@ Also use the <tt>:concatenate</tt> key.
 To concatenate one field from a set of associated records as a combined field in the parent record, use a group (or vertical) concatenation. A group concatenation should contain an <tt>:association_name</tt> key (the association name for the included model), a <tt>:field</tt> key (the field on the included model to concatenate), and an optional <tt>:as</tt> key (also the name of the result of the concatenation). For example, to concatenate all <tt>Post#body</tt> contents into the parent's <tt>responses</tt> field:
   :concatenate => [{:association_name => 'posts', :field => 'body', :as => 'responses'}]
 
-The keys <tt>:facet</tt>, <tt>:sortable</tt>, <tt>:conditions</tt>, <tt>:function_sql</tt>, <tt>:class_name</tt>, and <tt>:association_sql</tt>, are also recognized.
+The keys <tt>:facet</tt>, <tt>:sortable</tt>, <tt>:order</tt>, <tt>:conditions</tt>, <tt>:function_sql</tt>, <tt>:class_name</tt>, and <tt>:association_sql</tt>, are also recognized. 
 
-Vertical concatenations are implemented with GROUP_CONCAT on MySQL and with an aggregate and a stored procedure on PostgreSQL.
+Vertical concatenations are implemented with GROUP_CONCAT on MySQL and with an aggregate and a stored procedure on PostgreSQL. Note that <tt>:order</tt> is useful if you need to order the grouping so that proximity search works correctly, and <tt>:conditions</tt> are currently ignored if you have <tt>:association_sql</tt> defined.
 
 == Custom joins
 
@@ -156,9 +166,8 @@ If the associations weren't just <tt>has_many</tt> and <tt>belongs_to</tt>, you
 =end
   
     def self.is_indexed opts = {}    
-      opts = HashWithIndifferentAccess.new(opts)
-          
-      opts.assert_valid_keys ['fields', 'concatenate', 'conditions', 'include', 'delta']
+      opts.stringify_keys!          
+      opts.assert_valid_keys ['fields', 'concatenate', 'conditions', 'include', 'delta', 'order']
 
       # Single options
       
@@ -172,7 +181,8 @@ If the associations weren't just <tt>has_many</tt> and <tt>belongs_to</tt>, you
         elsif opts['delta'].is_a? String
           opts['delta'] = {'field' => opts['delta']} 
         end
-        opts['delta'].stringify_keys!
+        
+        opts['delta']._stringify_all!
         opts['delta'].assert_valid_keys ['field']
       end
       
@@ -182,27 +192,35 @@ If the associations weren't just <tt>has_many</tt> and <tt>belongs_to</tt>, you
       opts['concatenate'] = Array(opts['concatenate'])
       opts['include'] = Array(opts['include'])
                   
-      opts['fields'].each do |entry|
+      opts['fields'].map! do |entry|
         if entry.is_a? Hash
-          entry.stringify_keys!
-          entry.assert_valid_keys ['field', 'as', 'facet', 'function_sql', 'sortable']
+          entry._stringify_all!('sortable', 'facet')
+          entry.assert_valid_keys ['field', 'as', 'facet', 'function_sql', 'sortable', 'table_alias']
+          entry
+        else
+          # Single strings
+          {'field' => entry.to_s}
         end
       end
       
       opts['concatenate'].each do |entry|
-        entry.stringify_keys!
-        entry.assert_valid_keys ['class_name', 'association_name', 'conditions', 'field', 'as', 'fields', 'association_sql', 'facet', 'function_sql', 'sortable']
+        entry._stringify_all!('fields', 'sortable', 'facet')
+      
+        entry.assert_valid_keys ['class_name', 'association_name', 'conditions', 'field', 'as', 'fields', 'association_sql', 'facet', 'function_sql', 'sortable', 'order', 'table_alias']
         raise Ultrasphinx::ConfigurationError, "You can't mix regular concat and group concats" if entry['fields'] and (entry['field'] or entry['class_name'] or entry['association_name'])
         raise Ultrasphinx::ConfigurationError, "Concatenations must specify an :as key" unless entry['as']
         raise Ultrasphinx::ConfigurationError, "Group concatenations must not have multiple fields" if entry['field'].is_a? Array
         raise Ultrasphinx::ConfigurationError, "Regular concatenations should have multiple fields" if entry['fields'] and !entry['fields'].is_a?(Array)
+        raise Ultrasphinx::ConfigurationError, "Regular concatenations can't specify an order" if entry['fields'] and entry['order']
+
+        entry['fields'].map!(&:to_s) if entry['fields'] # Stringify fields array
       end
       
       opts['include'].each do |entry|
-        entry.stringify_keys!
-        entry.assert_valid_keys ['class_name', 'association_name', 'field', 'as', 'association_sql', 'facet', 'function_sql', 'sortable']
+        entry._stringify_all!('sortable', 'facet')
+        entry.assert_valid_keys ['class_name', 'association_name', 'field', 'as', 'association_sql', 'facet', 'function_sql', 'sortable', 'table_alias', 'dont_group_by']
       end
-            
+                  
       Ultrasphinx::MODEL_CONFIGURATION[self.name] = opts
     end
   end

data/lib/ultrasphinx/postgresql/concat_ws.sql

@@ -1,11 +1,11 @@
 
 /* http://osdir.com/ml/db.postgresql.admIN/2003-08/msg00057.html */
 
-CREATE OR REPLACE FUNCTION MAKE_CONCAT_WS() RETURNS text AS '
+CREATE FUNCTION MAKE_CONCAT_WS() RETURNS text AS '
 declare
   v_args int := 32;
-  v_first text := ''CREATE OR REPLACE FUNCTION CONCAT_WS(text,text,text) RETURNS text AS ''''SELECT CASE WHEN $1 IS NULL THEN NULL WHEN $3 IS NULL THEN $2 ELSE $2 || $1 || $3 END'''' LANGUAGE sql IMMUTABLE'';
-  v_part1 text := ''CREATE OR REPLACE FUNCTION CONCAT_WS(text,text'';
+  v_first text := ''CREATE FUNCTION CONCAT_WS(text,text,text) RETURNS text AS ''''SELECT CASE WHEN $1 IS NULL THEN NULL WHEN $3 IS NULL THEN $2 ELSE $2 || $1 || $3 END'''' LANGUAGE sql IMMUTABLE'';
+  v_part1 text := ''CREATE FUNCTION CONCAT_WS(text,text'';
   v_part2 text := '') RETURNS text AS ''''SELECT CONCAT_WS($1,CONCAT_WS($1,$2'';
   v_part3 text := '')'''' LANGUAGE sql IMMUTABLE'';  
   v_sql text;

data/lib/ultrasphinx/postgresql/crc32.sql

@@ -1,7 +1,7 @@
 
 /* Fake CRC32 */
 
-CREATE OR REPLACE FUNCTION crc32(text)
+CREATE FUNCTION crc32(text)
 RETURNS bigint AS $$
   DECLARE
     tmp bigint;

data/lib/ultrasphinx/postgresql/group_concat.sql

@@ -6,7 +6,7 @@
  Note: For DISTINCT and ORDER BY a subquery is required
 */
 
-CREATE OR REPLACE FUNCTION _group_concat(text, text)
+CREATE FUNCTION _group_concat(text, text)
 RETURNS text AS $$
   SELECT CASE
     WHEN $2 IS NULL THEN $1
@@ -15,8 +15,6 @@ RETURNS text AS $$
   END
 $$ IMMUTABLE LANGUAGE SQL;
 
-DROP AGGREGATE IF EXISTS group_concat(text);
-
 CREATE AGGREGATE group_concat (
 	BASETYPE = text,
 	SFUNC = _group_concat,

data/lib/ultrasphinx/postgresql/hex_to_int.sql

@@ -1,5 +1,5 @@
 
-CREATE OR REPLACE FUNCTION hex_to_int(varchar) RETURNS int4 AS '
+CREATE FUNCTION hex_to_int(varchar) RETURNS int4 AS '
   DECLARE
     h alias for $1;
     exec varchar;

data/lib/ultrasphinx/postgresql/unix_timestamp.sql

@@ -6,7 +6,7 @@
  UNIX_TIMESTAMP(date)
 */
 
-CREATE OR REPLACE FUNCTION unix_timestamp(timestamp without time zone)
+CREATE FUNCTION unix_timestamp(timestamp without time zone)
 RETURNS bigint AS $$
   SELECT EXTRACT(EPOCH FROM $1)::bigint
 $$ VOLATILE LANGUAGE SQL;

data/lib/ultrasphinx/search.rb

@@ -42,8 +42,11 @@ The hash lets you customize internal aspects of the search.
 <tt>:weights</tt>:: A hash. Text-field names and associated query weighting. The default weight for every field is 1.0. Example: <tt>:weights => {'title' => 2.0}</tt>
 <tt>:filters</tt>:: A hash. Names of numeric or date fields and associated values. You can use a single value, an array of values, or a range. (See the bottom of the ActiveRecord::Base page for an example.)
 <tt>:facets</tt>:: An array of fields for grouping/faceting. You can access the returned facet values and their result counts with the <tt>facets</tt> method.
+<tt>:location</tt>:: A hash. Specify the names of your latititude and longitude attributes as declared in your is_indexed calls. To sort the results by distance, set <tt>:sort_mode => 'extended'</tt> and <tt>:sort_by => 'distance asc'.</tt>
 <tt>:indexes</tt>:: An array of indexes to search. Currently only <tt>Ultrasphinx::MAIN_INDEX</tt> and <tt>Ultrasphinx::DELTA_INDEX</tt> are available. Defaults to both; changing this is rarely needed.
 
+== Query Defaults
+
 Note that you can set up your own query defaults in <tt>environment.rb</tt>: 
   
   self.class.query_defaults = HashWithIndifferentAccess.new({
@@ -54,6 +57,34 @@ Note that you can set up your own query defaults in <tt>environment.rb</tt>:
 
 = Advanced features
 
+== Geographic distance
+
+If you pass a <tt>:location</tt> Hash, distance from the location in meters will be available in your result records via the <tt>distance</tt> accessor:
+
+  @search = Ultrasphinx::Search.new(:class_names => 'Point', 
+            :query => 'pizza',
+            :sort_mode => 'extended',
+            :sort_by => 'distance',
+            :location => {
+              :lat => 40.3,
+              :long => -73.6
+            })
+            
+   @search.run.first.distance #=> 1402.4
+
+Note that Sphinx expects lat/long to be indexed as radians. If you have degrees in your database, do the conversion in the <tt>is_indexed</tt> as so:
+  
+    is_indexed 'fields' => [
+        'name', 
+        'description',
+        {:field => 'lat', :function_sql => "RADIANS(?)"}, 
+        {:field => 'lng', :function_sql => "RADIANS(?)"}
+      ]
+
+Then, set <tt>Ultrasphinx::Search.client_options[:location][:units] = 'degrees'</tt>.
+
+The MySQL <tt>:double</tt> column type is recommended for storing location data. For Postgres, use <tt>:float</tt.
+
 == Interlock integration
   
 Ultrasphinx uses the <tt>find_all_by_id</tt> method to instantiate records. If you set <tt>with_finders: true</tt> in {Interlock's}[http://blog.evanweaver.com/files/doc/fauna/interlock] <tt>config/memcached.yml</tt>, Interlock overrides <tt>find_all_by_id</tt> with a caching version.
@@ -108,7 +139,12 @@ Note that your database is never changed by anything Ultrasphinx does.
       :weights => {},
       :class_names => [],
       :filters => {},
-      :facets => []
+      :facets => [],
+      :location => HashWithIndifferentAccess.new({
+        :lat_attribute_name  => 'lat',
+        :long_attribute_name => 'lng',
+        :units => 'radians'
+      })
     })
     
     cattr_accessor :excerpting_options
@@ -239,6 +275,9 @@ Note that your database is never changed by anything Ultrasphinx does.
       require_run    
       (total_entries / per_page.to_f).ceil
     end
+
+    # add alias for total_pages as expected by latest will_paginate
+    alias total_pages page_count
             
     # Returns the previous page number.
     def previous_page 
@@ -262,12 +301,19 @@ Note that your database is never changed by anything Ultrasphinx does.
       opts = Hash[HashWithIndifferentAccess.new(opts._deep_dup._coerce_basic_types)]
       unless self.class.query_defaults.instance_of? Hash
         self.class.query_defaults = Hash[self.class.query_defaults]
+        self.class.query_defaults['location'] = Hash[self.class.query_defaults['location']]
+        
         self.class.client_options = Hash[self.class.client_options]
         self.class.excerpting_options = Hash[self.class.excerpting_options]
         self.class.excerpting_options['content_methods'].map! {|ary| ary.map {|m| m.to_s}}
       end    
+
+      # We need an annoying deep merge on the :location parameter
+      opts['location'].reverse_merge!(self.class.query_defaults['location']) if opts['location']
+
+      # Merge the rest of the defaults      
+      @options = self.class.query_defaults.merge(opts)
       
-      @options = self.class.query_defaults.merge(opts)            
       @options['query'] = @options['query'].to_s
       @options['class_names'] = Array(@options['class_names'])
       @options['facets'] = Array(@options['facets'])
@@ -281,7 +327,7 @@ Note that your database is never changed by anything Ultrasphinx does.
       @results, @subtotals, @facets, @response = [], {}, {}, {}
         
       extra_keys = @options.keys - (self.class.query_defaults.keys + INTERNAL_KEYS)
-      say "discarded invalid keys: #{extra_keys * ', '}" if extra_keys.any? and RAILS_ENV != "test" 
+      log "discarded invalid keys: #{extra_keys * ', '}" if extra_keys.any? and RAILS_ENV != "test" 
     end
     
     # Run the search, filling results with an array of ActiveRecord objects. Set the parameter to false 
@@ -289,11 +335,11 @@ Note that your database is never changed by anything Ultrasphinx does.
     def run(reify = true)
       @request = build_request_with_options(@options)
 
-      say "searching for #{@options.inspect}"
+      log "searching for #{@options.inspect}"
 
       perform_action_with_retries do
         @response = @request.query(parsed_query, @options['indexes'])
-        say "search returned #{total_entries}/#{response[:total_found].to_i} in #{time.to_f} seconds."
+        log "search returned #{total_entries}/#{response[:total_found].to_i} in #{time.to_f} seconds."
           
         if self.class.client_options['with_subtotals']        
           @subtotals = get_subtotals(@request, parsed_query) 
@@ -342,7 +388,7 @@ Note that your database is never changed by anything Ultrasphinx does.
       # Fetch the actual field contents
       docs = results_with_content_methods.map do |result, methods|
         methods.map do |method| 
-          method and strip_bogus_characters(result.send(method)) or ""
+          method and strip_bogus_characters(result.send(method).to_s) or ""
         end
       end.flatten
       
@@ -395,6 +441,10 @@ Note that your database is never changed by anything Ultrasphinx does.
       end
     end
   
+    def log msg #:nodoc:
+      Ultrasphinx.log msg
+    end
+
     def say msg #:nodoc:
       Ultrasphinx.say msg
     end