ultrasphinx 1.9 → 1.11

data/lib/ultrasphinx/search/internals.rb

@@ -2,6 +2,9 @@
 module Ultrasphinx
   class Search
     module Internals
+    
+      INFINITY = 1/0.0
+    
       include Associations
 
       # These methods are kept stateless to ease debugging
@@ -11,6 +14,8 @@ module Ultrasphinx
       def build_request_with_options opts
       
         request = Riddle::Client.new
+        
+        # Basic options
         request.instance_eval do          
           @server = Ultrasphinx::CLIENT_SETTINGS['server_host']
           @port = Ultrasphinx::CLIENT_SETTINGS['server_port']          
@@ -20,8 +25,32 @@ module Ultrasphinx
           @max_matches = [@offset + @limit + Ultrasphinx::Search.client_options['max_matches_offset'], MAX_MATCHES].min
         end
           
+        # Geosearch location
+        loc = opts['location']
+        loc.stringify_keys!
+        lat, long = loc['lat'], loc['long']
+        if lat and long
+          # Convert degrees to radians, if requested
+          if loc['units'] == 'degrees'
+            lat = degrees_to_radians(lat)
+            long = degrees_to_radians(long)
+          end
+          # Set the location/anchor point
+          request.set_anchor(loc['lat_attribute_name'], lat, loc['long_attribute_name'], long)
+        end
+                  
         # Sorting
         sort_by = opts['sort_by']
+        if options['location']
+          case sort_by
+            when "distance asc", "distance" then sort_by = "@geodist asc"
+            when "distance desc" then sort_by = "@geodist desc"
+          end
+        end
+        
+        # Use the additional sortable column if it is a text type
+        sort_by += "_sortable" if Fields.instance.types[sort_by] == "text"
+        
         unless sort_by.blank?
           if opts['sort_mode'].to_s == 'relevance'
             # If you're sorting by a field you don't want 'relevance' order
@@ -62,21 +91,35 @@ module Ultrasphinx
         end          
 
         # Extract raw filters 
-        # XXX We should coerce based on the Field values, not on the class
+        # XXX This is poorly done. We should coerce based on the Field types, not the value class.
+        # That would also allow us to move numeric filters from the query string into the hash.
         Array(opts['filters']).each do |field, value|          
-          field = field.to_s
-          type = Fields.instance.types[field]
-          unless type
-            raise UsageError, "field #{field.inspect} is invalid"
+
+          field = field.to_s          
+          type = Fields.instance.types[field]             
+          
+          # Special derived attribute
+          if field == 'distance' and options['location']
+            field, type = '@geodist', 'float'
           end
+
+          raise UsageError, "field #{field.inspect} is invalid" unless type
           
+          exclude = false
+          
+          # check for exclude flag attached to filter
+          if value.is_a?(Hash)
+            exclude = value[:exclude]
+            value = value[:value]
+          end
+
           begin
             case value
               when Integer, Float, BigDecimal, NilClass, Array
                 # XXX Hack to force floats to be floats
                 value = value.to_f if type == 'float'
                 # Just bomb the filter in there
-                request.filters << Riddle::Client::Filter.new(field, Array(value), false)
+                request.filters << Riddle::Client::Filter.new(field, Array(value), exclude)
               when Range
                 # Make sure ranges point in the right direction
                 min, max = [value.begin, value.end].map {|x| x._to_numeric }
@@ -84,7 +127,7 @@ module Ultrasphinx
                 min, max = max, min if min > max
                 # XXX Hack to force floats to be floats
                 min, max = min.to_f, max.to_f if type == 'float'
-                request.filters << Riddle::Client::Filter.new(field, min..max, false)
+                request.filters << Riddle::Client::Filter.new(field, min..max, exclude)
               when String
                 # XXX Hack to move text filters into the query
                 opts['parsed_query'] << " @#{field} #{value}"
@@ -92,7 +135,7 @@ module Ultrasphinx
                 raise NoMethodError
             end
           rescue NoMethodError => e
-            raise UsageError, "filter value #{value.inspect} for field #{field.inspect} is invalid"
+            raise UsageError, "Filter value #{value.inspect} for field #{field.inspect} is invalid"
           end
         end
         
@@ -208,12 +251,19 @@ module Ultrasphinx
                 (configuration['association_sql'] or "LEFT OUTER JOIN #{association_model.table_name} AS #{table_alias} ON #{table_alias}.#{klass.to_s.downcase}_id = #{klass.table_name}.#{association_model.primary_key}")
               ]
             when 'concatenate'
-              # Wait for someone to complain before worrying about this
-              raise "Concatenation text facets have not been implemented"
+              raise "Concatenation text facets have only been implemented for when :association_sql is defined" if configuration['association_sql'].blank?
+              
+              table_alias = configuration['table_alias']
+              
+              [ "#{table_alias}.#{configuration['field']}",
+                configuration['association_sql']
+              ]
           end
           
-          klass.connection.execute("SELECT #{field_string} AS value, #{SQL_FUNCTIONS[ADAPTER]['hash']._interpolate(field_string)} AS hash FROM #{klass.table_name} #{join_string} GROUP BY value").each do |value, hash|
-            FACET_CACHE[facet][hash.to_i] = value
+          query = "SELECT #{field_string} AS value, #{SQL_FUNCTIONS[ADAPTER]['hash']._interpolate(field_string)} AS hash FROM #{klass.table_name} #{join_string} GROUP BY value"
+          
+          klass.connection.execute(query).each do |hash|
+            FACET_CACHE[facet][hash[1].to_i] = hash[0]
           end                            
           klass
         end
@@ -226,13 +276,16 @@ module Ultrasphinx
             
       # Inverse-modulus map the Sphinx ids to the table-specific ids
       def convert_sphinx_ids(sphinx_ids)    
-        number_of_models = IDS_TO_MODELS.size
+        
+        number_of_models = IDS_TO_MODELS.size        
+        raise ConfigurationError, "No model mappings were found. Your #{RAILS_ENV}.conf file is corrupted, or your application container needs to be restarted." if number_of_models == 0
+        
         sphinx_ids.sort_by do |item| 
           item[:index]
         end.map do |item|
           class_name = IDS_TO_MODELS[item[:doc] % number_of_models]
           raise DaemonError, "Impossible Sphinx document id #{item[:doc]} in query result" unless class_name
-          [class_name, item[:doc] / number_of_models]
+          [class_name, (item[:doc] / number_of_models).to_i]
         end
       end
 
@@ -254,7 +307,7 @@ module Ultrasphinx
             end or
               # XXX This default is kind of buried, but I'm not sure why you would need it to be 
               # configurable, since you can use ['finder_methods'].
-              "find_all_by_id"
+              "find_all_by_#{klass.primary_key}"
             )
 
           records = klass.send(finder, ids_hash[class_name])
@@ -286,6 +339,16 @@ module Ultrasphinx
             end
           end
         end
+
+        # Add an accessor for distance, if requested
+        if self.options['location']['lat'] and self.options['location']['long']
+          results.each_with_index do |result, index|
+            if result
+              distance = (response[:matches][index][:attributes]['@geodist'] or INFINITY)
+              result.instance_variable_get('@attributes')['distance'] = distance
+            end
+          end
+        end
         
         results.compact!
         
@@ -299,14 +362,10 @@ module Ultrasphinx
       
       def perform_action_with_retries
         tries = 0
+        exceptions = [NoMethodError, Riddle::VersionError, Riddle::ResponseError, Errno::ECONNREFUSED, Errno::ECONNRESET,  Errno::EPIPE]
         begin
           yield
-        rescue NoMethodError,
-            Riddle::VersionError,
-            Riddle::ResponseError,
-            Errno::ECONNREFUSED, 
-            Errno::ECONNRESET, 
-            Errno::EPIPE => e
+        rescue *exceptions => e
           tries += 1
           if tries <= Ultrasphinx::Search.client_options['max_retries']
             say "restarting query (#{tries} attempts already) (#{e})"            
@@ -314,7 +373,9 @@ module Ultrasphinx
             retry
           else
             say "query failed"
-            raise DaemonError, e.to_s
+            # Clear the rescue list, retry one last time, and let the error fail up the stack
+            exceptions = []
+            retry
           end
         end
       end
@@ -329,7 +390,11 @@ module Ultrasphinx
         # Also removes apostrophes in the middle of words so that they don't get split in two.
         s.gsub(/(^|\s)(AND|OR|NOT|\@\w+)(\s|$)/i, "").gsub(/(\w)\'(\w)/, '\1\2')
       end 
+      
+      def degrees_to_radians(value)
+        Math::PI * value / 180.0
+      end
     
     end
   end  
-end
+end

data/lib/ultrasphinx/search/parser.rb

@@ -111,6 +111,13 @@ module Ultrasphinx
           end        
         end
         
+        # Account for trailing operators
+        operators = OPERATORS.to_a.flatten
+        if operators.include?(token_stream.first)
+          token_stream.pop until !operators.include?(token_stream.last)
+          token_stream << '' until token_stream.size > 0 && token_stream.size.even?
+        end
+        
         if token_stream.size.zero? or token_stream.size.odd?
           raise Error, "#{token_stream.inspect} is not a valid token stream"
         end
@@ -124,7 +131,14 @@ module Ultrasphinx
           # Remove some spaces
           content.gsub!(/^"\s+|\s+"$/, '"')
           # Convert fields into sphinx style, reformat the stream object
-          if content =~ /(.*?):(.*)/
+          if content =~ /(^(http|https):\/\/[a-z0-9]+([-.]{1}[a-z0-9]*)+. [a-z]{2,5}(([0-9]{1,5})?\/.*)?$)/ix
+            # XXX hack, its somewhat common to search for URLs.  be sure to add 
+            # " @, /," in the charset_type of the US config to search on all 
+            # URLs and email addresses, and add:
+            # prefix_fields = url, domain
+            # to your US config
+            token_hash[nil] += [[operator, content]]
+          elsif content =~ /(.*?):(.*)/
             token_hash[$1] += [[operator, $2]]
           else
             token_hash[nil] += [[operator, content]]
@@ -136,4 +150,4 @@ module Ultrasphinx
 
     end
   end
-end
+end

data/lib/ultrasphinx/spell.rb

@@ -1,5 +1,4 @@
 
-
 module Ultrasphinx
 
 =begin rdoc
@@ -41,6 +40,8 @@ If <tt>@correction</tt> is not <tt>nil</tt>, go ahead and suggest it to the user
     
     def self.correct string
       return nil unless SP
+      return nil if string =~ /\d+/
+
       correction = string.gsub(/[\w\']+/) do |word| 
         unless SP.check(word)
           SP.suggest(word).first

data/lib/ultrasphinx/ultrasphinx.rb

@@ -51,7 +51,9 @@ module Ultrasphinx
   }
   
   CONNECTION_DEFAULTS = {
-    :host => 'localhost'
+    :host => 'localhost',
+    :password => '',
+    :username => 'root'
   }
      
   mattr_accessor :with_rake
@@ -62,53 +64,43 @@ module Ultrasphinx
 
   SQL_FUNCTIONS = {
     'mysql' => {
-      'group_concat' => "CAST(GROUP_CONCAT(DISTINCT ? SEPARATOR ' ') AS CHAR)",
+      'group_concat' => "CAST(GROUP_CONCAT(DISTINCT ? ? SEPARATOR ' ') AS CHAR)",
       'delta' => "DATE_SUB(NOW(), INTERVAL ? SECOND)",      
       'hash' => "CAST(CRC32(?) AS unsigned)",
-      'range_cast' => "?",
-      'stored_procedures' => {}
+      'range_cast' => "?"
     },
     'postgresql' => {
       'group_concat' => "GROUP_CONCAT(?)",
       'delta' => "(NOW() - '? SECOND'::interval)",
       'range_cast' => "cast(coalesce(?,1) AS integer)",
-      'hash' => "CRC32(?)",
-      'stored_procedures' => Hash[*(
-        ['hex_to_int', 'group_concat', 'concat_ws', 'unix_timestamp', 'crc32'].map do |name|
-          [name, load_stored_procedure(name)]
-        end.flatten
-        )
-      ]
+      'hash' => "CRC32(?)"
     }      
   }
+  SQL_FUNCTIONS['jdbcmysql'] = SQL_FUNCTIONS['mysql']
   
   DEFAULTS = {
     'mysql' => %(
-type = mysql
-sql_query_pre = SET SESSION group_concat_max_len = 65535
-sql_query_pre = SET NAMES utf8
-  ), 
+      type = mysql
+      sql_query_pre = SET SESSION group_concat_max_len = 65535
+      sql_query_pre = SET NAMES utf8
+    ), 
     'postgresql' => %(
-type = pgsql
-sql_query_pre = ) + SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join(' ') + %(
-  )
-}
+      type = pgsql
+      sql_query_pre =
+    )
+  }
+  DEFAULTS['jdbcmysql'] = DEFAULTS['mysql']
     
   ADAPTER = ActiveRecord::Base.connection.instance_variable_get("@config")[:adapter] rescue 'mysql'
-  
-  # Install the stored procedures.
-  # XXX This shouldn't be done at every index, say the Postgres people.
-  SQL_FUNCTIONS[ADAPTER]['stored_procedures'].each do |key, value|
-    ActiveRecord::Base.connection.execute(value)
-  end
-  
-  # Logger.
+    
+  # Warn-mode logger. Also called from rake tasks.  
   def self.say msg
+    # XXX Method name is stupid.
     if with_rake
       puts msg[0..0].upcase + msg[1..-1]
     else
       msg = "** ultrasphinx: #{msg}"
-      if defined? RAILS_DEFAULT_LOGGER
+      if defined?(RAILS_DEFAULT_LOGGER) && RAILS_DEFAULT_LOGGER
         RAILS_DEFAULT_LOGGER.warn msg
       else
         STDERR.puts msg
@@ -117,6 +109,16 @@ sql_query_pre = ) + SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join
     nil # Explicitly return nil
   end
   
+  # Debug-mode logger.  
+  def self.log msg
+    # XXX Method name is stupid.
+    if defined?(RAILS_DEFAULT_LOGGER) && RAILS_DEFAULT_LOGGER
+      RAILS_DEFAULT_LOGGER.debug msg
+    else
+      STDERR.puts msg
+    end
+  end    
+  
   # Configuration file parser.
   def self.options_for(heading, path)
     # Evaluate ERB
@@ -127,12 +129,20 @@ sql_query_pre = ) + SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join
     section = contents[/^#{heading.gsub('/', '__')}\s*?\{(.*?)\}/m, 1]
     
     if section
+      # Strip comments and leading/trailing whitespace
+      section.gsub!(/^\s*(.*?)\s*(?:#.*)?$/, '\1')
+
       # Convert to a hash
-      options = section.split("\n").map do |line|
-        line =~ /\s*(.*?)\s*=\s*([^\#]*)/
-        $1 ? [$1, $2.strip] : []
-      end      
-      Hash[*options.flatten] 
+      returning({}) do |options|
+        lines = section.split(/\n+/)
+        while line = lines.shift
+          if line =~ /(.*?)\s*=\s*(.*)/
+            key, value = $1, [$2]
+            value << (line = lines.shift) while line =~ /\\$/
+            options[key] = value.join("\n    ")
+          end
+        end
+      end
     else
       # XXX Is it safe to raise here?
       Ultrasphinx.say "warning; heading #{heading} not found in #{path}; it may be corrupted. "
@@ -186,22 +196,6 @@ sql_query_pre = ) + SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join
   STOPWORDS_PATH = "#{Ultrasphinx::INDEX_SETTINGS['path']}/#{DICTIONARY}-stopwords.txt"
 
   MODEL_CONFIGURATION = {}     
-
-  # Complain if the database names go out of sync.
-  def self.verify_database_name
-    if File.exist? CONF_PATH
-      begin
-        if options_for(
-          "source #{MODEL_CONFIGURATION.keys.first.tableize}_#{MAIN_INDEX}", 
-          CONF_PATH
-        )['sql_db'] != ActiveRecord::Base.connection.instance_variable_get("@config")[:database]
-          say "warning; configured database name is out-of-date"
-          say "please run 'rake ultrasphinx:configure'"
-        end 
-      rescue Object
-      end
-    end
-  end
   
   # See if a delta index was defined.
   def self.delta_index_present?

data/tasks/ultrasphinx.rake

@@ -1,11 +1,16 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../lib')
 
 ENV['RAILS_ENV'] ||= "development"
 
+module Ultrasphinx
+end
+
 namespace :ultrasphinx do  
 
   task :_environment => [:environment] do
     # We can't just chain :environment because we want to make 
     # sure it's set only for known Sphinx tasks
+    require 'ultrasphinx'
     Ultrasphinx.with_rake = true
   end
   
@@ -31,6 +36,11 @@ namespace :ultrasphinx do
       ultrasphinx_index(Ultrasphinx::DELTA_INDEX)
     end
     
+    desc "Merge the delta index into the main index."
+    task :merge =>  [:_environment] do
+      ultrasphinx_merge
+    end
+    
   end
 
   desc "Reindex and rotate all indexes."  
@@ -43,7 +53,7 @@ namespace :ultrasphinx do
     task :start => [:_environment] do
       FileUtils.mkdir_p File.dirname(Ultrasphinx::DAEMON_SETTINGS["log"]) rescue nil
       raise Ultrasphinx::DaemonError, "Already running" if ultrasphinx_daemon_running?
-      system "searchd --config '#{Ultrasphinx::CONF_PATH}'"
+      system "searchd --config #{Ultrasphinx::CONF_PATH}"
       sleep(4) # give daemon a chance to write the pid file
       if ultrasphinx_daemon_running?
         say "started successfully"
@@ -103,7 +113,9 @@ namespace :ultrasphinx do
         end
       end
       say "writing #{words.size} words"
-      File.open(tmpfile, 'w').write(words.join("\n"))
+      File.open(tmpfile, 'w') do |f|
+        f.write(words.join("\n"))
+      end
       say "loading dictionary '#{Ultrasphinx::DICTIONARY}' into aspell"
       system("aspell --lang=en create master #{Ultrasphinx::DICTIONARY}.rws < #{tmpfile}")
     end
@@ -121,6 +133,7 @@ namespace :us do
   task :in => ["ultrasphinx:index"]
   task :main => ["ultrasphinx:index:main"]
   task :delta => ["ultrasphinx:index:delta"]
+  task :merge => ["ultrasphinx:index:merge"]
   task :spell => ["ultrasphinx:spelling:build"]
   task :conf => ["ultrasphinx:configure"]  
   task :boot => ["ultrasphinx:bootstrap"]  
@@ -133,7 +146,7 @@ def ultrasphinx_daemon_pid
 end
 
 def ultrasphinx_daemon_running?
-  if ultrasphinx_daemon_pid and `ps #{ultrasphinx_daemon_pid} | wc`.to_i > 1 
+  if ultrasphinx_daemon_pid and `ps -p#{ultrasphinx_daemon_pid} | wc`.to_i > 1 
     true
   else
     # Remove bogus lockfiles.
@@ -144,30 +157,57 @@ end
 
 def ultrasphinx_index(index)
   rotate = ultrasphinx_daemon_running?
-  index_path = Ultrasphinx::INDEX_SETTINGS['path']
-  mkdir_p index_path unless File.directory? index_path
+  ultrasphinx_create_index_path
   
-  cmd = "indexer --config '#{Ultrasphinx::CONF_PATH}'"
+  cmd = "indexer --config #{Ultrasphinx::CONF_PATH}"
   cmd << " #{ENV['OPTS']} " if ENV['OPTS']
   cmd << " --rotate" if rotate
   cmd << " #{index}"
   
+  say "$ #{cmd}"
+  system cmd
+    
+  ultrasphinx_check_rotate if rotate    
+end
+
+def ultrasphinx_merge
+  rotate = ultrasphinx_daemon_running?
+
+  indexes = [Ultrasphinx::MAIN_INDEX, Ultrasphinx::DELTA_INDEX]
+  indexes.each do |index|
+    raise "#{index} index is missing" unless File.exist? "#{Ultrasphinx::INDEX_SETTINGS['path']}/sphinx_index_#{index}.spa"
+  end
+  
+  cmd = "indexer --config #{Ultrasphinx::CONF_PATH}"
+  cmd << " #{ENV['OPTS']} " if ENV['OPTS']
+  cmd << " --rotate" if rotate
+  cmd << " --merge #{indexes.join(' ')}"
+  
   say "$ #{cmd}"
   system cmd
       
-  if rotate
-    sleep(4)
-    failed = Dir[index_path + "/*.new.*"]
-    if failed.any?
-      say "warning; index failed to rotate! Deleting new indexes"
-      failed.each {|f| File.delete f }
-    else
-      say "index rotated ok"
-    end
+  ultrasphinx_check_rotate if rotate
+end
+
+def ultrasphinx_check_rotate
+  sleep(4)
+  failed = Dir[Ultrasphinx::INDEX_SETTINGS['path'] + "/*.new.*"]
+  if failed.any?
+    say "warning; index failed to rotate! Deleting new indexes"
+    say "try 'killall searchd' and then 'rake ultrasphinx:daemon:start'"
+    failed.each {|f| File.delete f }
+  else
+    say "index rotated ok"
+  end
+end
+
+def ultrasphinx_create_index_path
+  unless File.directory? Ultrasphinx::INDEX_SETTINGS['path']
+    mkdir_p Ultrasphinx::INDEX_SETTINGS['path'] 
   end
 end
 
 def say msg
   Ultrasphinx.say msg
 end
-  
+