ultrasphinx 1.5.2 → 1.5.3

data/lib/ultrasphinx/hex_to_int.sql

@@ -0,0 +1,15 @@
+
+CREATE OR REPLACE FUNCTION hex_to_int(varchar) RETURNS int4 AS '
+  DECLARE
+    h alias for $1;
+    exec varchar;
+    curs refcursor;
+    res int;
+  BEGIN
+    exec := ''SELECT x'''''' || h || ''''''::int4'';
+    OPEN curs FOR EXECUTE exec;
+    FETCH curs INTO res;
+    CLOSE curs;
+    return res;
+  END;'
+LANGUAGE 'plpgsql' IMMUTABLE STRICT;

data/lib/ultrasphinx/is_indexed.rb

@@ -38,6 +38,8 @@ Use the <tt>:include</tt> key.
 
 Accepts an array of hashes. 
 
+  :include => [{:class_name => 'Category', :field => 'name', :as => 'category'}]
+
 Each should contain a <tt>:class_name</tt> key (the class name of the included model), a <tt>:field</tt> key (the name of the field to include), and an optional <tt>:as</tt> key (what to name the field in the parent). You can use the optional key <tt>:association_sql</tt> if you need to pass a custom JOIN string, in which case the default JOIN for <tt>belongs_to</tt> will not be generated.
 
 The keys <tt>:facet</tt>, <tt>:sortable</tt>, and <tt>:function_sql</tt> are also recognized, just like for regular fields.
@@ -68,7 +70,7 @@ Also use the <tt>:concatenate</tt> key.
 To concatenate one field from a set of associated records as a combined field in the parent record, use a group (or vertical) concatenation. A group concatenation should contain a <tt>:class_name</tt> key (the class name of the included model), a <tt>:field</tt> key (the field on the included model to concatenate), and an optional <tt>:as</tt> key (also the name of the result of the concatenation). For example, to concatenate all <tt>Post#body</tt> contents into the parent's <tt>responses</tt> field:
   :concatenate => [{:class_name => 'Post', :field => 'body', :as => 'responses'}]
 
-Optional group concatenation keys are <tt>:association_name</tt> (if your <tt>has_many</tt> association can't be derived from the model name), <tt>:association_sql</tt>, if you need to pass a custom JOIN string (for example, a double JOIN for a <tt>has_many :through</tt>), and <tt>:conditions</tt> (if you need custom WHERE conditions for this particular association).
+Optional group concatenation keys are <tt>:association_sql</tt>, if you need to pass a custom JOIN string (for example, a double JOIN for a <tt>has_many :through</tt>), and <tt>:conditions</tt> (if you need custom WHERE conditions for this particular association).
 
 The keys <tt>:facet</tt>, <tt>:sortable</tt>, and <tt>:function_sql</tt> are also recognized, just like for regular fields.
 
@@ -92,8 +94,7 @@ Here's an example configuration using most of the options, taken from production
       :concatenate => [
         {:fields => ['title', 'long_description', 'short_description'], 
           :as => 'editorial'},
-        {:class_name => 'Page', :field => 'body', :as => 'body', 
-          :association_name => 'pages'},
+        {:class_name => 'Page', :field => 'body', :as => 'body'},
         {:class_name => 'Comment', :field => 'body', :as => 'comments', 
           :conditions => "comments.item_type = '#{base_class}'"}
       ],
@@ -135,10 +136,11 @@ If the associations weren't just <tt>has_many</tt> and <tt>belongs_to</tt>, you
       
       Array(opts['concatenate']).each do |entry|
         entry.stringify_keys!
-        entry.assert_valid_keys ['class_name', 'conditions', 'field', 'as', 'fields', 'association_name', 'association_sql', 'facet', 'function_sql', 'sortable']
-        raise Ultrasphinx::ConfigurationError, "You can't mix regular concat and group concats" if entry['fields'] and (entry['field'] or entry['class_name'] or entry['association_name'])
-        raise Ultrasphinx::ConfigurationError, "Group concats must not have multiple fields" if entry['field'].is_a? Array
-        raise Ultrasphinx::ConfigurationError, "Regular concats should have multiple fields" if entry['fields'] and !entry['fields'].is_a?(Array)
+        entry.assert_valid_keys ['class_name', 'conditions', 'field', 'as', 'fields', 'association_sql', 'facet', 'function_sql', 'sortable']
+        raise Ultrasphinx::ConfigurationError, "You can't mix regular concat and group concats" if entry['fields'] and (entry['field'] or entry['class_name'])
+        raise Ultrasphinx::ConfigurationError, "Concatenations must specify an :as key" unless entry['as']
+        raise Ultrasphinx::ConfigurationError, "Group concatenations must not have multiple fields" if entry['field'].is_a? Array
+        raise Ultrasphinx::ConfigurationError, "Regular concatenations should have multiple fields" if entry['fields'] and !entry['fields'].is_a?(Array)
       end
       
       Array(opts['include']).each do |entry|

data/lib/ultrasphinx/search.rb

@@ -45,11 +45,11 @@ The hash lets you customize internal aspects of the search.
 
 Note that you can set up your own query defaults in <tt>environment.rb</tt>: 
   
-  Ultrasphinx::Search.query_defaults = {
+  Ultrasphinx::Search.query_defaults = HashWithIndifferentAccess.new({
     :per_page => 10,
     :sort_mode => 'relevance',
     :weights => {'title' => 2.0}
-  }
+  })
 
 = Advanced features
 
@@ -75,14 +75,14 @@ You need to set the <tt>content_methods</tt> key on Ultrasphinx::Search.excerpti
   
 There are some other keys you can set, such as excerpt size, HTML tags to highlight with, and number of words on either side of each excerpt chunk. Example (in <tt>environment.rb</tt>):
   
-  Ultrasphinx::Search.excerpting_options = {
+  Ultrasphinx::Search.excerpting_options = HashWithIndifferentAccess.new({
     :before_match => '<strong>', 
     :after_match => '</strong>',
     :chunk_separator => "...",
     :limit => 256,
     :around => 3,
     :content_methods => [['title'], ['body', 'description', 'content'], ['metadata']] 
-  }
+  })
   
 Note that your database is never changed by anything Ultrasphinx does.
 
@@ -151,8 +151,18 @@ Note that your database is never changed by anything Ultrasphinx does.
       else
         begin  
           lines = open(CONF_PATH).readlines          
-          sources = lines.select {|s| s =~ /^source \w/ }.map {|s| s[/source ([\w\d_-]*)/, 1].classify }
-          ids = lines.select {|s| s =~ /^sql_query / }.map {|s| s[/(\d*) AS class_id/, 1].to_i }
+
+          sources = lines.select do |line| 
+            line =~ /^source \w/
+          end.map do |line| 
+            line[/source ([\w\d_-]*)/, 1].gsub('__', '/').classify
+          end
+          
+          ids = lines.select do |line| 
+            line =~ /^sql_query /
+          end.map do |line| 
+            line[/(\d*) AS class_id/, 1].to_i
+          end
           
           raise unless sources.size == ids.size          
           Hash[*sources.zip(ids).flatten]
@@ -195,43 +205,47 @@ Note that your database is never changed by anything Ultrasphinx does.
     
     # Returns the facet map for this query, if facets were used.
     def facets
-      raise UsageError, "No facet field was configured" unless @options['facets']
       run?(true)
+      raise UsageError, "No facet field was configured" unless @options['facets']
       @facets
     end      
           
     # Returns the raw response from the Sphinx client.
     def response
+      run?(true)
       @response
     end
     
     def class_name #:nodoc:
       # Legacy accessor
-      @options['class_name']
+      @options['class_names']
     end
     
     # Returns a hash of total result counts, scoped to each available model. This requires extra queries against the search daemon right now. Set <tt>Ultrasphinx::Search.client_options[:with_subtotals] = true</tt> to enable the extra queries. Most of the overhead is in instantiating the AR result sets, so the performance hit is not usually significant.
     def subtotals
+      run?(true)
       raise UsageError, "Subtotals are not enabled" unless self.class.client_options['with_subtotals']
       @subtotals
     end
 
     # Returns the total result count.
     def total_entries
+      run?(true)
       [response['total_found'] || 0, MAX_MATCHES].min
     end  
   
     # Returns the response time of the query, in milliseconds.
     def time
+      run?(true)
       response['time']
     end
 
     # Returns whether the query has been run.  
     def run?(should_raise = false)
-      if response.blank? and should_raise
+      if @response.blank? and should_raise
         raise UsageError, "Search has not yet been run" unless run?
       else
-        !response.blank?
+        !@response.blank?
       end
     end
  
@@ -247,6 +261,7 @@ Note that your database is never changed by anything Ultrasphinx does.
         
     # Returns the last available page number in the result set.  
     def page_count
+      run?(true)    
       (total_entries / per_page) + (total_entries % per_page == 0 ? 0 : 1)
     end
             
@@ -302,7 +317,6 @@ Note that your database is never changed by anything Ultrasphinx does.
       say "searching for #{@options.inspect}"
 
       begin
-              
         @response = @request.Query(parsed_query)
         say "search returned, error #{@request.GetLastError.inspect}, warning #{@request.GetLastWarning.inspect}, returned #{total_entries}/#{response['total_found']} in #{time} seconds."  
 
@@ -317,14 +331,14 @@ Note that your database is never changed by anything Ultrasphinx does.
         # if you don't reify, you'll have to do the modulus reversal yourself to get record ids
         @results = reify_results(@results) if reify
                                 
-      rescue Sphinx::SphinxResponseError, Sphinx::SphinxTemporaryError, Errno::EPIPE => e
+      rescue Sphinx::SphinxConnectError, Sphinx::SphinxResponseError, Sphinx::SphinxTemporaryError, Errno::ECONNRESET, Errno::EPIPE => e
         if (tries += 1) <= self.class.client_options['max_retries']
           say "restarting query (#{tries} attempts already) (#{e})"
           sleep(self.class.client_options['retry_sleep_time']) if tries == self.class.client_options['max_retries']
           retry
         else
           say "query failed"
-          raise e
+          raise Sphinx::SphinxConnectError, e.to_s
         end
       end
       

data/lib/ultrasphinx/search/internals.rb

@@ -22,7 +22,12 @@ module Ultrasphinx
         offset, limit = opts['per_page'] * (opts['page'] - 1), opts['per_page']
         
         request.SetLimits offset, limit, [offset + limit, MAX_MATCHES].min
-        request.SetSortMode SPHINX_CLIENT_PARAMS['sort_mode'][opts['sort_mode']], opts['sort_by'].to_s
+        
+        if SPHINX_CLIENT_PARAMS['sort_mode'][opts['sort_mode']]
+          request.SetSortMode SPHINX_CLIENT_PARAMS['sort_mode'][opts['sort_mode']], opts['sort_by'].to_s
+        else
+          raise UsageError, "Sort mode #{opts['sort_mode'].inspect} is invalid"
+        end
       
         if weights = opts['weights']
           # Order the weights hash according to the field order for Sphinx, and set the missing fields to 1.0
@@ -30,10 +35,14 @@ module Ultrasphinx
             array << (weights[field] || 1.0)
           end)
         end
-      
+
         unless opts['class_names'].compact.empty?
-          request.SetFilter 'class_id', opts['class_names'].map{|m| MODELS_TO_IDS[m.to_s]}
-        end        
+          request.SetFilter('class_id', (opts['class_names'].map do |model| 
+            MODELS_TO_IDS[model.to_s] or 
+            MODELS_TO_IDS[model.to_s.constantize.base_class.to_s] or 
+            raise UsageError, "Invalid class name #{model.inspect}"
+          end))
+        end
       
         # Extract ranged raw filters 
         # Some of this mangling might not be necessary
@@ -90,7 +99,11 @@ module Ultrasphinx
         request.SetLimits 0, limit, [limit, MAX_MATCHES].min
         
         # Run the query
-        matches = request.Query(query)['matches']
+        begin
+          matches = request.Query(query)['matches']
+        rescue Sphinx::SphinxInternalError
+          raise ConfigurationError, "Index is out of date. Run 'rake ultrasphinx:index'"
+        end
                 
         # Map the facets back to something sane
         facets = {}
@@ -100,7 +113,7 @@ module Ultrasphinx
           facets[match['@groupby']] = match['@count']
         end
                 
-        # Invert crc's, if we have them
+        # Invert hash's, if we have them
         reverse_map_facets(facets, original_facet)
       end
       
@@ -109,9 +122,9 @@ module Ultrasphinx
       
         if Fields.instance.types[facet] == 'text'        
           # Apply the map, rebuilding if the cache is missing or out-of-date
-          facets = Hash[*(facets.map do |crc, value|
-            rebuild_facet_cache(facet) unless FACET_CACHE[facet] and FACET_CACHE[facet].has_key?(crc)
-            [FACET_CACHE[facet][crc], value]
+          facets = Hash[*(facets.map do |hash, value|
+            rebuild_facet_cache(facet) unless FACET_CACHE[facet] and FACET_CACHE[facet].has_key?(hash)
+            [FACET_CACHE[facet][hash], value]
           end.flatten)]
         end
         
@@ -119,18 +132,25 @@ module Ultrasphinx
       end
       
       def rebuild_facet_cache(facet)
-        # Cache the reverse CRC map for the textual facet if it hasn't been done yet
+        # Cache the reverse hash map for the textual facet if it hasn't been done yet
         # XXX not necessarily optimal since it requires a direct DB hit once per mongrel
-        Ultrasphinx.say "caching CRC reverse map for text facet #{facet}"
+        Ultrasphinx.say "caching hash reverse map for text facet #{facet}"
         
         Fields.instance.classes[facet].each do |klass|
           # you can only use a facet from your own self right now; no includes allowed
-          field = (MODEL_CONFIGURATION[klass.name]['fields'].detect do |field_hash|
+          field = MODEL_CONFIGURATION[klass.name]['fields'].detect do |field_hash|
             field_hash['as'] == facet
-          end)['field']
-      
-          klass.connection.execute("SELECT #{field} AS value, CRC32(#{field}) AS crc FROM #{klass.table_name} GROUP BY #{field}").each_hash do |hash|
-            (FACET_CACHE[facet] ||= {})[hash['crc'].to_i] = hash['value']
+          end
+                    
+          raise ConfigurationError, "Model #{klass.name} has the requested '#{facet}' field, but it was not configured for faceting" unless field
+          field = field['field']
+          
+          if hash_stored_procedure = ADAPTER_SQL_FUNCTIONS[ADAPTER]['hash_stored_procedure']
+            klass.connection.execute(hash_stored_procedure)
+          end
+                
+          klass.connection.execute("SELECT #{field} AS value, #{ADAPTER_SQL_FUNCTIONS[ADAPTER]['hash']._interpolate(field)} AS hash FROM #{klass.table_name} GROUP BY #{field}").each_hash do |hash|
+            (FACET_CACHE[facet] ||= {})[hash['hash'].to_i] = hash['value']
           end                            
         end
         FACET_CACHE[facet]

data/lib/ultrasphinx/spell.rb

@@ -10,13 +10,14 @@ In order to spellcheck your user's query, Ultrasphinx bundles a small spelling m
 
 Make sure Aspell and the Rubygem <tt>raspell</tt> are installed. See http://blog.evanweaver.com/files/doc/fauna/raspell/ for detailed instructions.
   
-Copy the <tt>examples/app.multi</tt> file into your Aspell dictionary folder (<tt>/opt/local/share/aspell/</tt> on Mac, <tt>/usr/lib/aspell-0.60/</tt> on Linux). This file lets Aspell load a custom wordlist generated by Sphinx from your app data. Modify the file if you don't want to also use the default American English dictionary.
+Copy the <tt>examples/ap.multi</tt> file into your Aspell dictionary folder (<tt>/opt/local/share/aspell/</tt> on Mac, <tt>/usr/lib/aspell-0.60/</tt> on Linux). This file lets Aspell load a custom wordlist generated by Sphinx from your app data (you can configure its filename in the <tt>config/ultrasphinx/*.base</tt> files). Modify the file if you don't want to also use the default American English dictionary.
   
 Finally, to build the custom wordlist, run:  
   sudo rake ultrasphinx:spelling:build  
 
 You need to use <tt>sudo</tt> because Ultrasphinx needs to write to the Aspell dictionary folder. Also note that Aspell, <tt>raspell</tt>, and the custom dictionary must be available on each application server, not on the Sphinx daemon server.
 
+
 == Usage
 
 Now you can see if a query is correctly spelled as so:
@@ -27,11 +28,19 @@ If <tt>@correction</tt> is not <tt>nil</tt>, go ahead and suggest it to the user
 =end
 
   module Spell  
-    SP = Aspell.new("app")   
-    SP.suggestion_mode = Aspell::NORMAL
-    SP.set_option("ignore-case", "true")
+  
+    begin    
+      SP = Aspell.new(Ultrasphinx::DICTIONARY)   
+      SP.suggestion_mode = Aspell::NORMAL
+      SP.set_option("ignore-case", "true")
+      Ultrasphinx.say "spelling support enabled"
+    rescue Object => e      
+      SP = nil
+      Ultrasphinx.say "spelling support not available (raspell configuration raised \"#{e}\")"
+    end    
     
     def self.correct string
+      return nil unless SP
       correction = string.gsub(/[\w\']+/) do |word| 
         unless SP.check(word)
           SP.suggest(word).first

data/lib/ultrasphinx/ultrasphinx.rb

@@ -1,13 +1,13 @@
 
 module Ultrasphinx
 
-  class Exception < ::Exception #:nodoc:
+  class Error < ::StandardError #:nodoc:
   end
-  class ConfigurationError < Exception #:nodoc:
+  class ConfigurationError < Error #:nodoc:
   end  
-  class DaemonError < Exception #:nodoc:
+  class DaemonError < Error #:nodoc:
   end
-  class UsageError < Exception #:nodoc:
+  class UsageError < Error #:nodoc:
   end
 
   # Internal file paths
@@ -15,6 +15,8 @@ module Ultrasphinx
   SUBDIR = "config/ultrasphinx"
   
   DIR = "#{RAILS_ROOT}/#{SUBDIR}"
+  
+  THIS_DIR = File.expand_path(File.dirname(__FILE__))
 
   CONF_PATH = "#{DIR}/#{RAILS_ENV}.conf"
   
@@ -55,29 +57,34 @@ module Ultrasphinx
   CONNECTION_DEFAULTS = {
     :host => 'localhost'
   }
-  
-  ADAPTER_DEFAULTS = {
-    'mysql' => %(
-type = mysql
-sql_query_pre = SET SESSION group_concat_max_len = 65535
-sql_query_pre = SET NAMES utf8
-  ), 
-    'postgresql' => %(
-type = pgsql
-  )}
-  
+
   ADAPTER_SQL_FUNCTIONS = {
     'mysql' => {
       'group_by' => 'GROUP BY id',
-      'timestamp' => 'UNIX_TIMESTAMP('
+      'timestamp' => 'UNIX_TIMESTAMP(?)',
+      'hash' => 'CRC32(?)'
     },    
     'postgresql' => {
       'group_by' => '',
-      'timestamp' => 'EXTRACT(EPOCH FROM '
+      'timestamp' => 'EXTRACT(EPOCH FROM ?)',
+      'hash' => 'hex_to_int(SUBSTRING(MD5(?) FROM 1 FOR 8))',
+      'hash_stored_procedure' => open("#{THIS_DIR}/hex_to_int.sql").read.gsub("\n", ' ')
     }      
   }
   
-  ADAPTER = ActiveRecord::Base.connection.instance_variable_get("@config")[:adapter]
+  ADAPTER_DEFAULTS = {
+    'mysql' => %(
+type = mysql
+sql_query_pre = SET SESSION group_concat_max_len = 65535
+sql_query_pre = SET NAMES utf8
+  ), 
+    'postgresql' => %(
+type = pgsql
+sql_query_pre = ) + ADAPTER_SQL_FUNCTIONS['postgresql']['hash_stored_procedure'] + %(
+  )
+}
+    
+  ADAPTER = ActiveRecord::Base.connection.instance_variable_get("@config")[:adapter] rescue 'mysql'
      
   mattr_accessor :with_rake
      
@@ -97,7 +104,7 @@ type = pgsql
   
   # Configuration file parser.
   def self.options_for(heading, path)
-    section = open(path).read[/^#{heading}\s*?\{(.*?)\}/m, 1]    
+    section = open(path).read[/^#{heading.gsub('/', '__')}\s*?\{(.*?)\}/m, 1]    
     
     unless section
       Ultrasphinx.say "warning; heading #{heading} not found in #{path}; it may be corrupted. "
@@ -121,8 +128,11 @@ type = pgsql
   
   # Make sure there's a trailing slash
   INDEX_SETTINGS['path'] = INDEX_SETTINGS['path'].chomp("/") + "/" 
+  
+  DICTIONARY = CLIENT_SETTINGS['dictionary_name'] || 'ap'  
+  raise ConfigurationError, "Aspell does not support dictionary names longer than two letters" if DICTIONARY.size > 2
 
-  STOPWORDS_PATH = "#{Ultrasphinx::INDEX_SETTINGS['path']}/stopwords.txt"
+  STOPWORDS_PATH = "#{Ultrasphinx::INDEX_SETTINGS['path']}/#{DICTIONARY}-stopwords.txt"
 
   MODEL_CONFIGURATION = {}     
 

data/tasks/ultrasphinx.rake

@@ -11,6 +11,8 @@ namespace :ultrasphinx do
   
   desc "Bootstrap a full Sphinx environment"
   task :bootstrap => [:_environment, :configure, :index, :"daemon:restart"] do
+    say "done"
+    say "please restart Mongrel"
   end
   
   desc "Rebuild the configuration file for this particular environment."
@@ -20,13 +22,27 @@ namespace :ultrasphinx do
   
   desc "Reindex the database and send an update signal to the search daemon."
   task :index => [:_environment] do
+    rotate = ultrasphinx_daemon_running?
     mkdir_p Ultrasphinx::INDEX_SETTINGS['path']
+    
     cmd = "indexer --config #{Ultrasphinx::CONF_PATH}"
     cmd << " #{ENV['OPTS']} " if ENV['OPTS']
-    cmd << " --rotate" if ultrasphinx_daemon_running?
+    cmd << " --rotate" if rotate
     cmd << " #{Ultrasphinx::UNIFIED_INDEX_NAME}"
+    
     say cmd
     system cmd
+        
+    if rotate
+      sleep(4)
+      failed = Dir[Ultrasphinx::INDEX_SETTINGS['path'] + "/*.new.*"]
+      if failed.any?
+        say "warning; index failed to rotate! Deleting new indexes"
+        failed.each {|f| File.delete f }
+      else
+        say "index rotated ok"
+      end
+    end
   end
   
   
@@ -36,7 +52,7 @@ namespace :ultrasphinx do
       FileUtils.mkdir_p File.dirname(Ultrasphinx::DAEMON_SETTINGS["log"]) rescue nil
       raise Ultrasphinx::DaemonError, "Already running" if ultrasphinx_daemon_running?
       system "searchd --config #{Ultrasphinx::CONF_PATH}"
-      sleep(2) # give daemon a chance to write the pid file
+      sleep(4) # give daemon a chance to write the pid file
       if ultrasphinx_daemon_running?
         say "started successfully"
       else
@@ -48,7 +64,16 @@ namespace :ultrasphinx do
     task :stop => [:_environment] do
       raise Ultrasphinx::DaemonError, "Doesn't seem to be running" unless ultrasphinx_daemon_running?
       system "kill #{pid = ultrasphinx_daemon_pid}"
-      say "stopped #{pid}."
+      sleep(1)
+      if ultrasphinx_daemon_running?
+        system "kill -9 #{pid}"  
+        sleep(1)
+      end
+      if ultrasphinx_daemon_running?
+        say "#{pid} could not be stopped"
+      else
+        say "stopped #{pid}"
+      end
     end
 
     desc "Restart the search daemon"
@@ -74,7 +99,7 @@ namespace :ultrasphinx do
     task :build => [:_environment] do    
       ENV['OPTS'] = "--buildstops #{Ultrasphinx::STOPWORDS_PATH} #{Ultrasphinx::MAX_WORDS} --buildfreqs"
       Rake::Task["ultrasphinx:index"].invoke
-      tmpfile = "/tmp/custom_words.txt"
+      tmpfile = "/tmp/ultrasphinx-stopwords.txt"
       words = []
       say "filtering"
       File.open(Ultrasphinx::STOPWORDS_PATH).each do |line|
@@ -87,8 +112,8 @@ namespace :ultrasphinx do
       end
       say "writing #{words.size} words"
       File.open(tmpfile, 'w').write(words.join("\n"))
-      say "loading into aspell"
-      system("aspell --lang=en create master custom.rws < #{tmpfile}")
+      say "loading dictionary '#{Ultrasphinx::DICTIONARY}' into aspell"
+      system("aspell --lang=en create master #{Ultrasphinx::DICTIONARY}.rws < #{tmpfile}")
     end
   end