ultrasphinx 1.8 → 1.9

data/lib/ultrasphinx/search/parser.rb

@@ -2,8 +2,7 @@
 module Ultrasphinx
   class Search
     module Parser    
-      # We could rewrite this in Treetop, but since it works well, what's the point? It's
-      # probably faster this way. Ragel would speed it up, but be too much work.
+      # We could rewrite this in Treetop, but for now it works well.
           
       class Error < RuntimeError
       end
@@ -50,7 +49,15 @@ module Ultrasphinx
           query << ")" if field and contents.size > 1        
         end
         
-        # XXX swap the first pair if the order is reversed
+        # Collapse fieldsets early so that the swap doesn't split them        
+        query.each_with_index do |token, index|
+          if token =~ /^@/
+            query[index] = "#{token} #{query[index + 1]}"
+            query[index + 1] = nil
+          end
+        end
+        
+        # Swap the first pair if the order is reversed
         if [OPERATORS['NOT'], OPERATORS['OR']].include? query.first.upcase
           query[0], query[1] = query[1], query[0]
         end
@@ -83,10 +90,11 @@ module Ultrasphinx
           if subtoken =~ /\"(.*)\"/
             subtoken.sub!($1, $1.gsub(/[()]/, ''))
           end
-          
-          # add to the stream, converting the operator
+         
+          # Add to the stream, converting the operator
           if !has_operator
-            if OPERATORS.to_a.flatten.include? subtoken and index != (query.size - 1) # Operators at the end of the string are not parsed
+            if OPERATORS.to_a.flatten.include? subtoken and index != (query.size - 1) 
+              # Note that operators at the end of the string are not parsed
               token_stream << OPERATORS[subtoken] || subtoken
               has_operator = true # flip
             else
@@ -103,7 +111,9 @@ module Ultrasphinx
           end        
         end
         
-        raise Error, "#{token_stream.inspect} is not a valid token stream" unless token_stream.size % 2 == 0        
+        if token_stream.size.zero? or token_stream.size.odd?
+          raise Error, "#{token_stream.inspect} is not a valid token stream"
+        end
         token_stream.in_groups_of(2) 
       end
       

data/lib/ultrasphinx/ultrasphinx.rb

@@ -32,12 +32,16 @@ module Ultrasphinx
 
   MAX_INT = 2**32-1
 
-  MAX_WORDS = 2**16 # maximum number of stopwords built  
+  MAX_WORDS = 2**16 # The maximum number of stopwords built  
   
-  UNIFIED_INDEX_NAME = "complete"
+  MAIN_INDEX = "main"
+  
+  DELTA_INDEX = "delta"
+  
+  INDEXES = [MAIN_INDEX, DELTA_INDEX]
 
   CONFIG_MAP = {
-    # These must be symbols for key mapping against Rails itself
+    # These must be symbols for key mapping against Rails itself.
     :username => 'sql_user',
     :password => 'sql_pass',
     :host => 'sql_host',
@@ -59,12 +63,14 @@ module Ultrasphinx
   SQL_FUNCTIONS = {
     'mysql' => {
       'group_concat' => "CAST(GROUP_CONCAT(DISTINCT ? SEPARATOR ' ') AS CHAR)",
+      'delta' => "DATE_SUB(NOW(), INTERVAL ? SECOND)",      
       'hash' => "CAST(CRC32(?) AS unsigned)",
       'range_cast' => "?",
       'stored_procedures' => {}
     },
     'postgresql' => {
       'group_concat' => "GROUP_CONCAT(?)",
+      'delta' => "(NOW() - '? SECOND'::interval)",
       'range_cast' => "cast(coalesce(?,1) AS integer)",
       'hash' => "CRC32(?)",
       'stored_procedures' => Hash[*(
@@ -90,7 +96,8 @@ sql_query_pre = ) + SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join
     
   ADAPTER = ActiveRecord::Base.connection.instance_variable_get("@config")[:adapter] rescue 'mysql'
   
-  # Install the stored procedures
+  # Install the stored procedures.
+  # XXX This shouldn't be done at every index, say the Postgres people.
   SQL_FUNCTIONS[ADAPTER]['stored_procedures'].each do |key, value|
     ActiveRecord::Base.connection.execute(value)
   end
@@ -106,34 +113,71 @@ sql_query_pre = ) + SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join
       else
         STDERR.puts msg
       end
-    end
+    end        
+    nil # Explicitly return nil
   end
   
   # Configuration file parser.
   def self.options_for(heading, path)
-    section = open(path).read[/^#{heading.gsub('/', '__')}\s*?\{(.*?)\}/m, 1]    
+    # Evaluate ERB
+    template = ERB.new(File.open(path) {|f| f.read})
+    contents = template.result(binding)
     
-    unless section
-      Ultrasphinx.say "warning; heading #{heading} not found in #{path}; it may be corrupted. "
-      {}
-    else      
+    # Find the correct heading.
+    section = contents[/^#{heading.gsub('/', '__')}\s*?\{(.*?)\}/m, 1]
+    
+    if section
+      # Convert to a hash
       options = section.split("\n").map do |line|
         line =~ /\s*(.*?)\s*=\s*([^\#]*)/
         $1 ? [$1, $2.strip] : []
       end      
       Hash[*options.flatten] 
-    end
-    
+    else
+      # XXX Is it safe to raise here?
+      Ultrasphinx.say "warning; heading #{heading} not found in #{path}; it may be corrupted. "
+      {}    
+    end    
   end
+  
+  def self.get_models_to_class_ids #:nodoc:
+    # Reading the conf file makes sure that we are in sync with the actual Sphinx index, not
+    # whatever you happened to change your models to most recently.
+    if File.exist? CONF_PATH
+      lines, hash = open(CONF_PATH).readlines, {}
+      msg = "#{CONF_PATH} file is corrupted. Please run 'rake ultrasphinx:configure'."
+      
+      lines.each_with_index do |line, index| 
+        # Find the main sources
+        if line =~ /^source ([\w\d_-]*)_#{MAIN_INDEX}/
+          # Derive the model name
+          model = $1.gsub('__', '/').classify
 
-  # Introspect on the existing generated conf files
+          # Get the id modulus out of the adjacent sql_query
+          query = lines[index..-1].detect do |query_line|
+            query_line =~ /^sql_query /
+          end
+          raise ConfigurationError, msg unless query
+          hash[model] = query[/(\d*) AS class_id/, 1].to_i
+        end  
+      end            
+      raise ConfigurationError, msg unless hash.values.size == hash.values.uniq.size      
+      hash          
+    else
+      # We can't raise here because you may be generating the configuration for the first time
+      Ultrasphinx.say "configuration file not found for #{RAILS_ENV.inspect} environment"
+      Ultrasphinx.say "please run 'rake ultrasphinx:configure'"
+    end      
+  end  
+
+  # Introspect on the existing generated conf files.
   INDEXER_SETTINGS = options_for('indexer', BASE_PATH)
   CLIENT_SETTINGS = options_for('client', BASE_PATH)
   DAEMON_SETTINGS = options_for('searchd', BASE_PATH)
   SOURCE_SETTINGS = options_for('source', BASE_PATH)
   INDEX_SETTINGS = options_for('index', BASE_PATH)
   
-  # Make sure there's a trailing slash
+  # Make sure there's a trailing slash.
   INDEX_SETTINGS['path'] = INDEX_SETTINGS['path'].chomp("/") + "/" 
   
   DICTIONARY = CLIENT_SETTINGS['dictionary_name'] || 'ap'  
@@ -148,7 +192,7 @@ sql_query_pre = ) + SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join
     if File.exist? CONF_PATH
       begin
         if options_for(
-          "source #{MODEL_CONFIGURATION.keys.first.tableize}", 
+          "source #{MODEL_CONFIGURATION.keys.first.tableize}_#{MAIN_INDEX}", 
           CONF_PATH
         )['sql_db'] != ActiveRecord::Base.connection.instance_variable_get("@config")[:database]
           say "warning; configured database name is out-of-date"
@@ -158,5 +202,14 @@ sql_query_pre = ) + SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join
       end
     end
   end
-        
+  
+  # See if a delta index was defined.
+  def self.delta_index_present?
+    if File.exist?(CONF_PATH) 
+      File.open(CONF_PATH).readlines.detect do |line|
+        line =~ /^index delta/
+      end
+    end
+  end
+  
 end

data/tasks/ultrasphinx.rake

@@ -12,7 +12,7 @@ namespace :ultrasphinx do
   desc "Bootstrap a full Sphinx environment"
   task :bootstrap => [:_environment, :configure, :index, :"daemon:restart"] do
     say "done"
-    say "please restart Mongrel"
+    say "please restart your application containers"
   end
   
   desc "Rebuild the configuration file for this particular environment."
@@ -20,31 +20,23 @@ namespace :ultrasphinx do
     Ultrasphinx::Configure.run
   end
   
-  desc "Reindex the database and send an update signal to the search daemon."
-  task :index => [:_environment] do
-    rotate = ultrasphinx_daemon_running?
-    mkdir_p Ultrasphinx::INDEX_SETTINGS['path']
-    
-    cmd = "indexer --config '#{Ultrasphinx::CONF_PATH}'"
-    cmd << " #{ENV['OPTS']} " if ENV['OPTS']
-    cmd << " --rotate" if rotate
-    cmd << " #{Ultrasphinx::UNIFIED_INDEX_NAME}"
-    
-    say cmd
-    system cmd
-        
-    if rotate
-      sleep(4)
-      failed = Dir[Ultrasphinx::INDEX_SETTINGS['path'] + "/*.new.*"]
-      if failed.any?
-        say "warning; index failed to rotate! Deleting new indexes"
-        failed.each {|f| File.delete f }
-      else
-        say "index rotated ok"
-      end
+  namespace :index do    
+    desc "Reindex and rotate the main index."
+    task :main => [:_environment] do
+      ultrasphinx_index(Ultrasphinx::MAIN_INDEX)
     end
+
+    desc "Reindex and rotate the delta index."    
+    task :delta => [:_environment] do
+      ultrasphinx_index(Ultrasphinx::DELTA_INDEX)
+    end
+    
+  end
+
+  desc "Reindex and rotate all indexes."  
+  task :index => [:_environment]  do
+    ultrasphinx_index("--all")
   end
-  
   
   namespace :daemon do
     desc "Start the search daemon"
@@ -125,30 +117,56 @@ namespace :us do
   task :restart => ["ultrasphinx:daemon:restart"]
   task :stop => ["ultrasphinx:daemon:stop"]
   task :stat => ["ultrasphinx:daemon:status"]
+  task :index => ["ultrasphinx:index"]
   task :in => ["ultrasphinx:index"]
+  task :main => ["ultrasphinx:index:main"]
+  task :delta => ["ultrasphinx:index:delta"]
   task :spell => ["ultrasphinx:spelling:build"]
   task :conf => ["ultrasphinx:configure"]  
   task :boot => ["ultrasphinx:bootstrap"]  
 end
 
-# support methods
+# Support methods
 
 def ultrasphinx_daemon_pid
-  open(open(Ultrasphinx::BASE_PATH).readlines.map do |line| 
-    line[/^\s*pid_file\s*=\s*([^\s\#]*)/, 1]
-  end.compact.first).readline.chomp rescue nil # XXX ridiculous
+  open(Ultrasphinx::DAEMON_SETTINGS['pid_file']).readline.chomp rescue nil
 end
 
 def ultrasphinx_daemon_running?
   if ultrasphinx_daemon_pid and `ps #{ultrasphinx_daemon_pid} | wc`.to_i > 1 
     true
   else
-    # remove bogus lockfiles
+    # Remove bogus lockfiles.
     Dir[Ultrasphinx::INDEX_SETTINGS["path"] + "*spl"].each {|file| File.delete(file)}
     false
   end  
 end
 
+def ultrasphinx_index(index)
+  rotate = ultrasphinx_daemon_running?
+  index_path = Ultrasphinx::INDEX_SETTINGS['path']
+  mkdir_p index_path unless File.directory? index_path
+  
+  cmd = "indexer --config '#{Ultrasphinx::CONF_PATH}'"
+  cmd << " #{ENV['OPTS']} " if ENV['OPTS']
+  cmd << " --rotate" if rotate
+  cmd << " #{index}"
+  
+  say "$ #{cmd}"
+  system cmd
+      
+  if rotate
+    sleep(4)
+    failed = Dir[index_path + "/*.new.*"]
+    if failed.any?
+      say "warning; index failed to rotate! Deleting new indexes"
+      failed.each {|f| File.delete f }
+    else
+      say "index rotated ok"
+    end
+  end
+end
+
 def say msg
   Ultrasphinx.say msg
 end

data/test/config/ultrasphinx/test.base

@@ -2,55 +2,84 @@
 #
 # Sphinx/Ultrasphinx user-configurable options.
 #
-# Copy this file to RAILS_ROOT/config/ultrasphinx.
-# You can use individual namespaces if you want (e.g. "development.base").
+# Copy this file to RAILS_ROOT/config/ultrasphinx. You can use individual 
+# namespaces if you want (e.g. development.base, production.base, 
+# test.base). Note that ERb is also allowed.
+#
+# This file should not be handed directly to Sphinx. Use the rake task 
+#
+#   rake ultrasphinx::configure
+#
+# to generate a parallel default.conf file. This is the file that Sphinx itself will
+# use. The Ultrasphinx rake tasks automatically pass the correct file to 
+# to Sphinx.
+# 
+# It is safe to edit .base files by hand. It is not safe to edit the generated 
+# .conf files. Do not symlink the .conf file to the .base file; it's wrong.
 #
 
+<% path = "/opt/local/var/db/sphinx/" %>
+
+# Indexing options
 indexer
-{
-  # Indexer running options
+{  
   mem_limit = 256M 
+
+  # Ultrasphinx-specific key
+  delta = <%= 1.day + 30.minutes %> 
 }
 
+# Daemon options
 searchd
-{
-  # Daemon options
+{  
   # What interface the search daemon should listen on and where to store its logs
   address = 0.0.0.0
   port = 3312
-  log = /opt/local/var/db/sphinx/log/searchd.log
-  query_log = /opt/local/var/db/sphinx/log/query.log
+  seamless_rotate = 1
+  log = <%= path %>log/searchd.log
+  query_log = <%= path %>log/query.log
   read_timeout = 5
   max_children = 300
-  pid_file = /opt/local/var/db/sphinx/log/searchd.pid
+  pid_file = <%= path %>log/searchd.pid
   max_matches = 100000
 }
 
+# Client options
 client
 {
-  # Client options
+  # Name of the Aspell dictionary (two letters max)
+  dictionary_name = ap
+  
   # How your application connects to the search daemon (not necessarily the same as above)
   server_host = localhost
   server_port = 3312
 }
 
+# Individual SQL source options
 source
-{
-  # Individual SQL source options
-  sql_range_step = 20000   
-  strip_html = 0
-  index_html_attrs =
+{  
+  sql_ranged_throttle = 0  
+  sql_range_step = 5000   
   sql_query_post =
 }
 
+# Index building options
 index
-{
-  # Index building options
-  path = /opt/local/var/db/sphinx/
-  docinfo = extern # just leave this alone
+{  
+  path = <%= path %>
+  docinfo = extern # Just leave this alone
   morphology = stem_en
-  stopwords = # /path/to/stopwords.txt
+  stopwords = # <%= path %>/ap-stopwords.txt
   min_word_len = 1
+
+  # HTML-specific options
+  html_strip = 0
+  html_index_attrs =
+  
+  # Enable these if you need wildcard searching. They will slow down indexing significantly.
+  #   min_infix_len = 1    
+  #   enable_star = 1 
+    
   charset_type = utf-8 # or sbcs (Single Byte Character Set)
-  charset_table = 0..9, A..Z->a..z, -, _, ., &, a..z,
+  charset_table = 0..9, A..Z->a..z, -, _, ., &, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F,U+C5->U+E5, U+E5, U+C4->U+E4, U+E4, U+D6->U+F6, U+F6, U+16B, U+0c1->a, U+0c4->a, U+0c9->e, U+0cd->i, U+0d3->o, U+0d4->o, U+0da->u, U+0dd->y, U+0e1->a, U+0e4->a, U+0e9->e, U+0ed->i, U+0f3->o, U+0f4->o, U+0fa->u, U+0fd->y, U+104->U+105, U+105, U+106->U+107, U+10c->c, U+10d->c, U+10e->d, U+10f->d, U+116->U+117, U+117, U+118->U+119, U+11a->e, U+11b->e, U+12E->U+12F, U+12F, U+139->l, U+13a->l, U+13d->l, U+13e->l, U+141->U+142, U+142, U+143->U+144, U+144,U+147->n, U+148->n, U+154->r, U+155->r, U+158->r, U+159->r, U+15A->U+15B, U+15B, U+160->s, U+160->U+161, U+161->s, U+164->t, U+165->t, U+16A->U+16B, U+16B, U+16e->u, U+16f->u, U+172->U+173, U+173, U+179->U+17A, U+17A, U+17B->U+17C, U+17C, U+17d->z, U+17e->z,
 }

data/test/integration/app/app/models/geo/address.rb

@@ -4,5 +4,6 @@ class Geo::Address < ActiveRecord::Base
   
   is_indexed 'fields' => ['name'],
     'concatenate' => [{'fields' => ['line_1', 'line_2', 'city', 'province_region', 'zip_postal_code'], 'as' => 'content'}],
-    'include' => [{'association_name' => 'state', 'field' => 'name', 'as' => 'state'}]
+    'include' => [{'association_name' => 'state', 'field' => 'name', 'as' => 'state'}],
+    'delta' => true
 end

data/test/integration/app/app/models/person/user.rb

@@ -3,8 +3,17 @@ class User < ActiveRecord::Base
   has_one   :address, :class_name => "Geo::Address"
 
   is_indexed :fields => ['login', 'email', 'deleted'], 
-    :include => [{:association_name => 'specific_seller', :field => 'company_name', :as => 'company'},
-      {:class_name => 'Seller', :field => 'sellers_two.company_name', :as => 'company_two', 'association_sql' => 'LEFT OUTER JOIN sellers AS sellers_two ON users.id = sellers_two.user_id', 'function_sql' => "REPLACE(?, '6', ' replacement ')"}],
-    :conditions => "deleted = '0'"
+    :include => [{:association_name => 'specific_seller', :field => 'company_name', :as => 'company', :facet => true},
+      {:class_name => 'Seller', :field => 'sellers_two.company_name', :as => 'company_two', :facet => true, 'association_sql' => 'LEFT OUTER JOIN sellers AS sellers_two ON users.id = sellers_two.user_id', 'function_sql' => "REPLACE(?, '6', ' replacement ')"}],
+    :conditions => "deleted = '0'",
+    :delta => {:field => 'created_at'}    
+  
+  def self.find_all_by_id(*args)
+    raise "Wrong finder"
+  end
+    
+  def self.custom_find(*args)
+    method_missing(:find_all_by_id, *args)
+  end  
   
 end