ultrasphinx 1.6.7 → 1.7

data/lib/ultrasphinx/search.rb

@@ -120,6 +120,7 @@ Note that your database is never changed by anything Ultrasphinx does.
     self.client_options ||= HashWithIndifferentAccess.new({ 
       :with_subtotals => false, 
       :ignore_missing_records => false,
+      :max_missing_records => 5, # Has no effect if :ignore_missing_records => false
       :max_retries => 4,
       :retry_sleep_time => 0.5,
       :max_facets => 100,

data/lib/ultrasphinx/search/internals.rb

@@ -2,6 +2,7 @@
 module Ultrasphinx
   class Search
     module Internals
+      include Associations
 
       # These methods are kept stateless to ease debugging
       
@@ -184,9 +185,10 @@ module Ultrasphinx
             when 'fields'
               [configuration['field'], ""]
             when 'include'
+              association_model = get_association_model(klass, configuration)
               # XXX Only handles the basic case. No test coverage.
               ["included.#{configuration['field']}", 
-                (configuration['association_sql'] or "LEFT OUTER JOIN #{configuration['table']} AS included ON included.#{configuration['class_name'].constantize.primary_key} = #{klass.table_name}.#{configuration['class_name'].underscore}_id")
+                (configuration['association_sql'] or "LEFT OUTER JOIN #{configuration['table']} AS included ON included.#{association_model.primary_key} = #{klass.table_name}.#{association_model.class_name.underscore}_id")
               ]
             when 'concatenate'
               # Wait for someone to complain before worrying about this
@@ -203,8 +205,8 @@ module Ultrasphinx
         raise ConfigurationError, "no classes were correctly configured for text faceting on '#{facet}'" if configured_classes.empty?      
         
         FACET_CACHE[facet]
-      end      
-      
+      end 
+            
       # Inverse-modulus map the Sphinx ids to the table-specific ids
       def convert_sphinx_ids(sphinx_ids)    
         sphinx_ids.sort_by do |item| 
@@ -251,6 +253,11 @@ module Ultrasphinx
           result._metaclass.send('define_method', 'result_index') { i }
         end
         
+        if ids.size - results.size > Ultrasphinx::Search.client_options['max_missing_records']
+          # Never reached if Ultrasphinx::Search.client_options['ignore_missing_records'] is false
+          raise ConfigurationError, "Too many results for this query returned ActiveRecord::RecordNotFound. The index is probably out of date" 
+        end
+        
         results        
       end  
       

data/lib/ultrasphinx/search/parser.rb

@@ -1,9 +1,12 @@
 
 module Ultrasphinx
   class Search
-    module Parser
-      
-      class Error < RuntimeError; end
+    module Parser    
+      # We could rewrite this in Treetop, but since it works well, what's the point? It's
+      # probably faster this way. Ragel would speed it up, but be too much work.
+          
+      class Error < RuntimeError
+      end
 
       OPERATORS = {
         'OR' => '|',
@@ -32,7 +35,7 @@ module Ultrasphinx
         token_hash.sort_by do |key, value| 
           key or ""
         end.each do |field, contents|
-          # first operator always goes outside
+          # First operator always goes outside
           query << contents.first.first 
           
           query << "@#{field}" if field
@@ -66,24 +69,24 @@ module Ultrasphinx
         
         query.each_with_index do |subtoken, index|
       
-          # recurse for parens, if necessary
+          # Recurse for parens, if necessary
           if subtoken =~ /^(.*?)\((.*)\)(.*?$)/
             subtoken = query[index] = "#{$1}(#{parse $2})#{$3}"
           end 
           
-          # reappend missing closing quotes
+          # Reappend missing closing quotes
           if subtoken =~ /(^|\:)\"/
             subtoken = subtoken.chomp('"') + '"'
           end
           
-          # strip parentheses within quoted strings
+          # Strip parentheses within quoted strings
           if subtoken =~ /\"(.*)\"/
             subtoken.sub!($1, $1.gsub(/[()]/, ''))
           end
           
           # add to the stream, converting the operator
           if !has_operator
-            if OPERATORS.to_a.flatten.include? subtoken and index != (query.size - 1) # operators at the end of the string are not parsed
+            if OPERATORS.to_a.flatten.include? subtoken and index != (query.size - 1) # Operators at the end of the string are not parsed
               token_stream << OPERATORS[subtoken] || subtoken
               has_operator = true # flip
             else
@@ -92,7 +95,7 @@ module Ultrasphinx
             end
           else
             if OPERATORS.to_a.flatten.include? subtoken
-              # drop extra operator
+              # Drop extra operator
             else
               token_stream << subtoken
               has_operator = false # flop
@@ -108,9 +111,9 @@ module Ultrasphinx
       def token_stream_to_hash(token_stream)
         token_hash = Hash.new([])        
         token_stream.map do |operator, content|
-          # remove some spaces
+          # Remove some spaces
           content.gsub!(/^"\s+|\s+"$/, '"')
-          # convert fields into sphinx style, reformat the stream object
+          # Convert fields into sphinx style, reformat the stream object
           if content =~ /(.*?):(.*)/
             token_hash[$1] += [[operator, $2]]
           else

data/lib/ultrasphinx/ultrasphinx.rb

@@ -56,13 +56,15 @@ module Ultrasphinx
     open("#{THIS_DIR}/postgresql/#{name}.sql").read.gsub(/\s+/, ' ')
   end
 
-  ADAPTER_SQL_FUNCTIONS = {
+  SQL_FUNCTIONS = {
     'mysql' => {
       'group_concat' => "CAST(GROUP_CONCAT(DISTINCT ? SEPARATOR ' ') AS CHAR)",
+      'range_cast' => "?",
       'stored_procedures' => {}
     },
     'postgresql' => {
       'group_concat' => "GROUP_CONCAT(?)",
+      'range_cast' => "cast(coalesce(?,1) AS integer)",
       'stored_procedures' => Hash[*(
         ['hex_to_int', 'group_concat', 'concat_ws', 'unix_timestamp', 'crc32'].map do |name|
           [name, load_stored_procedure(name)]
@@ -72,7 +74,7 @@ module Ultrasphinx
     }      
   }
   
-  ADAPTER_DEFAULTS = {
+  DEFAULTS = {
     'mysql' => %(
 type = mysql
 sql_query_pre = SET SESSION group_concat_max_len = 65535
@@ -80,14 +82,14 @@ sql_query_pre = SET NAMES utf8
   ), 
     'postgresql' => %(
 type = pgsql
-sql_query_pre = ) + ADAPTER_SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join(' ') + %(
+sql_query_pre = ) + SQL_FUNCTIONS['postgresql']['stored_procedures'].values.join(' ') + %(
   )
 }
     
   ADAPTER = ActiveRecord::Base.connection.instance_variable_get("@config")[:adapter] rescue 'mysql'
   
   # Install the stored procedures
-  ADAPTER_SQL_FUNCTIONS[ADAPTER]['stored_procedures'].each do |key, value|
+  SQL_FUNCTIONS[ADAPTER]['stored_procedures'].each do |key, value|
     ActiveRecord::Base.connection.execute(value)
   end
   

data/test/integration/app/app/models/geo/address.rb

@@ -4,5 +4,5 @@ class Geo::Address < ActiveRecord::Base
   
   is_indexed 'fields' => ['name'],
     'concatenate' => [{'fields' => ['line_1', 'line_2', 'city', 'province_region', 'zip_postal_code'], 'as' => 'content'}],
-    'include' => [{'class_name' => 'Geo::State', 'field' => 'name', 'as' => 'state'}]
+    'include' => [{'association_name' => 'state', 'field' => 'name', 'as' => 'state'}]
 end

data/test/integration/app/app/models/geo/country.rb

@@ -0,0 +1,4 @@
+
+class Geo::Country < ActiveRecord::Base
+  is_indexed :fields => ['name']
+end

data/test/integration/app/config/boot.rb

@@ -1,45 +1,108 @@
-# Don't change this file. Configuration is done in config/environment.rb and config/environments/*.rb
+# Don't change this file!
+# Configure your app in config/environment.rb and config/environments/*.rb
 
-unless defined?(RAILS_ROOT)
-  root_path = File.join(File.dirname(__FILE__), '..')
+RAILS_ROOT = "#{File.dirname(__FILE__)}/.." unless defined?(RAILS_ROOT)
 
-  unless RUBY_PLATFORM =~ /(:?mswin|mingw)/
-    require 'pathname'
-    root_path = Pathname.new(root_path).cleanpath(true).to_s
-  end
+module Rails
+  class << self
+    def boot!
+      unless booted?
+        preinitialize
+        pick_boot.run
+      end
+    end
 
-  RAILS_ROOT = root_path
-end
+    def booted?
+      defined? Rails::Initializer
+    end
+
+    def pick_boot
+      (vendor_rails? ? VendorBoot : GemBoot).new
+    end
+
+    def vendor_rails?
+      File.exist?("#{RAILS_ROOT}/vendor/rails")
+    end
 
-unless defined?(Rails::Initializer)
-  if File.directory?("#{RAILS_ROOT}/vendor/rails")
-    require "#{RAILS_ROOT}/vendor/rails/railties/lib/initializer"
-  else
-    require 'rubygems'
+    def preinitialize
+      load(preinitializer_path) if File.exists?(preinitializer_path)
+    end
 
-    environment_without_comments = IO.readlines(File.dirname(__FILE__) + '/environment.rb').reject { |l| l =~ /^#/ }.join
-    environment_without_comments =~ /[^#]RAILS_GEM_VERSION = '([\d.]+)'/
-    rails_gem_version = $1
+    def preinitializer_path
+      "#{RAILS_ROOT}/config/preinitializer.rb"
+    end
+  end
 
-    if version = defined?(RAILS_GEM_VERSION) ? RAILS_GEM_VERSION : rails_gem_version
-      # Asking for 1.1.6 will give you 1.1.6.5206, if available -- makes it easier to use beta gems
-      rails_gem = Gem.cache.search('rails', "~>#{version}.0").sort_by { |g| g.version.version }.last
+  class Boot
+    def run
+      load_initializer
+      Rails::Initializer.run(:set_load_path)
+    end
+  end
 
-      if rails_gem
-        gem "rails", "=#{rails_gem.version.version}"
-        require rails_gem.full_gem_path + '/lib/initializer'
+  class VendorBoot < Boot
+    def load_initializer
+      require "#{RAILS_ROOT}/vendor/rails/railties/lib/initializer"
+    end
+  end
+
+  class GemBoot < Boot
+    def load_initializer
+      self.class.load_rubygems
+      load_rails_gem
+      require 'initializer'
+    end
+
+    def load_rails_gem
+      if version = self.class.gem_version
+        gem 'rails', version
       else
-        STDERR.puts %(Cannot find gem for Rails ~>#{version}.0:
-    Install the missing gem with 'gem install -v=#{version} rails', or
-    change environment.rb to define RAILS_GEM_VERSION with your desired version.
-  )
+        gem 'rails'
+      end
+    rescue Gem::LoadError => load_error
+      $stderr.puts %(Missing the Rails #{version} gem. Please `gem install -v=#{version} rails`, update your RAILS_GEM_VERSION setting in config/environment.rb for the Rails version you do have installed, or comment out RAILS_GEM_VERSION to use the latest version installed.)
+      exit 1
+    end
+
+    class << self
+      def rubygems_version
+        Gem::RubyGemsVersion if defined? Gem::RubyGemsVersion
+      end
+
+      def gem_version
+        if defined? RAILS_GEM_VERSION
+          RAILS_GEM_VERSION
+        elsif ENV.include?('RAILS_GEM_VERSION')
+          ENV['RAILS_GEM_VERSION']
+        else
+          parse_gem_version(read_environment_rb)
+        end
+      end
+
+      def load_rubygems
+        require 'rubygems'
+
+        unless rubygems_version >= '0.9.4'
+          $stderr.puts %(Rails requires RubyGems >= 0.9.4 (you have #{rubygems_version}). Please `gem update --system` and try again.)
+          exit 1
+        end
+
+      rescue LoadError
+        $stderr.puts %(Rails requires RubyGems >= 0.9.4. Please install RubyGems and try again: http://rubygems.rubyforge.org)
         exit 1
       end
-    else
-      gem "rails"
-      require 'initializer'
+
+      def parse_gem_version(text)
+        $1 if text =~ /^[^#]*RAILS_GEM_VERSION\s*=\s*'([!~<>=]*\s*[\d.]+)'/
+      end
+
+      private
+        def read_environment_rb
+          File.read("#{RAILS_ROOT}/config/environment.rb")
+        end
     end
   end
-
-  Rails::Initializer.run(:set_load_path)
 end
+
+# All that for this:
+Rails.boot!

data/test/integration/app/config/environment.rb

@@ -1,8 +1,11 @@
 
+RAILS_GEM_VERSION = ENV['MULTIRAILS_RAILS_VERSION'] if ENV['MULTIRAILS_RAILS_VERSION']
+
 require File.join(File.dirname(__FILE__), 'boot')
+require 'action_controller'
 
 Rails::Initializer.run do |config|
-  config.action_controller.session = { :session_key => "_myapp_session", :secret => "this is a super secret phrase" }
+  config.action_controller.session = { :session_key => "_myapp_session", :secret => "7c74979e7db2230f84adbb4b3eb77d05" }
   config.load_paths << "#{RAILS_ROOT}/app/models/person" # moduleless model path
 end
 

data/test/integration/app/config/environments/development.rb

@@ -1,7 +1,7 @@
-config.cache_classes = false
+config.cache_classes = ENV['PRODUCTION']
 config.whiny_nils = true
 config.action_controller.consider_all_requests_local = true
-config.action_controller.perform_caching             = false
-config.action_view.cache_template_extensions         = false
-config.action_view.debug_rjs                         = true
+config.action_controller.perform_caching = ENV['PRODUCTION']
+config.action_view.cache_template_extensions = false
+config.action_view.debug_rjs = true
 config.action_mailer.raise_delivery_errors = false

data/test/integration/app/config/ultrasphinx/development.conf.canonical

@@ -1,5 +1,5 @@
 
-# Auto-generated at Wed Dec 12 02:47:44 -0500 2007.
+# Auto-generated at Sat Jan 12 08:05:26 -0500 2008.
 # Hand modifications will be overwritten.
 # /Users/eweaver/Desktop/projects/chow/vendor/plugins/ultrasphinx/test/integration/app/config/ultrasphinx/default.base
 indexer {
@@ -33,8 +33,8 @@ sql_db = ultrasphinx_development
 sql_host = localhost
 sql_pass = 
 sql_user = root
-sql_query_range = SELECT MIN(id), MAX(id) FROM states
-sql_query = SELECT (states.id * 4 + 0) AS id, CAST(GROUP_CONCAT(DISTINCT addresses.name SEPARATOR ' ') AS CHAR) AS address_name, 0 AS capitalization, 'Geo::State' AS class, 0 AS class_id, '' AS company, '' AS company_name, 0 AS company_name_facet, '' AS company_two, '' AS content, 18000 AS created_at, 0 AS deleted, '' AS email, '' AS login, '' AS mission_statement, '' AS name, '' AS state, 0 AS user_id FROM states LEFT OUTER JOIN addresses AS addresses ON states.id = addresses.state_id WHERE states.id >= $start AND states.id <= $end GROUP BY states.id
+sql_query_range = SELECT MIN(id) ,  MAX(id) FROM states
+sql_query = SELECT (states.id * 5 + 0) AS id, CAST(GROUP_CONCAT(DISTINCT addresses.name SEPARATOR ' ') AS CHAR) AS address_name, 0 AS capitalization, 'Geo::State' AS class, 0 AS class_id, '' AS company, '' AS company_name, 0 AS company_name_facet, '' AS company_two, '' AS content, 18000 AS created_at, 0 AS deleted, '' AS email, '' AS login, '' AS mission_statement, '' AS name, '' AS state, 0 AS user_id FROM states LEFT OUTER JOIN addresses AS addresses ON states.id = addresses.state_id WHERE states.id >= $start AND states.id <= $end GROUP BY states.id
 
 sql_group_column = capitalization
 sql_group_column = class_id
@@ -43,7 +43,7 @@ sql_date_column = created_at
 sql_group_column = deleted
 sql_str2ordinal_column = mission_statement
 sql_group_column = user_id
-sql_query_info = SELECT * FROM states WHERE states.id = (($id - 0) / 4)
+sql_query_info = SELECT * FROM states WHERE states.id = (($id - 0) / 5)
 }
 
 
@@ -64,8 +64,8 @@ sql_db = ultrasphinx_development
 sql_host = localhost
 sql_pass = 
 sql_user = root
-sql_query_range = SELECT MIN(id), MAX(id) FROM sellers
-sql_query = SELECT (sellers.id * 4 + 1) AS id, '' AS address_name, sellers.capitalization AS capitalization, 'Seller' AS class, 1 AS class_id, '' AS company, sellers.company_name AS company_name, CRC32(sellers.company_name) AS company_name_facet, '' AS company_two, '' AS content, UNIX_TIMESTAMP(sellers.created_at) AS created_at, 0 AS deleted, '' AS email, '' AS login, sellers.mission_statement AS mission_statement, '' AS name, '' AS state, sellers.user_id AS user_id FROM sellers WHERE sellers.id >= $start AND sellers.id <= $end GROUP BY sellers.id
+sql_query_range = SELECT MIN(id) ,  MAX(id) FROM sellers
+sql_query = SELECT (sellers.id * 5 + 1) AS id, '' AS address_name, sellers.capitalization AS capitalization, 'Seller' AS class, 1 AS class_id, '' AS company, sellers.company_name AS company_name, CRC32(sellers.company_name) AS company_name_facet, '' AS company_two, '' AS content, UNIX_TIMESTAMP(sellers.created_at) AS created_at, 0 AS deleted, '' AS email, '' AS login, sellers.mission_statement AS mission_statement, '' AS name, '' AS state, sellers.user_id AS user_id FROM sellers WHERE sellers.id >= $start AND sellers.id <= $end GROUP BY sellers.id
 
 sql_group_column = capitalization
 sql_group_column = class_id
@@ -74,7 +74,7 @@ sql_date_column = created_at
 sql_group_column = deleted
 sql_str2ordinal_column = mission_statement
 sql_group_column = user_id
-sql_query_info = SELECT * FROM sellers WHERE sellers.id = (($id - 1) / 4)
+sql_query_info = SELECT * FROM sellers WHERE sellers.id = (($id - 1) / 5)
 }
 
 
@@ -95,8 +95,8 @@ sql_db = ultrasphinx_development
 sql_host = localhost
 sql_pass = 
 sql_user = root
-sql_query_range = SELECT MIN(id), MAX(id) FROM addresses
-sql_query = SELECT (addresses.id * 4 + 2) AS id, '' AS address_name, 0 AS capitalization, 'Geo::Address' AS class, 2 AS class_id, '' AS company, '' AS company_name, 0 AS company_name_facet, '' AS company_two, CONCAT_WS(' ', addresses.line_1, addresses.line_2, addresses.city, addresses.province_region, addresses.zip_postal_code) AS content, 18000 AS created_at, 0 AS deleted, '' AS email, '' AS login, '' AS mission_statement, addresses.name AS name, states.name AS state, 0 AS user_id FROM addresses LEFT OUTER JOIN states AS states ON states.id = addresses.state_id WHERE addresses.id >= $start AND addresses.id <= $end GROUP BY addresses.id
+sql_query_range = SELECT MIN(id) ,  MAX(id) FROM addresses
+sql_query = SELECT (addresses.id * 5 + 2) AS id, '' AS address_name, 0 AS capitalization, 'Geo::Address' AS class, 2 AS class_id, '' AS company, '' AS company_name, 0 AS company_name_facet, '' AS company_two, CONCAT_WS(' ', addresses.line_1, addresses.line_2, addresses.city, addresses.province_region, addresses.zip_postal_code) AS content, 18000 AS created_at, 0 AS deleted, '' AS email, '' AS login, '' AS mission_statement, addresses.name AS name, states.name AS state, 0 AS user_id FROM addresses LEFT OUTER JOIN states AS states ON states.id = addresses.state_id WHERE addresses.id >= $start AND addresses.id <= $end GROUP BY addresses.id
 
 sql_group_column = capitalization
 sql_group_column = class_id
@@ -105,7 +105,7 @@ sql_date_column = created_at
 sql_group_column = deleted
 sql_str2ordinal_column = mission_statement
 sql_group_column = user_id
-sql_query_info = SELECT * FROM addresses WHERE addresses.id = (($id - 2) / 4)
+sql_query_info = SELECT * FROM addresses WHERE addresses.id = (($id - 2) / 5)
 }
 
 
@@ -126,8 +126,8 @@ sql_db = ultrasphinx_development
 sql_host = localhost
 sql_pass = 
 sql_user = root
-sql_query_range = SELECT MIN(id), MAX(id) FROM users
-sql_query = SELECT (users.id * 4 + 3) AS id, '' AS address_name, 0 AS capitalization, 'User' AS class, 3 AS class_id, sellers.company_name AS company, '' AS company_name, 0 AS company_name_facet, REPLACE(sellers_two.company_name, '6', ' replacement ') AS company_two, '' AS content, 18000 AS created_at, users.deleted AS deleted, users.email AS email, users.login AS login, '' AS mission_statement, '' AS name, '' AS state, 0 AS user_id FROM users LEFT OUTER JOIN sellers AS sellers ON users.id = sellers.user_id LEFT OUTER JOIN sellers AS sellers_two ON users.id = sellers_two.user_id WHERE users.id >= $start AND users.id <= $end AND (deleted = '0') GROUP BY users.id
+sql_query_range = SELECT MIN(id) ,  MAX(id) FROM users
+sql_query = SELECT (users.id * 5 + 3) AS id, '' AS address_name, 0 AS capitalization, 'User' AS class, 3 AS class_id, sellers.company_name AS company, '' AS company_name, 0 AS company_name_facet, REPLACE(sellers_two.company_name, '6', ' replacement ') AS company_two, '' AS content, 18000 AS created_at, users.deleted AS deleted, users.email AS email, users.login AS login, '' AS mission_statement, '' AS name, '' AS state, 0 AS user_id FROM users LEFT OUTER JOIN sellers AS sellers ON users.id = sellers.user_id LEFT OUTER JOIN sellers AS sellers_two ON users.id = sellers_two.user_id WHERE users.id >= $start AND users.id <= $end AND (deleted = '0') GROUP BY users.id
 
 sql_group_column = capitalization
 sql_group_column = class_id
@@ -136,7 +136,38 @@ sql_date_column = created_at
 sql_group_column = deleted
 sql_str2ordinal_column = mission_statement
 sql_group_column = user_id
-sql_query_info = SELECT * FROM users WHERE users.id = (($id - 3) / 4)
+sql_query_info = SELECT * FROM users WHERE users.id = (($id - 3) / 5)
+}
+
+
+# Source configuration
+
+source geo__countries
+{
+  strip_html = 0
+  sql_range_step = 20000
+  index_html_attrs = 
+  sql_query_post = 
+
+type = mysql
+sql_query_pre = SET SESSION group_concat_max_len = 65535
+sql_query_pre = SET NAMES utf8
+  
+sql_db = ultrasphinx_development
+sql_host = localhost
+sql_pass = 
+sql_user = root
+sql_query_range = SELECT MIN(id) ,  MAX(id) FROM countries
+sql_query = SELECT (countries.id * 5 + 4) AS id, '' AS address_name, 0 AS capitalization, 'Geo::Country' AS class, 4 AS class_id, '' AS company, '' AS company_name, 0 AS company_name_facet, '' AS company_two, '' AS content, 18000 AS created_at, 0 AS deleted, '' AS email, '' AS login, '' AS mission_statement, countries.name AS name, '' AS state, 0 AS user_id FROM countries WHERE countries.id >= $start AND countries.id <= $end GROUP BY countries.id
+
+sql_group_column = capitalization
+sql_group_column = class_id
+sql_group_column = company_name_facet
+sql_date_column = created_at
+sql_group_column = deleted
+sql_str2ordinal_column = mission_statement
+sql_group_column = user_id
+sql_query_info = SELECT * FROM countries WHERE countries.id = (($id - 4) / 5)
 }
 
 
@@ -145,6 +176,7 @@ sql_query_info = SELECT * FROM users WHERE users.id = (($id - 3) / 4)
 index complete
 {
   source = geo__addresses
+  source = geo__countries
   source = geo__states
   source = sellers
   source = users