dpickett-thinking-sphinx 1.1.12 → 1.1.23

data/README.textile

@@ -25,11 +25,18 @@ Then install the ginger gem. The steps are the same, except that you might need
   cd ginger
   rake gem
   sudo gem install pkg/ginger-1.1.0.gem
+  
+Alternatively, install the ginger gem directly from the freelancing-god github repository
+
+  sudo gem sources -a http://gems.github.com
+  sudo gem install freelancing-god-ginger
 
 Then set up your database:
 
   cp spec/fixtures/database.yml.default spec/fixtures/database.yml
   mysqladmin -u root create thinking_sphinx
+  
+This last step can be done automatically by the contribute.rb script if all dependencies are met.
 
 Make sure you don't have another Sphinx daemon (searchd) running. If you do, quit it with "rake ts:stop"
 in the app root.
@@ -124,3 +131,15 @@ Since I first released this library, there's been quite a few people who have su
 * Mark Lane
 * Eric Lindvall
 * Lawrence Pit
+* Mike Bailey
+* Bill Leeper
+* Michael Reinsch
+* Anderson Dias
+* Jerome Riga
+* Tien Dung
+* Johannes Kaefer
+* Paul Campbell
+* Matthew Beale
+* Tom Simnett
+* Erik Ostrom
+* Ole Riesenberg

data/lib/thinking_sphinx.rb

@@ -18,6 +18,7 @@ require 'thinking_sphinx/class_facet'
 require 'thinking_sphinx/facet_collection'
 require 'thinking_sphinx/field'
 require 'thinking_sphinx/index'
+require 'thinking_sphinx/source'
 require 'thinking_sphinx/rails_additions'
 require 'thinking_sphinx/search'
 require 'thinking_sphinx/deltas'
@@ -36,7 +37,7 @@ module ThinkingSphinx
   module Version #:nodoc:
     Major = 1
     Minor = 1
-    Tiny  = 12
+    Tiny  = 23
     
     String = [Major, Minor, Tiny].join('.')
   end
@@ -137,7 +138,39 @@ module ThinkingSphinx
     )
   end
   
+  @@remote_sphinx = false
+  
+  # An indication of whether Sphinx is running on a remote machine instead of
+  # the same machine.
+  # 
+  def self.remote_sphinx?
+    @@remote_sphinx
+  end
+  
+  # Tells Thinking Sphinx that Sphinx is running on a different machine, and
+  # thus it can't reliably guess whether it is running or not (ie: the 
+  # #sphinx_running? method), and so just assumes it is.
+  # 
+  # Useful for multi-machine deployments. Set it in your production.rb file.
+  # 
+  #   ThinkingSphinx.remote_sphinx = true
+  # 
+  def self.remote_sphinx=(value)
+    @@remote_sphinx = value
+  end
+  
+  # Check if Sphinx is running. If remote_sphinx is set to true (indicating
+  # Sphinx is on a different machine), this will always return true, and you
+  # will have to handle any connection errors yourself.
+  # 
   def self.sphinx_running?
+    remote_sphinx? || sphinx_running_by_pid?
+  end
+  
+  # Check if Sphinx is actually running, provided the pid is on the same
+  # machine as this code.
+  # 
+  def self.sphinx_running_by_pid?
     !!sphinx_pid && pid_active?(sphinx_pid)
   end
   
@@ -174,7 +207,8 @@ module ThinkingSphinx
   end
   
   def self.mysql?
-    ::ActiveRecord::Base.connection.class.name.demodulize == "MysqlAdapter" || (
+    ::ActiveRecord::Base.connection.class.name.demodulize == "MysqlAdapter" ||
+    ::ActiveRecord::Base.connection.class.name.demodulize == "MysqlplusAdapter" || (
       jruby? && ::ActiveRecord::Base.connection.config[:adapter] == "jdbcmysql"
     )
   end

data/lib/thinking_sphinx/active_record.rb

@@ -66,7 +66,7 @@ module ThinkingSphinx
             return unless ThinkingSphinx.define_indexes?
             
             self.sphinx_indexes ||= []
-            index = Index.new(self, &block)
+            index = ThinkingSphinx::Index::Builder.generate(self, &block)
             
             self.sphinx_indexes << index
             unless ThinkingSphinx.indexed_models.include?(self.name)
@@ -83,6 +83,17 @@ module ThinkingSphinx
             include ThinkingSphinx::ActiveRecord::AttributeUpdates
             
             index
+            
+            # We want to make sure that if the database doesn't exist, then Thinking
+            # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop
+            # and db:migrate). It's a bit hacky, but I can't think of a better way.
+          rescue StandardError => err
+            case err.class.name
+            when "Mysql::Error", "Java::JavaSql::SQLException", "ActiveRecord::StatementInvalid"
+              return
+            else
+              raise err
+            end
           end
           alias_method :sphinx_index, :define_index
           
@@ -151,7 +162,9 @@ module ThinkingSphinx
             self.sphinx_indexes.select { |ts_index|
               ts_index.model == self
             }.each_with_index do |ts_index, i|
-              index.sources << ts_index.to_riddle_for_core(offset, i)
+              index.sources += ts_index.sources.collect { |source|
+                source.to_riddle_for_core(offset, i)
+              }
             end
             
             index
@@ -163,7 +176,9 @@ module ThinkingSphinx
             index.path = File.join(ThinkingSphinx::Configuration.instance.searchd_file_path, index.name)
             
             self.sphinx_indexes.each_with_index do |ts_index, i|
-              index.sources << ts_index.to_riddle_for_delta(offset, i) if ts_index.delta?
+              index.sources += ts_index.sources.collect { |source|
+                source.to_riddle_for_delta(offset, i)
+              } if ts_index.delta?
             end
             
             index

data/lib/thinking_sphinx/adapters/postgresql_adapter.rb

@@ -11,7 +11,12 @@ module ThinkingSphinx
     
     def concatenate(clause, separator = ' ')
       clause.split(', ').collect { |field|
-        "COALESCE(CAST(#{field} as varchar), '')"
+        case field
+        when /COALESCE/, "'')"
+          field
+        else
+          "COALESCE(CAST(#{field} as varchar), '')"
+        end
       }.join(" || '#{separator}' || ")
     end
     
@@ -32,7 +37,8 @@ module ThinkingSphinx
     end
     
     def convert_nulls(clause, default = '')
-      default = "'#{default}'" if default.is_a?(String)
+      default = "'#{default}'"  if default.is_a?(String)
+      default = 'NULL'          if default.nil?
       
       "COALESCE(#{clause}, #{default})"
     end
@@ -127,4 +133,4 @@ module ThinkingSphinx
       execute function, true
     end
   end
-end
+end

data/lib/thinking_sphinx/association.rb

@@ -103,13 +103,16 @@ module ThinkingSphinx
       if @reflection.options[:through]
         @reflection.source_reflection.options[:foreign_key] ||
         @reflection.source_reflection.primary_key_name
+      elsif @reflection.macro == :has_and_belongs_to_many
+        @reflection.association_foreign_key
       else
         nil
       end
     end
     
     def table
-      if @reflection.options[:through]
+      if @reflection.options[:through] ||
+        @reflection.macro == :has_and_belongs_to_many
         @join.aliased_join_table_name
       else
         @join.aliased_table_name

data/lib/thinking_sphinx/attribute.rb

@@ -9,7 +9,7 @@ module ThinkingSphinx
   # associations. Which can get messy. Use Index.link!, it really helps.
   # 
   class Attribute < ThinkingSphinx::Property
-    attr_accessor :source
+    attr_accessor :query_source
     
     # To create a new attribute, you'll need to pass in either a single Column
     # or an array of them, and some (optional) options.
@@ -67,15 +67,17 @@ module ThinkingSphinx
     # If you're creating attributes for latitude and longitude, don't forget
     # that Sphinx expects these values to be in radians.
     #  
-    def initialize(columns, options = {})
+    def initialize(source, columns, options = {})
       super
       
-      @type     = options[:type]
-      @source   = options[:source]
-      @crc      = options[:crc]
+      @type           = options[:type]
+      @query_source   = options[:source]
+      @crc            = options[:crc]
       
-      @type   ||= :multi    unless @source.nil?
-      @type     = :integer  if @type == :string && @crc
+      @type         ||= :multi    unless @query_source.nil?
+      @type           = :integer  if @type == :string && @crc
+      
+      source.attributes << self
     end
     
     # Get the part of the SELECT clause related to this attribute. Don't forget
@@ -87,17 +89,17 @@ module ThinkingSphinx
     def to_select_sql
       return nil unless include_as_association?
       
+      separator = all_ints? || @crc ? ',' : ' '
+      
       clause = @columns.collect { |column|
-        column_with_prefix(column)
+        part = column_with_prefix(column)
+        type == :string ? adapter.convert_nulls(part) : part
       }.join(', ')
       
-      separator = all_ints? ? ',' : ' '
-      
-      clause = adapter.concatenate(clause, separator)       if concat_ws?
-      clause = adapter.group_concatenate(clause, separator) if is_many?
       clause = adapter.cast_to_datetime(clause)             if type == :datetime
-      clause = adapter.convert_nulls(clause)                if type == :string
       clause = adapter.crc(clause)                          if @crc
+      clause = adapter.concatenate(clause, separator)       if concat_ws?
+      clause = adapter.group_concatenate(clause, separator) if is_many?
       
       "#{clause} AS #{quote_column(unique_name)}"
     end
@@ -114,7 +116,7 @@ module ThinkingSphinx
     end
     
     def include_as_association?
-      ! (type == :multi && (source == :query || source == :ranged_query))
+      ! (type == :multi && (query_source == :query || query_source == :ranged_query))
     end
     
     # Returns the configuration value that should be used for
@@ -122,10 +124,10 @@ module ThinkingSphinx
     # Special case is the multi-valued attribute that needs some
     # extra configuration. 
     # 
-    def config_value(offset = nil)
+    def config_value(offset = nil, delta = false)
       if type == :multi
         multi_config = include_as_association? ? "field" :
-          source_value(offset).gsub(/\n\s*/, " ")
+          source_value(offset, delta).gsub(/\s+/m, " ").strip
         "uint #{unique_name} from #{multi_config}"
       else
         unique_name
@@ -168,37 +170,65 @@ module ThinkingSphinx
       object.send(column.__name)
     end
     
+    def all_ints?
+      @columns.all? { |col|
+        klasses = @associations[col].empty? ? [@model] :
+          @associations[col].collect { |assoc| assoc.reflection.klass }
+        klasses.all? { |klass|
+          column = klass.columns.detect { |column| column.name == col.__name.to_s }
+          !column.nil? && column.type == :integer
+        }
+      }
+    end
+    
     private
     
-    def source_value(offset)
+    def source_value(offset, delta)
       if is_string?
-        "#{source.to_s.dasherize}; #{columns.first.__name}"
-      elsif source == :ranged_query
-        "ranged-query; #{query offset} #{query_clause}; #{range_query}"
+        return "#{query_source.to_s.dasherize}; #{columns.first.__name}"
+      end
+      
+      query = query(offset)
+
+      if query_source == :ranged_query
+        query += query_clause
+        query += " AND #{query_delta.strip}" if delta
+        "ranged-query; #{query}; #{range_query}"
       else
-        "query; #{query offset}"
+        query += "WHERE #{query_delta.strip}" if delta
+        "query; #{query}"
       end
     end
     
     def query(offset)
-      assoc = association_for_mva
-      raise "Could not determine SQL for MVA" if assoc.nil?
+      base_assoc = base_association_for_mva
+      end_assoc  = end_association_for_mva
+      raise "Could not determine SQL for MVA" if base_assoc.nil?
       
       <<-SQL
-SELECT #{foreign_key_for_mva assoc}
+SELECT #{foreign_key_for_mva base_assoc}
   #{ThinkingSphinx.unique_id_expression(offset)} AS #{quote_column('id')},
-  #{primary_key_for_mva(assoc)} AS #{quote_column(unique_name)}
-FROM #{quote_table_name assoc.table}
+  #{primary_key_for_mva(end_assoc)} AS #{quote_column(unique_name)}
+FROM #{quote_table_name base_assoc.table} #{association_joins}
       SQL
     end
     
     def query_clause
-      foreign_key = foreign_key_for_mva association_for_mva
+      foreign_key = foreign_key_for_mva base_association_for_mva
       "WHERE #{foreign_key} >= $start AND #{foreign_key} <= $end"
     end
     
+    def query_delta
+      foreign_key = foreign_key_for_mva base_association_for_mva
+      <<-SQL
+#{foreign_key} IN (SELECT #{quote_column model.primary_key}
+FROM #{model.quoted_table_name}
+WHERE #{@source.index.delta_object.clause(model, true)})
+      SQL
+    end
+    
     def range_query
-      assoc       = association_for_mva
+      assoc       = base_association_for_mva
       foreign_key = foreign_key_for_mva assoc
       "SELECT MIN(#{foreign_key}), MAX(#{foreign_key}) FROM #{quote_table_name assoc.table}"
     end
@@ -213,27 +243,38 @@ FROM #{quote_table_name assoc.table}
       quote_with_table assoc.table, assoc.reflection.primary_key_name
     end
     
-    def association_for_mva
+    def end_association_for_mva
       @association_for_mva ||= associations[columns.first].detect { |assoc|
         assoc.has_column?(columns.first.__name)
       }
     end
     
+    def base_association_for_mva
+      @first_association_for_mva ||= begin
+        assoc = end_association_for_mva
+        while !assoc.parent.nil?
+          assoc = assoc.parent
+        end
+        
+        assoc
+      end
+    end
+    
+    def association_joins
+      joins = []
+      assoc = end_association_for_mva
+      while assoc != base_association_for_mva
+        joins << assoc.to_sql
+        assoc = assoc.parent
+      end
+      
+      joins.join(' ')
+    end
+    
     def is_many_ints?
       concat_ws? && all_ints?
     end
         
-    def all_ints?
-      @columns.all? { |col|
-        klasses = @associations[col].empty? ? [@model] :
-          @associations[col].collect { |assoc| assoc.reflection.klass }
-        klasses.all? { |klass|
-          column = klass.columns.detect { |column| column.name == col.__name.to_s }
-          !column.nil? && column.type == :integer
-        }
-      }
-    end
-    
     def type_from_database
       klass = @associations.values.flatten.first ? 
         @associations.values.flatten.first.reflection.klass : @model
@@ -254,9 +295,10 @@ FROM #{quote_table_name assoc.table}
       else
         raise <<-MESSAGE
 
-Cannot automatically map column type #{type_from_db} to an equivalent Sphinx
-type (integer, float, boolean, datetime, string as ordinal). You could try to
-explicitly convert the column's value in your define_index block:
+Cannot automatically map attribute #{unique_name} in #{@model.name} to an
+equivalent Sphinx type (integer, float, boolean, datetime, string as ordinal).
+You could try to explicitly convert the column's value in your define_index
+block:
   has "CAST(column AS INT)", :type => :integer, :as => :column
         MESSAGE
       end

data/lib/thinking_sphinx/configuration.rb

@@ -19,12 +19,14 @@ module ThinkingSphinx
   # min infix length::      1
   # mem limit::             64M
   # max matches::           1000
-  # morphology::            stem_en
+  # morphology::            nil
   # charset type::          utf-8
   # charset table::         nil
   # ignore chars::          nil
   # html strip::            false
   # html remove elements::  ''
+  # searchd_binary_name::   searchd
+  # indexer_binary_name::   indexer
   #
   # If you want to change these settings, create a YAML file at
   # config/sphinx.yml with settings for each environment, in a similar
@@ -32,7 +34,8 @@ module ThinkingSphinx
   # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
   # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
   # max_matches, morphology, charset_type, charset_table, ignore_chars,
-  # html_strip, html_remove_elements, delayed_job_priority.
+  # html_strip, html_remove_elements, delayed_job_priority,
+  # searchd_binary_name, indexer_binary_name.
   #
   # I think you've got the idea.
   # 
@@ -54,11 +57,13 @@ module ThinkingSphinx
       min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars
       ngram_len phrase_boundary phrase_boundary_step preopen stopwords
       wordforms )
+    
+    CustomOptions = %w( disable_range )
         
     attr_accessor :config_file, :searchd_log_file, :query_log_file,
       :pid_file, :searchd_file_path, :address, :port, :allow_star,
       :database_yml_file, :app_root, :bin_path, :model_directories,
-      :delayed_job_priority
+      :delayed_job_priority, :searchd_binary_name, :indexer_binary_name
     
     attr_accessor :source_options, :index_options
     
@@ -71,10 +76,19 @@ module ThinkingSphinx
       self.reset
     end
     
-    def reset
-      self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
-      self.app_root          = Merb.root  if defined?(Merb)
-      self.app_root        ||= app_root
+    def self.configure(&block)
+      yield instance
+      instance.reset(instance.app_root)
+    end
+    
+    def reset(custom_app_root=nil)
+      if custom_app_root
+        self.app_root = custom_app_root
+      else
+        self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
+        self.app_root          = Merb.root  if defined?(Merb)
+        self.app_root        ||= app_root
+      end
       
       @configuration = Riddle::Configuration.new
       @configuration.searchd.address    = "127.0.0.1"
@@ -94,9 +108,11 @@ module ThinkingSphinx
       
       self.source_options  = {}
       self.index_options   = {
-        :charset_type => "utf-8",
-        :morphology   => "stem_en"
+        :charset_type => "utf-8"
       }
+      
+      self.searchd_binary_name = "searchd"
+      self.indexer_binary_name = "indexer"
             
       parse_config
       
@@ -141,6 +157,8 @@ module ThinkingSphinx
     # messy dependencies issues).
     # 
     def load_models
+      return if defined?(Rails) && Rails.configuration.cache_classes
+      
       self.model_directories.each do |base|
         Dir["#{base}**/*.rb"].each do |file|
           model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
@@ -156,6 +174,8 @@ module ThinkingSphinx
             model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
           rescue NameError
             next
+          rescue StandardError
+            puts "Warning: Error loading #{file}"
           end
         end
       end
@@ -213,10 +233,11 @@ module ThinkingSphinx
       conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
       
       conf.each do |key,value|
-        self.send("#{key}=", value) if self.methods.include?("#{key}=")
+        self.send("#{key}=", value) if self.respond_to?("#{key}=")
         
         set_sphinx_setting self.source_options, key, value, SourceOptions
         set_sphinx_setting self.index_options,  key, value, IndexOptions
+        set_sphinx_setting self.index_options,  key, value, CustomOptions
         set_sphinx_setting @configuration.searchd, key, value
         set_sphinx_setting @configuration.indexer, key, value
       end unless conf.nil?
@@ -233,8 +254,8 @@ module ThinkingSphinx
       if object.is_a?(Hash)
         object[key.to_sym] = value if allowed.include?(key.to_s)
       else
-        object.send("#{key}=", value) if object.methods.include?("#{key}")
-        send("#{key}=", value) if self.methods.include?("#{key}")
+        object.send("#{key}=", value) if object.respond_to?("#{key}")
+        send("#{key}=", value) if self.respond_to?("#{key}")
       end
     end
   end