DrMark-thinking-sphinx 1.1.15 → 1.2.5

data/README.textile

@@ -25,11 +25,18 @@ Then install the ginger gem. The steps are the same, except that you might need
   cd ginger
   rake gem
   sudo gem install pkg/ginger-1.1.0.gem
+  
+Alternatively, install the ginger gem directly from the freelancing-god github repository
+
+  sudo gem sources -a http://gems.github.com
+  sudo gem install freelancing-god-ginger
 
 Then set up your database:
 
   cp spec/fixtures/database.yml.default spec/fixtures/database.yml
   mysqladmin -u root create thinking_sphinx
+  
+This last step can be done automatically by the contribute.rb script if all dependencies are met.
 
 Make sure you don't have another Sphinx daemon (searchd) running. If you do, quit it with "rake ts:stop"
 in the app root.
@@ -125,3 +132,18 @@ Since I first released this library, there's been quite a few people who have su
 * Eric Lindvall
 * Lawrence Pit
 * Mike Bailey
+* Bill Leeper
+* Michael Reinsch
+* Anderson Dias
+* Jerome Riga
+* Tien Dung
+* Johannes Kaefer
+* Paul Campbell
+* Matthew Beale
+* Tom Simnett
+* Erik Ostrom
+* Ole Riesenberg
+* Josh Kalderimis
+* J.D. Hollis
+* Jeffrey Chupp
+* Rob Anderton

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:major: 1
+:minor: 2
+:patch: 5

data/lib/thinking_sphinx/active_record/scopes.rb

@@ -0,0 +1,39 @@
+module ThinkingSphinx
+  module ActiveRecord
+    module Scopes
+      def self.included(base)
+        base.class_eval do
+          extend ThinkingSphinx::ActiveRecord::Scopes::ClassMethods
+        end
+      end
+    
+      module ClassMethods
+        def sphinx_scope(method, &block)
+          @sphinx_scopes ||= []
+          @sphinx_scopes << method
+          
+          metaclass.instance_eval do
+            define_method(method) do |*args|
+              options = {:classes => classes_option}
+              options.merge! block.call(*args)
+              
+              ThinkingSphinx::Search.new(options)
+            end
+          end
+        end
+        
+        def sphinx_scopes
+          @sphinx_scopes || []
+        end
+        
+        def remove_sphinx_scopes
+          sphinx_scopes.each do |scope|
+            metaclass.send(:undef_method, scope)
+          end
+          
+          sphinx_scopes.clear
+        end
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/active_record.rb

@@ -1,7 +1,7 @@
 require 'thinking_sphinx/active_record/attribute_updates'
 require 'thinking_sphinx/active_record/delta'
-require 'thinking_sphinx/active_record/search'
 require 'thinking_sphinx/active_record/has_many_association'
+require 'thinking_sphinx/active_record/scopes'
 
 module ThinkingSphinx
   # Core additions to ActiveRecord models - define_index for creating indexes
@@ -13,6 +13,15 @@ module ThinkingSphinx
       base.class_eval do
         class_inheritable_array :sphinx_indexes, :sphinx_facets
         class << self
+          
+          def set_sphinx_primary_key(attribute)
+            @sphinx_primary_key_attribute = attribute
+          end
+          
+          def primary_key_for_sphinx
+            @sphinx_primary_key_attribute || primary_key
+          end
+          
           # Allows creation of indexes for Sphinx. If you don't do this, there
           # isn't much point trying to search (or using this plugin at all,
           # really).
@@ -80,7 +89,9 @@ module ThinkingSphinx
             
             after_destroy :toggle_deleted
             
+            include ThinkingSphinx::SearchMethods
             include ThinkingSphinx::ActiveRecord::AttributeUpdates
+            include ThinkingSphinx::ActiveRecord::Scopes
             
             index
             
@@ -140,12 +151,12 @@ module ThinkingSphinx
               ThinkingSphinx::AbstractAdapter.detect(self)
           end
           
-          private
-          
           def sphinx_name
             self.name.underscore.tr(':/\\', '_')
           end
           
+          private
+          
           def sphinx_delta?
             self.sphinx_indexes.any? { |index| index.delta? }
           end
@@ -215,7 +226,6 @@ module ThinkingSphinx
       end
       
       base.send(:include, ThinkingSphinx::ActiveRecord::Delta)
-      base.send(:include, ThinkingSphinx::ActiveRecord::Search)
       
       ::ActiveRecord::Associations::HasManyAssociation.send(
         :include, ThinkingSphinx::ActiveRecord::HasManyAssociation
@@ -264,13 +274,23 @@ module ThinkingSphinx
       # nothing
     end
     
+    # Returns the unique integer id for the object. This method uses the
+    # attribute hash to get around ActiveRecord always mapping the #id method
+    # to whatever the real primary key is (which may be a unique string hash).
+    # 
+    # @return [Integer] Unique record id for the purposes of Sphinx.
+    # 
+    def primary_key_for_sphinx
+      attributes[self.class.primary_key_for_sphinx.to_s]
+    end
+    
     def sphinx_document_id
-      (self.id * ThinkingSphinx.indexed_models.size) +
+      primary_key_for_sphinx * ThinkingSphinx.indexed_models.size +
         ThinkingSphinx.indexed_models.index(self.class.source_of_sphinx_index.name)
     end
-    
+
     private
-    
+
     def sphinx_index_name(suffix)
       "#{self.class.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_#{suffix}"
     end

data/lib/thinking_sphinx/adapters/postgresql_adapter.rb

@@ -11,7 +11,12 @@ module ThinkingSphinx
     
     def concatenate(clause, separator = ' ')
       clause.split(', ').collect { |field|
-        field[/COALESCE/] ? field : "COALESCE(CAST(#{field} as varchar), '')"
+        case field
+        when /COALESCE/, "'')"
+          field
+        else
+          "COALESCE(CAST(#{field} as varchar), '')"
+        end
       }.join(" || '#{separator}' || ")
     end
     
@@ -32,7 +37,8 @@ module ThinkingSphinx
     end
     
     def convert_nulls(clause, default = '')
-      default = "'#{default}'" if default.is_a?(String)
+      default = "'#{default}'"  if default.is_a?(String)
+      default = 'NULL'          if default.nil?
       
       "COALESCE(#{clause}, #{default})"
     end
@@ -127,4 +133,4 @@ module ThinkingSphinx
       execute function, true
     end
   end
-end
+end

data/lib/thinking_sphinx/association.rb

@@ -103,13 +103,16 @@ module ThinkingSphinx
       if @reflection.options[:through]
         @reflection.source_reflection.options[:foreign_key] ||
         @reflection.source_reflection.primary_key_name
+      elsif @reflection.macro == :has_and_belongs_to_many
+        @reflection.association_foreign_key
       else
         nil
       end
     end
     
     def table
-      if @reflection.options[:through]
+      if @reflection.options[:through] ||
+        @reflection.macro == :has_and_belongs_to_many
         @join.aliased_join_table_name
       else
         @join.aliased_table_name

data/lib/thinking_sphinx/attribute.rb

@@ -89,14 +89,21 @@ module ThinkingSphinx
     def to_select_sql
       return nil unless include_as_association?
       
+      separator = all_ints? || all_datetimes? || @crc ? ',' : ' '
+      
       clause = @columns.collect { |column|
         part = column_with_prefix(column)
-        type == :string ? adapter.convert_nulls(part) : part
+        case type
+        when :string
+          adapter.convert_nulls(part)
+        when :datetime
+          adapter.cast_to_datetime(part)
+        else
+          part
+        end
       }.join(', ')
       
-      separator = all_ints? || @crc ? ',' : ' '
-      
-      clause = adapter.cast_to_datetime(clause)             if type == :datetime
+      # clause = adapter.cast_to_datetime(clause)             if type == :datetime
       clause = adapter.crc(clause)                          if @crc
       clause = adapter.concatenate(clause, separator)       if concat_ws?
       clause = adapter.group_concatenate(clause, separator) if is_many?
@@ -124,10 +131,10 @@ module ThinkingSphinx
     # Special case is the multi-valued attribute that needs some
     # extra configuration. 
     # 
-    def config_value(offset = nil)
+    def config_value(offset = nil, delta = false)
       if type == :multi
         multi_config = include_as_association? ? "field" :
-          source_value(offset).gsub(/\n\s*/, " ")
+          source_value(offset, delta).gsub(/\s+/m, " ").strip
         "uint #{unique_name} from #{multi_config}"
       else
         unique_name
@@ -142,6 +149,8 @@ module ThinkingSphinx
     def type
       @type ||= begin
         base_type = case
+        when is_many_datetimes?
+          :datetime
         when is_many?, is_many_ints?
           :multi
         when @associations.values.flatten.length > 1
@@ -171,47 +180,61 @@ module ThinkingSphinx
     end
     
     def all_ints?
-      @columns.all? { |col|
-        klasses = @associations[col].empty? ? [@model] :
-          @associations[col].collect { |assoc| assoc.reflection.klass }
-        klasses.all? { |klass|
-          column = klass.columns.detect { |column| column.name == col.__name.to_s }
-          !column.nil? && column.type == :integer
-        }
-      }
+      all_of_type?(:integer)
+    end
+    
+    def all_datetimes?
+      all_of_type?(:datetime, :date, :timestamp)
     end
     
     private
     
-    def source_value(offset)
+    def source_value(offset, delta)
       if is_string?
-        "#{query_source.to_s.dasherize}; #{columns.first.__name}"
-      elsif query_source == :ranged_query
-        "ranged-query; #{query offset} #{query_clause}; #{range_query}"
+        return "#{query_source.to_s.dasherize}; #{columns.first.__name}"
+      end
+      
+      query = query(offset)
+
+      if query_source == :ranged_query
+        query += query_clause
+        query += " AND #{query_delta.strip}" if delta
+        "ranged-query; #{query}; #{range_query}"
       else
-        "query; #{query offset}"
+        query += "WHERE #{query_delta.strip}" if delta
+        "query; #{query}"
       end
     end
     
     def query(offset)
-      assoc = association_for_mva
-      raise "Could not determine SQL for MVA" if assoc.nil?
+      base_assoc = base_association_for_mva
+      end_assoc  = end_association_for_mva
+      raise "Could not determine SQL for MVA" if base_assoc.nil?
       
       <<-SQL
-SELECT #{foreign_key_for_mva assoc}
+SELECT #{foreign_key_for_mva base_assoc}
   #{ThinkingSphinx.unique_id_expression(offset)} AS #{quote_column('id')},
-  #{primary_key_for_mva(assoc)} AS #{quote_column(unique_name)}
-FROM #{quote_table_name assoc.table}
+  #{primary_key_for_mva(end_assoc)} AS #{quote_column(unique_name)}
+FROM #{quote_table_name base_assoc.table} #{association_joins}
       SQL
     end
     
     def query_clause
-      foreign_key = foreign_key_for_mva association_for_mva
+      foreign_key = foreign_key_for_mva base_association_for_mva
       "WHERE #{foreign_key} >= $start AND #{foreign_key} <= $end"
     end
     
+    def query_delta
+      foreign_key = foreign_key_for_mva base_association_for_mva
+      <<-SQL
+#{foreign_key} IN (SELECT #{quote_column model.primary_key}
+FROM #{model.quoted_table_name}
+WHERE #{@source.index.delta_object.clause(model, true)})
+      SQL
+    end
+    
     def range_query
-      assoc       = association_for_mva
+      assoc       = base_association_for_mva
       foreign_key = foreign_key_for_mva assoc
       "SELECT MIN(#{foreign_key}), MAX(#{foreign_key}) FROM #{quote_table_name assoc.table}"
     end
@@ -226,23 +249,50 @@ FROM #{quote_table_name assoc.table}
       quote_with_table assoc.table, assoc.reflection.primary_key_name
     end
     
-    def association_for_mva
+    def end_association_for_mva
       @association_for_mva ||= associations[columns.first].detect { |assoc|
         assoc.has_column?(columns.first.__name)
       }
     end
     
+    def base_association_for_mva
+      @first_association_for_mva ||= begin
+        assoc = end_association_for_mva
+        while !assoc.parent.nil?
+          assoc = assoc.parent
+        end
+        
+        assoc
+      end
+    end
+    
+    def association_joins
+      joins = []
+      assoc = end_association_for_mva
+      while assoc != base_association_for_mva
+        joins << assoc.to_sql
+        assoc = assoc.parent
+      end
+      
+      joins.join(' ')
+    end
+    
     def is_many_ints?
       concat_ws? && all_ints?
     end
-        
+    
+    def is_many_datetimes?
+      is_many? && all_datetimes?
+    end
+       
     def type_from_database
       klass = @associations.values.flatten.first ? 
         @associations.values.flatten.first.reflection.klass : @model
       
-      klass.columns.detect { |col|
+      column = klass.columns.detect { |col|
         @columns.collect { |c| c.__name.to_s }.include? col.name
-      }.type
+      }
+      column.nil? ? nil : column.type
     end
     
     def translated_type_from_database
@@ -264,5 +314,16 @@ block:
         MESSAGE
       end
     end
+    
+    def all_of_type?(*column_types)
+      @columns.all? { |col|
+        klasses = @associations[col].empty? ? [@model] :
+          @associations[col].collect { |assoc| assoc.reflection.klass }
+        klasses.all? { |klass|
+          column = klass.columns.detect { |column| column.name == col.__name.to_s }
+          !column.nil? && column_types.include?(column.type)
+        }
+      }
+    end
   end
 end

data/lib/thinking_sphinx/configuration.rb

@@ -19,12 +19,14 @@ module ThinkingSphinx
   # min infix length::      1
   # mem limit::             64M
   # max matches::           1000
-  # morphology::            stem_en
+  # morphology::            nil
   # charset type::          utf-8
   # charset table::         nil
   # ignore chars::          nil
   # html strip::            false
   # html remove elements::  ''
+  # searchd_binary_name::   searchd
+  # indexer_binary_name::   indexer
   #
   # If you want to change these settings, create a YAML file at
   # config/sphinx.yml with settings for each environment, in a similar
@@ -32,7 +34,8 @@ module ThinkingSphinx
   # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
   # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
   # max_matches, morphology, charset_type, charset_table, ignore_chars,
-  # html_strip, html_remove_elements, delayed_job_priority.
+  # html_strip, html_remove_elements, delayed_job_priority,
+  # searchd_binary_name, indexer_binary_name.
   #
   # I think you've got the idea.
   # 
@@ -54,11 +57,13 @@ module ThinkingSphinx
       min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars
       ngram_len phrase_boundary phrase_boundary_step preopen stopwords
       wordforms )
+    
+    CustomOptions = %w( disable_range )
         
     attr_accessor :config_file, :searchd_log_file, :query_log_file,
       :pid_file, :searchd_file_path, :address, :port, :allow_star,
       :database_yml_file, :app_root, :bin_path, :model_directories,
-      :delayed_job_priority
+      :delayed_job_priority, :searchd_binary_name, :indexer_binary_name
     
     attr_accessor :source_options, :index_options
     
@@ -71,10 +76,19 @@ module ThinkingSphinx
       self.reset
     end
     
-    def reset
-      self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
-      self.app_root          = Merb.root  if defined?(Merb)
-      self.app_root        ||= app_root
+    def self.configure(&block)
+      yield instance
+      instance.reset(instance.app_root)
+    end
+    
+    def reset(custom_app_root=nil)
+      if custom_app_root
+        self.app_root = custom_app_root
+      else
+        self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
+        self.app_root          = Merb.root  if defined?(Merb)
+        self.app_root        ||= app_root
+      end
       
       @configuration = Riddle::Configuration.new
       @configuration.searchd.address    = "127.0.0.1"
@@ -94,9 +108,11 @@ module ThinkingSphinx
       
       self.source_options  = {}
       self.index_options   = {
-        :charset_type => "utf-8",
-        :morphology   => "stem_en"
+        :charset_type => "utf-8"
       }
+      
+      self.searchd_binary_name = "searchd"
+      self.indexer_binary_name = "indexer"
             
       parse_config
       
@@ -141,6 +157,8 @@ module ThinkingSphinx
     # messy dependencies issues).
     # 
     def load_models
+      return if defined?(Rails) && Rails.configuration.cache_classes
+      
       self.model_directories.each do |base|
         Dir["#{base}**/*.rb"].each do |file|
           model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
@@ -156,6 +174,8 @@ module ThinkingSphinx
             model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
           rescue NameError
             next
+          rescue StandardError
+            puts "Warning: Error loading #{file}"
           end
         end
       end
@@ -201,6 +221,24 @@ module ThinkingSphinx
       @configuration.searchd.query_log = file
     end
     
+    def client
+      client = Riddle::Client.new address, port
+      client.max_matches = configuration.searchd.max_matches || 1000
+      client
+    end
+    
+    def models_by_crc
+      @models_by_crc ||= begin
+        ThinkingSphinx.indexed_models.inject({}) do |hash, model|
+          hash[model.constantize.to_crc32] = model
+          Object.subclasses_of(model.constantize).each { |subclass|
+            hash[subclass.to_crc32] = subclass.name
+          }
+          hash
+        end
+      end
+    end
+    
     private
     
     # Parse the config/sphinx.yml file - if it exists - then use the attribute
@@ -213,10 +251,11 @@ module ThinkingSphinx
       conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
       
       conf.each do |key,value|
-        self.send("#{key}=", value) if self.methods.include?("#{key}=")
+        self.send("#{key}=", value) if self.respond_to?("#{key}=")
         
         set_sphinx_setting self.source_options, key, value, SourceOptions
         set_sphinx_setting self.index_options,  key, value, IndexOptions
+        set_sphinx_setting self.index_options,  key, value, CustomOptions
         set_sphinx_setting @configuration.searchd, key, value
         set_sphinx_setting @configuration.indexer, key, value
       end unless conf.nil?
@@ -233,8 +272,8 @@ module ThinkingSphinx
       if object.is_a?(Hash)
         object[key.to_sym] = value if allowed.include?(key.to_s)
       else
-        object.send("#{key}=", value) if object.methods.include?("#{key}")
-        send("#{key}=", value) if self.methods.include?("#{key}")
+        object.send("#{key}=", value) if object.respond_to?("#{key}")
+        send("#{key}=", value) if self.respond_to?("#{key}")
       end
     end
   end

data/lib/thinking_sphinx/deltas/datetime_delta.rb

@@ -18,8 +18,8 @@ module ThinkingSphinx
         config = ThinkingSphinx::Configuration.instance
         rotate = ThinkingSphinx.sphinx_running? ? "--rotate" : ""
         
-        output = `#{config.bin_path}indexer --config #{config.config_file} #{rotate} #{delta_index_name model}`
-        output += `#{config.bin_path}indexer --config #{config.config_file} #{rotate} --merge #{core_index_name model} #{delta_index_name model} --merge-dst-range sphinx_deleted 0 0`
+        output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} #{rotate} #{delta_index_name model}`
+        output += `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} #{rotate} --merge #{core_index_name model} #{delta_index_name model} --merge-dst-range sphinx_deleted 0 0`
         puts output unless ThinkingSphinx.suppress_delta_output?
         
         true

data/lib/thinking_sphinx/deltas/default_delta.rb

@@ -17,7 +17,7 @@ module ThinkingSphinx
         client = Riddle::Client.new config.address, config.port
         rotate = ThinkingSphinx.sphinx_running? ? "--rotate" : ""
         
-        output = `#{config.bin_path}indexer --config #{config.config_file} #{rotate} #{delta_index_name model}`
+        output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} #{rotate} #{delta_index_name model}`
         puts(output) unless ThinkingSphinx.suppress_delta_output?
         
         client.update(

data/lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb

@@ -14,7 +14,7 @@ module ThinkingSphinx
         config = ThinkingSphinx::Configuration.instance
         client = Riddle::Client.new config.address, config.port
         
-        output = `#{config.bin_path}indexer --config #{config.config_file} --rotate #{index}`
+        output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} --rotate #{index}`
         puts output unless ThinkingSphinx.suppress_delta_output?
         
         true

data/lib/thinking_sphinx/deltas/delayed_delta.rb

@@ -8,6 +8,9 @@ module ThinkingSphinx
   module Deltas
     class DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
       def index(model, instance = nil)
+        return true unless ThinkingSphinx.updates_enabled? && ThinkingSphinx.deltas_enabled?
+        return true if instance && !toggled(instance)
+      
         ThinkingSphinx::Deltas::Job.enqueue(
           ThinkingSphinx::Deltas::DeltaJob.new(delta_index_name(model)),
           ThinkingSphinx::Configuration.instance.delayed_job_priority

data/lib/thinking_sphinx/deploy/capistrano.rb

@@ -1,11 +1,26 @@
 Capistrano::Configuration.instance(:must_exist).load do
   namespace :thinking_sphinx do
     namespace :install do
-      desc "Install Sphinx by source"
+      desc <<-DESC
+        Install Sphinx by source
+        
+        If Postgres is available, Sphinx will use it.
+        
+        If the variable :thinking_sphinx_configure_args is set, it will
+        be passed to the Sphinx configure script. You can use this to
+        install Sphinx in a non-standard location:
+        
+          set :thinking_sphinx_configure_args, "--prefix=$HOME/software"
+DESC
+
       task :sphinx do
         with_postgres = false
-        run "which pg_config" do |channel, stream, data|
-          with_postgres = !(data.nil? || data == "")
+        begin
+          run "which pg_config" do |channel, stream, data|
+            with_postgres = !(data.nil? || data == "")
+          end
+        rescue Capistrano::CommandError => e
+          puts "Continuing despite error: #{e.message}"
         end
       
         args = []
@@ -14,14 +29,15 @@ Capistrano::Configuration.instance(:must_exist).load do
             args << "--with-pgsql=#{data}"
           end
         end
-      
+        args << fetch(:thinking_sphinx_configure_args, '')
+        
         commands = <<-CMD
         wget -q http://www.sphinxsearch.com/downloads/sphinx-0.9.8.1.tar.gz >> sphinx.log
         tar xzvf sphinx-0.9.8.1.tar.gz
         cd sphinx-0.9.8.1
         ./configure #{args.join(" ")}
         make
-        sudo make install
+        #{try_sudo} make install
         rm -rf sphinx-0.9.8.1 sphinx-0.9.8.1.tar.gz
         CMD
         run commands.split(/\n\s+/).join(" && ")
@@ -29,7 +45,7 @@ Capistrano::Configuration.instance(:must_exist).load do
     
       desc "Install Thinking Sphinx as a gem from GitHub"
       task :ts do
-        sudo "gem install freelancing-god-thinking-sphinx --source http://gems.github.com"
+        run "#{try_sudo} gem install freelancing-god-thinking-sphinx --source http://gems.github.com"
       end
     end
   
@@ -70,12 +86,13 @@ Capistrano::Configuration.instance(:must_exist).load do
   
     desc "Add the shared folder for sphinx files for the production environment"
     task :shared_sphinx_folder, :roles => :web do
-      sudo "mkdir -p #{shared_path}/db/sphinx/production"
+      run "mkdir -p #{shared_path}/db/sphinx/production"
     end
 
     def rake(*tasks)
+      rails_env = fetch(:rails_env, "production")
       tasks.each do |t|
-        run "cd #{current_path} && rake #{t} RAILS_ENV=production"
+        run "cd #{current_path} && rake #{t} RAILS_ENV=#{rails_env}"
       end
     end
   end