DrMark-thinking-sphinx 0.9.9 → 1.1.6

data/README

@@ -10,28 +10,54 @@ Keep in mind that while Thinking Sphinx works for ActiveRecord with Merb, it doe
 
 Fork on GitHub and after you've committed tested patches, send a pull request.
 
+To quickly see if your system is ready to run the thinking sphinx specs, run the contribute.rb script found in the project root directory. Use the following instructions to install any missing requirements.
+
 To get the spec suite running, you will need to install the not-a-mock gem if you don't already have it:
 
   git clone git://github.com/freelancing-god/not-a-mock.git
   cd not-a-mock
   rake gem
   gem install pkg/not_a_mock-1.1.0.gem
+  
+Then install the ginger gem. The steps are the same, except that you might need to sudo the gem install:
+
+  git clone git://github.com/freelancing-god/ginger.git
+  cd ginger
+  rake gem
+  sudo gem install pkg/ginger-1.1.0.gem
 
-Then set up your database
+Then install the faker gem:
+
+  sudo gem install faker
+
+Then set up your database:
 
   cp spec/fixtures/database.yml.default spec/fixtures/database.yml
+  cp spec/fixtures/database.yml.default features/support/db/database.yml
   mysqladmin -u root create thinking_sphinx
+  
+Make sure you don't have another Sphinx daemon (searchd) running. If you do, quit it with "rake ts:stop"
+in the app root.
 
 You should now have a passing test suite from which to build your patch on.
 
   rake spec
+	rake features  (Note: You will need MySQL and Postgres gems to run the full suite. You may want to tweak your rakefile)
+  
+If you get the message "Failed to start searchd daemon", run the spec with sudo:
+
+  sudo rake spec
+  sudo rake features  
+  
+If you quit the spec suite before it's completed, you may be left with data in the test
+database, causing the next run to have failures. Let that run complete and then try again.
 
 == Contributors
 
 Since I first released this library, there's been quite a few people who have submitted patches, to my immense gratitude. Others have suggested syntax changes and general improvements. So my thanks to the following people:
 
 - Joost Hietbrink
-- Jonathon Conway
+- Jonathan Conway
 - Gregory Mirzayantz
 - Tung Nguyen
 - Sean Cribbs
@@ -63,3 +89,39 @@ Since I first released this library, there's been quite a few people who have su
 - David Eisinger
 - Shay Arnett
 - Minh Tran
+- Jeremy Durham
+- Piotr Sarnacki
+- Matt Johnson
+- Nicolas Blanco
+- Max Lapshin
+- Josh Natanson
+- Philip Hallstrom
+- Christian Rishøj
+- Mike Flester
+- Jim Remsik
+- Kennon Ballou
+- Henrik Nyh
+- Emil Tin
+- Doug Cole
+- Ed Hickey
+- Evan Weaver
+- Thibaut Barrere
+- Kristopher Chambers
+- Dmitrij Smalko
+- Aleksey Yeschenko
+- Lachie Cox
+- Lourens Naude
+- Tom Davies
+- Dan Pickett
+- Alex Caudill
+- Jim Benton
+- John Aughey
+- Keith Pitty
+- Jeff Talbot
+- Dana Contreras
+- Menno van der Sman
+- Bill Harding
+- Isaac Feliu
+- Andrei Bocan
+- László Bácsi
+- Peter Wagenet

data/lib/thinking_sphinx.rb

@@ -1,27 +1,41 @@
+Dir[File.join(File.dirname(__FILE__), '../vendor/*/lib')].each do |path|
+  $LOAD_PATH.unshift path
+end
+
 require 'active_record'
 require 'riddle'
+require 'after_commit'
 
+require 'thinking_sphinx/core/string'
 require 'thinking_sphinx/active_record'
 require 'thinking_sphinx/association'
 require 'thinking_sphinx/attribute'
 require 'thinking_sphinx/collection'
 require 'thinking_sphinx/configuration'
+require 'thinking_sphinx/facet'
+require 'thinking_sphinx/class_facet'
+require 'thinking_sphinx/facet_collection'
 require 'thinking_sphinx/field'
 require 'thinking_sphinx/index'
 require 'thinking_sphinx/rails_additions'
 require 'thinking_sphinx/search'
+require 'thinking_sphinx/deltas'
+
+require 'thinking_sphinx/adapters/abstract_adapter'
+require 'thinking_sphinx/adapters/mysql_adapter'
+require 'thinking_sphinx/adapters/postgresql_adapter'
 
 ActiveRecord::Base.send(:include, ThinkingSphinx::ActiveRecord)
 
 Merb::Plugins.add_rakefiles(
-  File.join(File.dirname(__FILE__), "..", "tasks", "thinking_sphinx_tasks")
+  File.join(File.dirname(__FILE__), "thinking_sphinx", "tasks")
 ) if defined?(Merb)
 
 module ThinkingSphinx
   module Version #:nodoc:
-    Major = 0
-    Minor = 9
-    Tiny  = 9
+    Major = 1
+    Minor = 1
+    Tiny  = 6
     
     String = [Major, Minor, Tiny].join('.')
   end
@@ -31,6 +45,15 @@ module ThinkingSphinx
   class ConnectionError < StandardError
   end
   
+  # A StaleIdsException is thrown by Collection.instances_from_matches if there
+  # are records in Sphinx but not in the database, so the search can be retried.
+  class StaleIdsException < StandardError
+    attr_accessor :ids
+    def initialize(ids)
+      self.ids = ids
+    end
+  end
+  
   # The collection of indexed models. Keep in mind that Rails lazily loads
   # its classes, so this may not actually be populated with _all_ the models
   # that have Sphinx indexes.
@@ -38,6 +61,10 @@ module ThinkingSphinx
     @@indexed_models ||= []
   end
   
+  def self.unique_id_expression(offset = nil)
+    "* #{ThinkingSphinx.indexed_models.size} + #{offset || 0}"
+  end
+  
   # Check if index definition is disabled.
   # 
   def self.define_indexes?
@@ -88,16 +115,66 @@ module ThinkingSphinx
     @@updates_enabled = value
   end
   
+  @@suppress_delta_output = false
+  
+  def self.suppress_delta_output?
+    @@suppress_delta_output
+  end
+  
+  def self.suppress_delta_output=(value)
+    @@suppress_delta_output = value
+  end
+  
   # Checks to see if MySQL will allow simplistic GROUP BY statements. If not,
   # or if not using MySQL, this will return false.
   # 
   def self.use_group_by_shortcut?
-    ::ActiveRecord::ConnectionAdapters.constants.include?("MysqlAdapter") &&
-    ::ActiveRecord::Base.connection.is_a?(
-      ::ActiveRecord::ConnectionAdapters::MysqlAdapter
-    ) &&
-    ::ActiveRecord::Base.connection.select_all(
-      "SELECT @@global.sql_mode, @@session.sql_mode;"
-    ).all? { |key,value| value.nil? || value[/ONLY_FULL_GROUP_BY/].nil? }
+    !!(
+      mysql? && ::ActiveRecord::Base.connection.select_all(
+        "SELECT @@global.sql_mode, @@session.sql_mode;"
+      ).all? { |key,value| value.nil? || value[/ONLY_FULL_GROUP_BY/].nil? }
+    )
+  end
+  
+  def self.sphinx_running?
+    !!sphinx_pid && pid_active?(sphinx_pid)
+  end
+  
+  def self.sphinx_pid
+    pid_file    = ThinkingSphinx::Configuration.instance.pid_file
+    cat_command = 'cat'
+    return nil unless File.exists?(pid_file)
+    
+    if microsoft?
+      pid_file.gsub!('/', '\\')
+      cat_command = 'type'
+    end
+    
+    `#{cat_command} #{pid_file}`[/\d+/]
+  end
+  
+  def self.pid_active?(pid)
+    return true if microsoft?
+    
+    begin
+      # In JRuby this returns -1 if the process doesn't exist
+      Process.getpgid(pid.to_i) != -1
+    rescue Exception => e
+      false
+    end
+  end
+  
+  def self.microsoft?
+    RUBY_PLATFORM =~ /mswin/
+  end
+  
+  def self.jruby?
+    defined?(JRUBY_VERSION)
+  end
+  
+  def self.mysql?
+    ::ActiveRecord::Base.connection.class.name.demodulize == "MysqlAdapter" || (
+      jruby? && ::ActiveRecord::Base.connection.config[:adapter] == "jdbcmysql"
+    )
   end
 end

data/lib/thinking_sphinx/active_record.rb

@@ -5,12 +5,12 @@ require 'thinking_sphinx/active_record/has_many_association'
 module ThinkingSphinx
   # Core additions to ActiveRecord models - define_index for creating indexes
   # for models. If you want to interrogate the index objects created for the
-  # model, you can use the class-level accessor :indexes.
+  # model, you can use the class-level accessor :sphinx_indexes.
   #
   module ActiveRecord
     def self.included(base)
       base.class_eval do
-        class_inheritable_array :indexes
+        class_inheritable_array :sphinx_indexes, :sphinx_facets
         class << self
           # Allows creation of indexes for Sphinx. If you don't do this, there
           # isn't much point trying to search (or using this plugin at all,
@@ -64,10 +64,10 @@ module ThinkingSphinx
           def define_index(&block)
             return unless ThinkingSphinx.define_indexes?
             
-            self.indexes ||= []
+            self.sphinx_indexes ||= []
             index = Index.new(self, &block)
             
-            self.indexes << index
+            self.sphinx_indexes << index
             unless ThinkingSphinx.indexed_models.include?(self.name)
               ThinkingSphinx.indexed_models << self.name
             end
@@ -83,6 +83,10 @@ module ThinkingSphinx
           end
           alias_method :sphinx_index, :define_index
           
+          def sphinx_index_options
+            sphinx_indexes.last.options
+          end
+          
           # Generate a unique CRC value for the model's name, to use to
           # determine which Sphinx documents belong to which AR records.
           # 
@@ -90,19 +94,105 @@ module ThinkingSphinx
           # you in some other way, awesome.
           # 
           def to_crc32
-            result = 0xFFFFFFFF
-            self.name.each_byte do |byte|
-              result ^= byte
-              8.times do
-                result = (result >> 1) ^ (0xEDB88320 * (result & 1))
-              end
-            end
-            result ^ 0xFFFFFFFF
+            self.name.to_crc32
           end
           
           def to_crc32s
             (subclasses << self).collect { |klass| klass.to_crc32 }
           end
+          
+          def source_of_sphinx_index
+            possible_models = self.sphinx_indexes.collect { |index| index.model }
+            return self if possible_models.include?(self)
+
+            parent = self.superclass
+            while !possible_models.include?(parent) && parent != ::ActiveRecord::Base
+              parent = parent.superclass
+            end
+
+            return parent
+          end
+          
+          def to_riddle(offset)
+            sphinx_database_adapter.setup
+            
+            indexes = [to_riddle_for_core(offset)]
+            indexes << to_riddle_for_delta(offset) if sphinx_delta?
+            indexes << to_riddle_for_distributed
+          end
+          
+          def sphinx_database_adapter
+            @sphinx_database_adapter ||=
+              ThinkingSphinx::AbstractAdapter.detect(self)
+          end
+          
+          private
+          
+          def sphinx_name
+            self.name.underscore.tr(':/\\', '_')
+          end
+          
+          def sphinx_delta?
+            self.sphinx_indexes.any? { |index| index.delta? }
+          end
+          
+          def to_riddle_for_core(offset)
+            index = Riddle::Configuration::Index.new("#{sphinx_name}_core")
+            index.path = File.join(
+              ThinkingSphinx::Configuration.instance.searchd_file_path, index.name
+            )
+            
+            set_configuration_options_for_indexes index
+            set_field_settings_for_indexes        index
+            
+            self.sphinx_indexes.select { |ts_index|
+              ts_index.model == self
+            }.each_with_index do |ts_index, i|
+              index.sources << ts_index.to_riddle_for_core(offset, i)
+            end
+            
+            index
+          end
+          
+          def to_riddle_for_delta(offset)
+            index = Riddle::Configuration::Index.new("#{sphinx_name}_delta")
+            index.parent = "#{sphinx_name}_core"
+            index.path = File.join(ThinkingSphinx::Configuration.instance.searchd_file_path, index.name)
+            
+            self.sphinx_indexes.each_with_index do |ts_index, i|
+              index.sources << ts_index.to_riddle_for_delta(offset, i) if ts_index.delta?
+            end
+            
+            index
+          end
+          
+          def to_riddle_for_distributed
+            index = Riddle::Configuration::DistributedIndex.new(sphinx_name)
+            index.local_indexes << "#{sphinx_name}_core"
+            index.local_indexes.unshift "#{sphinx_name}_delta" if sphinx_delta?
+            index
+          end
+          
+          def set_configuration_options_for_indexes(index)
+            ThinkingSphinx::Configuration.instance.index_options.each do |key, value|
+              index.send("#{key}=".to_sym, value)
+            end
+            
+            self.sphinx_indexes.each do |ts_index|
+              ts_index.options.each do |key, value|
+                index.send("#{key}=".to_sym, value) if ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s) && !value.nil?
+              end
+            end
+          end
+          
+          def set_field_settings_for_indexes(index)
+            field_names = lambda { |field| field.unique_name.to_s }
+            
+            self.sphinx_indexes.each do |ts_index|
+              index.prefix_field_names += ts_index.prefix_fields.collect(&field_names)
+              index.infix_field_names  += ts_index.infix_fields.collect(&field_names)
+            end
+          end
         end
       end
       
@@ -117,29 +207,54 @@ module ThinkingSphinx
       )
     end
     
+    def in_index?(suffix)
+      self.class.search_for_id self.sphinx_document_id, sphinx_index_name(suffix)
+    end
+    
     def in_core_index?
-      @in_core_index ||= self.class.search_for_id(self.id, "#{self.class.name.downcase}_core")
+      in_index? "core"
+    end
+    
+    def in_delta_index?
+      in_index? "delta"
+    end
+    
+    def in_both_indexes?
+      in_core_index? && in_delta_index?
     end
     
     def toggle_deleted
-      return unless ThinkingSphinx.updates_enabled?
+      return unless ThinkingSphinx.updates_enabled? && ThinkingSphinx.sphinx_running?
       
-      config = ThinkingSphinx::Configuration.new
+      config = ThinkingSphinx::Configuration.instance
       client = Riddle::Client.new config.address, config.port
       
       client.update(
-        "#{self.class.indexes.first.name}_core",
+        "#{self.class.sphinx_indexes.first.name}_core",
         ['sphinx_deleted'],
-        {self.id => 1}
+        {self.sphinx_document_id => 1}
       ) if self.in_core_index?
       
       client.update(
-        "#{self.class.indexes.first.name}_delta",
+        "#{self.class.sphinx_indexes.first.name}_delta",
         ['sphinx_deleted'],
-        {self.id => 1}
+        {self.sphinx_document_id => 1}
       ) if ThinkingSphinx.deltas_enabled? &&
-        self.class.indexes.any? { |index| index.delta? } &&
-        self.delta?
+        self.class.sphinx_indexes.any? { |index| index.delta? } &&
+        self.toggled_delta?
+    rescue ::ThinkingSphinx::ConnectionError
+      # nothing
+    end
+    
+    def sphinx_document_id
+      (self.id * ThinkingSphinx.indexed_models.size) +
+        ThinkingSphinx.indexed_models.index(self.class.source_of_sphinx_index.name)
+    end
+    
+    private
+    
+    def sphinx_index_name(suffix)
+      "#{self.class.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_#{suffix}"
     end
   end
 end