zipme-thinking-sphinx 1.3.14

data/features/support/models/tagging.rb

@@ -0,0 +1,4 @@
+class Tagging < ActiveRecord::Base
+  belongs_to :tag
+  belongs_to :taggable, :polymorphic => true
+end

data/lib/cucumber/thinking_sphinx/external_world.rb

@@ -0,0 +1,8 @@
+require 'thinking_sphinx/test'
+
+class Cucumber::ThinkingSphinx::ExternalWorld
+  def initialize(suppress_delta_output = true)
+    ::ThinkingSphinx::Test.init
+    ::ThinkingSphinx::Test.start_with_autostop
+  end
+end

data/lib/cucumber/thinking_sphinx/internal_world.rb

@@ -0,0 +1,126 @@
+require 'cucumber/thinking_sphinx/sql_logger'
+
+module Cucumber
+  module ThinkingSphinx
+    class InternalWorld
+      attr_accessor :temporary_directory, :migrations_directory,
+        :models_directory, :fixtures_directory, :database_file
+      attr_accessor :adapter, :database, :username,
+        :password, :host
+      
+      def initialize
+        @temporary_directory  = "#{Dir.pwd}/tmp"
+        @migrations_directory = "features/support/db/migrations"
+        @models_directory     = "features/support/models"
+        @fixtures_directory   = "features/support/db/fixtures"
+        @database_file        = "features/support/database.yml"
+        
+        @adapter  = ENV['DATABASE'] || 'mysql'
+        @database = 'thinking_sphinx'
+        @username = 'thinking_sphinx'
+        # @password = 'thinking_sphinx'
+        @host     = 'localhost'
+      end
+      
+      def setup
+        make_temporary_directory
+        
+        configure_cleanup
+        configure_thinking_sphinx
+        configure_active_record
+        
+        prepare_data
+        setup_sphinx
+        
+        self
+      end
+      
+      def configure_database
+        ActiveRecord::Base.establish_connection database_settings
+        self
+      end
+      
+      private
+      
+      def config
+        @config ||= ::ThinkingSphinx::Configuration.instance
+      end
+      
+      def make_temporary_directory
+        FileUtils.mkdir_p temporary_directory
+        Dir["#{temporary_directory}/*"].each do |file|
+          FileUtils.rm_rf file
+        end
+      end
+      
+      def configure_thinking_sphinx
+        config.config_file        = "#{temporary_directory}/sphinx.conf"
+        config.searchd_log_file   = "#{temporary_directory}/searchd.log"
+        config.query_log_file     = "#{temporary_directory}/searchd.query.log"
+        config.pid_file           = "#{temporary_directory}/searchd.pid"
+        config.searchd_file_path  = "#{temporary_directory}/indexes/"
+        
+        ::ThinkingSphinx.suppress_delta_output = true
+      end
+      
+      def configure_cleanup
+        Kernel.at_exit do
+          ::ThinkingSphinx::Configuration.instance.controller.stop
+          sleep(0.5) # Ensure Sphinx has shut down completely
+          ActiveRecord::Base.logger.close
+        end
+      end
+      
+      def yaml_database_settings
+        return {} unless File.exist?(@database_file)
+        
+        YAML.load open(@database_file)
+      end
+      
+      def database_settings
+        {
+          'adapter'  => @adapter,
+          'database' => @database,
+          'username' => @username,
+          'password' => @password,
+          'host'     => @host
+        }.merge yaml_database_settings
+      end
+      
+      def configure_active_record
+        ActiveRecord::Base.logger = Logger.new(
+          open("#{temporary_directory}/active_record.log", "a")
+        )
+        
+        ActiveRecord::Base.connection.class.send(
+          :include, Cucumber::ThinkingSphinx::SqlLogger
+        )
+      end
+      
+      def prepare_data
+        ::ThinkingSphinx.deltas_enabled = false
+        
+        load_files migrations_directory
+        load_files models_directory
+        load_files fixtures_directory
+        
+        ::ThinkingSphinx.deltas_enabled = true
+      end
+      
+      def load_files(path)
+        files = Dir["#{path}/*.rb"].sort!
+        files.each do |file|
+          require file.gsub(/\.rb$/, '')
+        end
+      end
+      
+      def setup_sphinx
+        FileUtils.mkdir_p config.searchd_file_path
+        
+        config.build
+        config.controller.index
+        config.controller.start
+      end
+    end
+  end
+end

data/lib/cucumber/thinking_sphinx/sql_logger.rb

@@ -0,0 +1,20 @@
+module Cucumber
+  module ThinkingSphinx
+    module SqlLogger
+      def self.included(base)
+        base.send :alias_method_chain, :execute, :query_record
+      end
+      
+      IGNORED_SQL = [
+        /^PRAGMA/, /^SELECT currval/, /^SELECT CAST/, /^SELECT @@IDENTITY/,
+        /^SELECT @@ROWCOUNT/, /^SHOW FIELDS/
+      ]
+      
+      def execute_with_query_record(sql, name = nil, &block)
+        $queries_executed ||= []
+        $queries_executed << sql unless IGNORED_SQL.any? { |r| sql =~ r }
+        execute_without_query_record(sql, name, &block)
+      end
+    end
+  end
+end

data/lib/thinking_sphinx.rb

@@ -0,0 +1,231 @@
+require 'active_record'
+require 'after_commit'
+require 'yaml'
+require 'cgi'
+require 'riddle'
+
+require 'thinking_sphinx/auto_version'
+require 'thinking_sphinx/core/array'
+require 'thinking_sphinx/core/string'
+require 'thinking_sphinx/property'
+require 'thinking_sphinx/active_record'
+require 'thinking_sphinx/association'
+require 'thinking_sphinx/attribute'
+require 'thinking_sphinx/configuration'
+require 'thinking_sphinx/context'
+require 'thinking_sphinx/excerpter'
+require 'thinking_sphinx/facet'
+require 'thinking_sphinx/class_facet'
+require 'thinking_sphinx/facet_search'
+require 'thinking_sphinx/field'
+require 'thinking_sphinx/index'
+require 'thinking_sphinx/source'
+require 'thinking_sphinx/rails_additions'
+require 'thinking_sphinx/search'
+require 'thinking_sphinx/search_methods'
+require 'thinking_sphinx/deltas'
+
+require 'thinking_sphinx/adapters/abstract_adapter'
+require 'thinking_sphinx/adapters/mysql_adapter'
+require 'thinking_sphinx/adapters/postgresql_adapter'
+
+ActiveRecord::Base.send(:include, ThinkingSphinx::ActiveRecord)
+
+Merb::Plugins.add_rakefiles(
+  File.join(File.dirname(__FILE__), "thinking_sphinx", "tasks")
+) if defined?(Merb)
+
+module ThinkingSphinx
+  # A ConnectionError will get thrown when a connection to Sphinx can't be
+  # made.
+  class ConnectionError < StandardError
+  end
+
+  # A StaleIdsException is thrown by Collection.instances_from_matches if there
+  # are records in Sphinx but not in the database, so the search can be retried.
+  class StaleIdsException < StandardError
+    attr_accessor :ids
+    def initialize(ids)
+      self.ids = ids
+    end
+  end
+  
+  # The current version of Thinking Sphinx.
+  # 
+  # @return [String] The version number as a string
+  # 
+  def self.version
+    open(File.join(File.dirname(__FILE__), '../VERSION')) { |f|
+      f.read.strip
+    }
+  end
+  
+  # The collection of indexed models. Keep in mind that Rails lazily loads
+  # its classes, so this may not actually be populated with _all_ the models
+  # that have Sphinx indexes.
+  def self.context
+    if Thread.current[:thinking_sphinx_context].nil?
+      Thread.current[:thinking_sphinx_context] = ThinkingSphinx::Context.new
+      Thread.current[:thinking_sphinx_context].prepare
+    end
+    
+    Thread.current[:thinking_sphinx_context]
+  end
+  
+  def self.reset_context!
+    Thread.current[:thinking_sphinx_context] = nil
+  end
+
+  def self.unique_id_expression(offset = nil)
+    "* #{context.indexed_models.size} + #{offset || 0}"
+  end
+
+  # Check if index definition is disabled.
+  #
+  def self.define_indexes?
+    if Thread.current[:thinking_sphinx_define_indexes].nil?
+      Thread.current[:thinking_sphinx_define_indexes] = true
+    end
+    
+    Thread.current[:thinking_sphinx_define_indexes]
+  end
+
+  # Enable/disable indexes - you may want to do this while migrating data.
+  #
+  #   ThinkingSphinx.define_indexes = false
+  #
+  def self.define_indexes=(value)
+    Thread.current[:thinking_sphinx_define_indexes] = value
+  end
+
+  # Check if delta indexing is enabled.
+  #
+  def self.deltas_enabled?
+    if Thread.current[:thinking_sphinx_deltas_enabled].nil?
+      Thread.current[:thinking_sphinx_deltas_enabled] = (
+        ThinkingSphinx::Configuration.environment != "test"
+      )
+    end
+    
+    Thread.current[:thinking_sphinx_deltas_enabled]
+  end
+
+  # Enable/disable all delta indexing.
+  #
+  #   ThinkingSphinx.deltas_enabled = false
+  #
+  def self.deltas_enabled=(value)
+    Thread.current[:thinking_sphinx_deltas_enabled] = value
+  end
+
+  # Check if updates are enabled. True by default, unless within the test
+  # environment.
+  #
+  def self.updates_enabled?
+    if Thread.current[:thinking_sphinx_updates_enabled].nil?
+      Thread.current[:thinking_sphinx_updates_enabled] = (
+        ThinkingSphinx::Configuration.environment != "test"
+      )
+    end
+    
+    Thread.current[:thinking_sphinx_updates_enabled]
+  end
+
+  # Enable/disable updates to Sphinx
+  #
+  #   ThinkingSphinx.updates_enabled = false
+  #
+  def self.updates_enabled=(value)
+    Thread.current[:thinking_sphinx_updates_enabled] = value
+  end
+
+  def self.suppress_delta_output?
+    Thread.current[:thinking_sphinx_suppress_delta_output] ||= false
+  end
+
+  def self.suppress_delta_output=(value)
+    Thread.current[:thinking_sphinx_suppress_delta_output] = value
+  end
+  
+  # Checks to see if MySQL will allow simplistic GROUP BY statements. If not,
+  # or if not using MySQL, this will return false.
+  #
+  def self.use_group_by_shortcut?
+    if Thread.current[:thinking_sphinx_use_group_by_shortcut].nil?
+      Thread.current[:thinking_sphinx_use_group_by_shortcut] = !!(
+        mysql? && ::ActiveRecord::Base.connection.select_all(
+          "SELECT @@global.sql_mode, @@session.sql_mode;"
+        ).all? { |key,value| value.nil? || value[/ONLY_FULL_GROUP_BY/].nil? }
+      )
+    end
+    
+    Thread.current[:thinking_sphinx_use_group_by_shortcut]
+  end
+
+  # An indication of whether Sphinx is running on a remote machine instead of
+  # the same machine.
+  #
+  def self.remote_sphinx?
+    Thread.current[:thinking_sphinx_remote_sphinx] ||= false
+  end
+
+  # Tells Thinking Sphinx that Sphinx is running on a different machine, and
+  # thus it can't reliably guess whether it is running or not (ie: the
+  # #sphinx_running? method), and so just assumes it is.
+  #
+  # Useful for multi-machine deployments. Set it in your production.rb file.
+  #
+  #   ThinkingSphinx.remote_sphinx = true
+  #
+  def self.remote_sphinx=(value)
+    Thread.current[:thinking_sphinx_remote_sphinx] = value
+  end
+
+  # Check if Sphinx is running. If remote_sphinx is set to true (indicating
+  # Sphinx is on a different machine), this will always return true, and you
+  # will have to handle any connection errors yourself.
+  #
+  def self.sphinx_running?
+    remote_sphinx? || sphinx_running_by_pid?
+  end
+
+  # Check if Sphinx is actually running, provided the pid is on the same
+  # machine as this code.
+  #
+  def self.sphinx_running_by_pid?
+    !!sphinx_pid && pid_active?(sphinx_pid)
+  end
+
+  def self.sphinx_pid
+    if File.exists?(ThinkingSphinx::Configuration.instance.pid_file)
+      File.read(ThinkingSphinx::Configuration.instance.pid_file)[/\d+/]
+    else
+      nil
+    end
+  end
+
+  def self.pid_active?(pid)
+    !!Process.kill(0, pid.to_i)
+  rescue Exception => e
+    false
+  end
+
+  def self.microsoft?
+    RUBY_PLATFORM =~ /mswin/
+  end
+
+  def self.jruby?
+    defined?(JRUBY_VERSION)
+  end
+
+  def self.mysql?
+    ::ActiveRecord::Base.connection.class.name.demodulize == "MysqlAdapter" ||
+    ::ActiveRecord::Base.connection.class.name.demodulize == "MysqlplusAdapter" || (
+      jruby? && ::ActiveRecord::Base.connection.config[:adapter] == "jdbcmysql"
+    )
+  end
+  
+  extend ThinkingSphinx::SearchMethods::ClassMethods
+end
+
+ThinkingSphinx::AutoVersion.detect

data/lib/thinking_sphinx/active_record.rb

@@ -0,0 +1,373 @@
+require 'thinking_sphinx/active_record/attribute_updates'
+require 'thinking_sphinx/active_record/delta'
+require 'thinking_sphinx/active_record/has_many_association'
+require 'thinking_sphinx/active_record/scopes'
+
+module ThinkingSphinx
+  # Core additions to ActiveRecord models - define_index for creating indexes
+  # for models. If you want to interrogate the index objects created for the
+  # model, you can use the class-level accessor :sphinx_indexes.
+  #
+  module ActiveRecord
+    def self.included(base)
+      base.class_eval do
+        class_inheritable_array :sphinx_indexes, :sphinx_facets
+        
+        extend ThinkingSphinx::ActiveRecord::ClassMethods
+        
+        class << self
+          attr_accessor :sphinx_index_blocks
+          
+          def set_sphinx_primary_key(attribute)
+            @sphinx_primary_key_attribute = attribute
+          end
+          
+          def primary_key_for_sphinx
+            @sphinx_primary_key_attribute || primary_key
+          end
+          
+          def sphinx_index_options
+            sphinx_indexes.last.options
+          end
+          
+          # Generate a unique CRC value for the model's name, to use to
+          # determine which Sphinx documents belong to which AR records.
+          # 
+          # Really only written for internal use - but hey, if it's useful to
+          # you in some other way, awesome.
+          # 
+          def to_crc32
+            self.name.to_crc32
+          end
+          
+          def to_crc32s
+            (subclasses << self).collect { |klass| klass.to_crc32 }
+          end
+          
+          def sphinx_database_adapter
+            @sphinx_database_adapter ||=
+              ThinkingSphinx::AbstractAdapter.detect(self)
+          end
+          
+          def sphinx_name
+            self.name.underscore.tr(':/\\', '_')
+          end
+          
+          #
+          # The above method to_crc32s is dependant on the subclasses being loaded consistently
+          # After a reset_subclasses is called (during a Dispatcher.cleanup_application in development)
+          # Our subclasses will be lost but our context will not reload them for us.
+          #
+          # We reset the context which causes the subclasses to be reloaded next time the context is called.
+          #
+          def reset_subclasses_with_thinking_sphinx
+            reset_subclasses_without_thinking_sphinx
+            ThinkingSphinx.reset_context!
+          end
+          
+          alias_method_chain :reset_subclasses, :thinking_sphinx
+          
+          private
+          
+          def defined_indexes?
+            @defined_indexes
+          end
+          
+          def defined_indexes=(value)
+            @defined_indexes = value
+          end
+          
+          def sphinx_delta?
+            self.sphinx_indexes.any? { |index| index.delta? }
+          end
+        end
+      end
+      
+      ::ActiveRecord::Associations::HasManyAssociation.send(
+        :include, ThinkingSphinx::ActiveRecord::HasManyAssociation
+      )
+      ::ActiveRecord::Associations::HasManyThroughAssociation.send(
+        :include, ThinkingSphinx::ActiveRecord::HasManyAssociation
+      )
+    end
+    
+    module ClassMethods
+      # Allows creation of indexes for Sphinx. If you don't do this, there
+      # isn't much point trying to search (or using this plugin at all,
+      # really).
+      #
+      # An example or two:
+      #
+      #   define_index
+      #     indexes :id, :as => :model_id
+      #     indexes name
+      #   end
+      #
+      # You can also grab fields from associations - multiple levels deep
+      # if necessary.
+      #
+      #   define_index do
+      #     indexes tags.name, :as => :tag
+      #     indexes articles.content
+      #     indexes orders.line_items.product.name, :as => :product
+      #   end
+      #
+      # And it will automatically concatenate multiple fields:
+      #
+      #   define_index do
+      #     indexes [author.first_name, author.last_name], :as => :author
+      #   end
+      #
+      # The #indexes method is for fields - if you want attributes, use
+      # #has instead. All the same rules apply - but keep in mind that
+      # attributes are for sorting, grouping and filtering, not searching.
+      #
+      #   define_index do
+      #     # fields ...
+      #     
+      #     has created_at, updated_at
+      #   end
+      #
+      # One last feature is the delta index. This requires the model to
+      # have a boolean field named 'delta', and is enabled as follows:
+      #
+      #   define_index do
+      #     # fields ...
+      #     # attributes ...
+      #     
+      #     set_property :delta => true
+      #   end
+      #
+      # Check out the more detailed documentation for each of these methods
+      # at ThinkingSphinx::Index::Builder.
+      # 
+      def define_index(name = nil, &block)
+        self.sphinx_index_blocks ||= []
+        self.sphinx_indexes      ||= []
+        self.sphinx_facets       ||= []
+        
+        ThinkingSphinx.context.add_indexed_model self
+        
+        if sphinx_index_blocks.empty?
+          before_validation :define_indexes
+          before_destroy    :define_indexes
+        end
+        
+        self.sphinx_index_blocks << lambda {
+          index = ThinkingSphinx::Index::Builder.generate self, name, &block
+          add_sphinx_callbacks_and_extend(index.delta?)
+          add_sphinx_index index
+        }
+        
+        include ThinkingSphinx::ActiveRecord::Scopes
+        include ThinkingSphinx::SearchMethods
+      end
+      
+      def define_indexes
+        superclass.define_indexes unless superclass == ::ActiveRecord::Base
+        
+        return if sphinx_index_blocks.nil? ||
+          defined_indexes?                 ||
+          !ThinkingSphinx.define_indexes?
+        
+        sphinx_index_blocks.each do |block|
+          block.call
+        end
+        
+        self.defined_indexes = true
+        
+        # We want to make sure that if the database doesn't exist, then Thinking
+        # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop
+        # and db:migrate). It's a bit hacky, but I can't think of a better way.
+      rescue StandardError => err
+        case err.class.name
+        when "Mysql::Error", "Java::JavaSql::SQLException", "ActiveRecord::StatementInvalid"
+          return
+        else
+          raise err
+        end
+      end
+      
+      def add_sphinx_index(index)
+        self.sphinx_indexes << index
+        subclasses.each { |klass| klass.add_sphinx_index(index) }
+      end
+      
+      def has_sphinx_indexes?
+        sphinx_indexes      && 
+        sphinx_index_blocks &&
+        (sphinx_indexes.length > 0 || sphinx_index_blocks.length > 0)
+      end
+      
+      def indexed_by_sphinx?
+        sphinx_indexes && sphinx_indexes.length > 0
+      end
+      
+      def delta_indexed_by_sphinx?
+        sphinx_indexes && sphinx_indexes.any? { |index| index.delta? }
+      end
+      
+      def sphinx_index_names
+        define_indexes
+        sphinx_indexes.collect(&:all_names).flatten
+      end
+      
+      def core_index_names
+        define_indexes
+        sphinx_indexes.collect(&:core_name)
+      end
+      
+      def delta_index_names
+        define_indexes
+        sphinx_indexes.select(&:delta?).collect(&:delta_name)
+      end
+      
+      def to_riddle
+        define_indexes
+        sphinx_database_adapter.setup
+        
+        local_sphinx_indexes.collect { |index|
+          index.to_riddle(sphinx_offset)
+        }.flatten
+      end
+      
+      def source_of_sphinx_index
+        define_indexes
+        possible_models = self.sphinx_indexes.collect { |index| index.model }
+        return self if possible_models.include?(self)
+
+        parent = self.superclass
+        while !possible_models.include?(parent) && parent != ::ActiveRecord::Base
+          parent = parent.superclass
+        end
+
+        return parent
+      end
+      
+      def delete_in_index(index, document_id)
+        return unless ThinkingSphinx.sphinx_running? &&
+          search_for_id(document_id, index)
+        
+        ThinkingSphinx::Configuration.instance.client.update(
+          index, ['sphinx_deleted'], {document_id => [1]}
+        )
+      end
+      
+      def sphinx_offset
+        ThinkingSphinx.context.superclass_indexed_models.
+          index eldest_indexed_ancestor
+      end
+      
+      # Temporarily disable delta indexing inside a block, then perform a single
+      # rebuild of index at the end.
+      #
+      # Useful when performing updates to batches of models to prevent
+      # the delta index being rebuilt after each individual update.
+      #
+      # In the following example, the delta index will only be rebuilt once,
+      # not 10 times.
+      #
+      #   SomeModel.suspended_delta do
+      #     10.times do
+      #       SomeModel.create( ... )
+      #     end
+      #   end
+      #
+      def suspended_delta(reindex_after = true, &block)
+        define_indexes
+        original_setting = ThinkingSphinx.deltas_enabled?
+        ThinkingSphinx.deltas_enabled = false
+        begin
+          yield
+        ensure
+          ThinkingSphinx.deltas_enabled = original_setting
+          self.index_delta if reindex_after
+        end
+      end
+      
+      private
+            
+      def local_sphinx_indexes
+        sphinx_indexes.select { |index|
+          index.model == self
+        }
+      end
+      
+      def add_sphinx_callbacks_and_extend(delta = false)
+        unless indexed_by_sphinx?
+          after_destroy :toggle_deleted
+          
+          include ThinkingSphinx::ActiveRecord::AttributeUpdates
+        end
+        
+        if delta && !delta_indexed_by_sphinx?
+          include ThinkingSphinx::ActiveRecord::Delta
+          
+          before_save   :toggle_delta
+          after_commit  :index_delta
+        end
+      end
+      
+      def eldest_indexed_ancestor
+        ancestors.reverse.detect { |ancestor|
+          ThinkingSphinx.context.indexed_models.include?(ancestor.name)
+        }.name
+      end
+    end
+    
+    def in_index?(suffix)
+      self.class.search_for_id self.sphinx_document_id, sphinx_index_name(suffix)
+    end
+    
+    def in_core_index?
+      in_index? "core"
+    end
+    
+    def in_delta_index?
+      in_index? "delta"
+    end
+    
+    def in_both_indexes?
+      in_core_index? && in_delta_index?
+    end
+    
+    def toggle_deleted
+      return unless ThinkingSphinx.updates_enabled?
+      
+      self.class.core_index_names.each do |index_name|
+        self.class.delete_in_index index_name, self.sphinx_document_id
+      end
+      self.class.delta_index_names.each do |index_name|
+        self.class.delete_in_index index_name, self.sphinx_document_id
+      end if self.class.delta_indexed_by_sphinx? && toggled_delta?
+      
+    rescue ::ThinkingSphinx::ConnectionError
+      # nothing
+    end
+    
+    # Returns the unique integer id for the object. This method uses the
+    # attribute hash to get around ActiveRecord always mapping the #id method
+    # to whatever the real primary key is (which may be a unique string hash).
+    # 
+    # @return [Integer] Unique record id for the purposes of Sphinx.
+    # 
+    def primary_key_for_sphinx
+      @primary_key_for_sphinx ||= read_attribute(self.class.primary_key_for_sphinx)
+    end
+    
+    def sphinx_document_id
+      primary_key_for_sphinx * ThinkingSphinx.context.indexed_models.size +
+        self.class.sphinx_offset
+    end
+
+    private
+
+    def sphinx_index_name(suffix)
+      "#{self.class.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_#{suffix}"
+    end
+    
+    def define_indexes
+      self.class.define_indexes
+    end
+  end
+end