thinking-sphinx 2.0.6 → 2.0.7

data/lib/thinking_sphinx/source.rb

@@ -0,0 +1,194 @@
+require 'thinking_sphinx/source/internal_properties'
+require 'thinking_sphinx/source/sql'
+
+module ThinkingSphinx
+  class Source
+    include ThinkingSphinx::Source::InternalProperties
+    include ThinkingSphinx::Source::SQL
+    
+    attr_accessor :model, :fields, :attributes, :joins, :conditions, :groupings,
+      :options
+    attr_reader :base, :index, :database_configuration
+    
+    def initialize(index, options = {})
+      @index        = index
+      @model        = index.model
+      @fields       = []
+      @attributes   = []
+      @joins        = []
+      @conditions   = []
+      @groupings    = []
+      @options      = options
+      @associations = {}
+      @database_configuration = @model.connection.
+        instance_variable_get(:@config).clone
+      
+      @base = join_dependency_class.new(
+        @model, [], initial_joins
+      )
+      
+      unless @model.descends_from_active_record?
+        stored_class = @model.store_full_sti_class ? @model.name : @model.name.demodulize
+        @conditions << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)} = '#{stored_class}'"
+      end
+      
+      add_internal_attributes_and_facets
+    end
+    
+    def name
+      index.name
+    end
+    
+    def to_riddle_for_core(offset, position)
+      source = Riddle::Configuration::SQLSource.new(
+        "#{index.core_name}_#{position}", adapter.sphinx_identifier
+      )
+      
+      set_source_database_settings  source
+      set_source_fields             source
+      set_source_attributes         source, offset
+      set_source_settings           source
+      set_source_sql                source, offset
+      
+      source
+    end
+    
+    def to_riddle_for_delta(offset, position)
+      source = Riddle::Configuration::SQLSource.new(
+        "#{index.delta_name}_#{position}", adapter.sphinx_identifier
+      )
+      source.parent = "#{index.core_name}_#{position}"
+      
+      set_source_database_settings  source
+      set_source_fields             source
+      set_source_attributes         source, offset, true
+      set_source_settings           source
+      set_source_sql                source, offset, true
+      
+      source
+    end
+    
+    def delta?
+      !@index.delta_object.nil?
+    end
+    
+    # Gets the association stack for a specific key.
+    # 
+    def association(key)
+      @associations[key] ||= Association.children(@model, key)
+    end
+    
+    private
+    
+    def adapter
+      @adapter ||= @model.sphinx_database_adapter
+    end
+    
+    def available_attributes
+      attributes.select { |attrib| attrib.available? }
+    end
+    
+    def set_source_database_settings(source)
+      config = @database_configuration
+      
+      source.sql_host = config[:host]           || "localhost"
+      source.sql_user = config[:username]       || config[:user] || 'root'
+      source.sql_pass = (config[:password].to_s || "").gsub('#', '\#')
+      source.sql_db   = config[:database]
+      source.sql_port = config[:port]
+      source.sql_sock = config[:socket]
+    end
+    
+    def set_source_fields(source)
+      fields.each do |field|
+        source.sql_file_field   << field.unique_name if field.file?
+        source.sql_field_string << field.unique_name if field.with_attribute?
+        source.sql_field_str2wordcount << field.unique_name if field.with_wordcount?
+      end
+    end
+    
+    def set_source_attributes(source, offset, delta = false)
+      available_attributes.each do |attrib|
+        source.send(attrib.type_to_config) << attrib.config_value(offset, delta)
+      end
+    end
+    
+    def set_source_sql(source, offset, delta = false)
+      source.sql_query        = to_sql(:offset => offset, :delta => delta).gsub(/\n/, ' ')
+      source.sql_query_range  = to_sql_query_range(:delta => delta)
+      source.sql_query_info   = to_sql_query_info(offset)
+      
+      source.sql_query_pre += send(!delta ? :sql_query_pre_for_core : :sql_query_pre_for_delta)
+      
+      if @index.local_options[:group_concat_max_len]
+        source.sql_query_pre << "SET SESSION group_concat_max_len = #{@index.local_options[:group_concat_max_len]}"
+      end
+      
+      source.sql_query_pre += [adapter.utf8_query_pre].compact if utf8?
+      source.sql_query_pre << adapter.utc_query_pre
+    end
+    
+    def set_source_settings(source)
+      config = ThinkingSphinx::Configuration.instance
+      config.source_options.each do |key, value|
+        source.send("#{key}=".to_sym, value)
+      end
+      
+      source_options = ThinkingSphinx::Configuration::SourceOptions
+      @options.each do |key, value|
+        if source_options.include?(key.to_s) && !value.nil?
+          source.send("#{key}=".to_sym, value)
+        end
+      end
+    end
+    
+    # Returns all associations used amongst all the fields and attributes.
+    # This includes all associations between the model and what the actual
+    # columns are from.
+    # 
+    def all_associations
+      @all_associations ||= (
+        # field associations
+        @fields.collect { |field|
+          field.associations.values
+        }.flatten +
+        # attribute associations
+        @attributes.collect { |attrib|
+          attrib.associations.values if attrib.include_as_association?
+        }.compact.flatten +
+        # explicit joins
+        @joins.collect { |join|
+          join.associations
+        }.flatten
+      ).uniq.collect { |assoc|
+        # get ancestors as well as column-level associations
+        assoc.ancestors
+      }.flatten.uniq
+    end
+    
+    def utf8?
+      @index.options[:charset_type] =~ /utf-8|zh_cn.utf-8/
+    end
+    
+    def join_dependency_class
+      if rails_3_1?
+        ::ActiveRecord::Associations::JoinDependency
+      else
+        ::ActiveRecord::Associations::ClassMethods::JoinDependency
+      end
+    end
+    
+    def initial_joins
+      if rails_3_1?
+        []
+      else
+        nil
+      end
+    end
+    
+    def rails_3_1?
+      ::ActiveRecord::Associations.constants.include?(:JoinDependency) ||
+      ::ActiveRecord::Associations.constants.include?('JoinDependency')
+    end
+  end
+end

data/lib/thinking_sphinx/tasks.rb

@@ -0,0 +1,132 @@
+require 'fileutils'
+require 'timeout'
+
+namespace :thinking_sphinx do
+  task :app_env do
+    if defined?(Rails)
+      Rails.application.require_environment!
+      Rails.configuration.cache_classes = false
+    elsif defined?(Merb)
+      Rake::Task[:merb_env].invoke
+    elsif defined?(Sinatra)
+      Sinatra::Application.environment = ENV['RACK_ENV']
+    end
+  end
+
+  desc "Output the current Thinking Sphinx version"
+  task :version => :app_env do
+    puts "Thinking Sphinx v" + ThinkingSphinx::Version
+  end
+
+  desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
+  task :running_start => :app_env do
+    Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
+    Rake::Task["thinking_sphinx:start"].invoke
+  end
+
+  desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings"
+  task :start => :app_env do
+    config = ThinkingSphinx::Configuration.instance
+
+    FileUtils.mkdir_p config.searchd_file_path
+    raise RuntimeError, "searchd is already running." if sphinx_running?
+
+    Dir["#{config.searchd_file_path}/*.spl"].each { |file| File.delete(file) }
+
+    config.controller.start
+
+    if sphinx_running?
+      puts "Started successfully (pid #{sphinx_pid})."
+    else
+      puts "Failed to start searchd daemon. Check #{config.searchd_log_file}"
+    end
+  end
+
+  desc "Stop Sphinx using Thinking Sphinx's settings"
+  task :stop => :app_env do
+    unless sphinx_running?
+      puts "searchd is not running"
+    else
+      config = ThinkingSphinx::Configuration.instance
+      pid    = sphinx_pid
+      config.controller.stop
+
+      # Ensure searchd is stopped, but don't try too hard
+      Timeout.timeout(config.stop_timeout) do
+        sleep(1) until config.controller.stop
+      end
+
+      puts "Stopped search daemon (pid #{pid})."
+    end
+  end
+
+  desc "Restart Sphinx"
+  task :restart => [:app_env, :stop, :start]
+
+  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
+  task :configure => :app_env do
+    config = ThinkingSphinx::Configuration.instance
+    puts "Generating Configuration to #{config.config_file}"
+    config.build
+  end
+
+  desc "Index data for Sphinx using Thinking Sphinx's settings"
+  task :index => :app_env do
+    config = ThinkingSphinx::Configuration.instance
+    unless ENV["INDEX_ONLY"] == "true"
+      puts "Generating Configuration to #{config.config_file}"
+      config.build
+    end
+
+    FileUtils.mkdir_p config.searchd_file_path
+    config.controller.index :verbose => true
+  end
+
+  desc "Reindex Sphinx without regenerating the configuration file"
+  task :reindex => :app_env do
+    config = ThinkingSphinx::Configuration.instance
+    FileUtils.mkdir_p config.searchd_file_path
+    output = config.controller.index
+    puts output
+    config.touch_reindex_file(output)
+  end
+
+  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
+  task :rebuild => :app_env do
+    Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
+    Rake::Task["thinking_sphinx:index"].invoke
+    Rake::Task["thinking_sphinx:start"].invoke
+  end
+end
+
+namespace :ts do
+  desc "Output the current Thinking Sphinx version"
+  task :version => "thinking_sphinx:version"
+  desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
+  task :run     => "thinking_sphinx:running_start"
+  desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings"
+  task :start   => "thinking_sphinx:start"
+  desc "Stop Sphinx using Thinking Sphinx's settings"
+  task :stop    => "thinking_sphinx:stop"
+  desc "Index data for Sphinx using Thinking Sphinx's settings"
+  task :in      => "thinking_sphinx:index"
+  task :index   => "thinking_sphinx:index"
+  desc "Reindex Sphinx without regenerating the configuration file"
+  task :reindex => "thinking_sphinx:reindex"
+  desc "Restart Sphinx"
+  task :restart => "thinking_sphinx:restart"
+  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
+  task :conf    => "thinking_sphinx:configure"
+  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
+  task :config  => "thinking_sphinx:configure"
+  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
+  task :rebuild => "thinking_sphinx:rebuild"
+end
+
+def sphinx_pid
+  ThinkingSphinx.sphinx_pid
+end
+
+def sphinx_running?
+  ThinkingSphinx.sphinx_running?
+end

data/lib/thinking_sphinx/test.rb

@@ -0,0 +1,55 @@
+class ThinkingSphinx::Test
+  def self.init(suppress_delta_output = true)
+    set_flags suppress_delta_output
+    create_indexes_folder
+  end
+  
+  def self.start
+    config.build
+    config.controller.index
+    config.controller.start
+  end
+  
+  def self.start_with_autostop
+    autostop
+    start
+  end
+  
+  def self.stop
+    config.controller.stop
+    sleep(0.5) # Ensure Sphinx has shut down completely
+  end
+  
+  def self.autostop
+    Kernel.at_exit do
+      ThinkingSphinx::Test.stop
+    end
+  end
+  
+  def self.run(&block)
+    begin
+      start
+      yield
+    ensure
+      stop
+    end
+  end
+  
+  def self.config
+    @config ||= ::ThinkingSphinx::Configuration.instance
+  end
+  
+  def self.index(*indexes)
+    config.controller.index *indexes
+  end
+  
+  def self.set_flags(suppress_delta_output)
+    ::ThinkingSphinx.deltas_enabled        = true
+    ::ThinkingSphinx.updates_enabled       = true
+    ::ThinkingSphinx.suppress_delta_output = suppress_delta_output
+  end
+  
+  def self.create_indexes_folder
+    FileUtils.mkdir_p config.searchd_file_path
+  end
+end

data/lib/thinking_sphinx/version.rb

@@ -0,0 +1,3 @@
+module ThinkingSphinx
+  Version = '2.0.7'
+end

data/lib/thinking_sphinx.rb

@@ -0,0 +1,296 @@
+require 'thread'
+require 'active_record'
+require 'yaml'
+require 'riddle'
+
+require 'thinking_sphinx/auto_version'
+require 'thinking_sphinx/core/string'
+require 'thinking_sphinx/property'
+require 'thinking_sphinx/active_record'
+require 'thinking_sphinx/association'
+require 'thinking_sphinx/attribute'
+require 'thinking_sphinx/bundled_search'
+require 'thinking_sphinx/configuration'
+require 'thinking_sphinx/context'
+require 'thinking_sphinx/excerpter'
+require 'thinking_sphinx/facet'
+require 'thinking_sphinx/class_facet'
+require 'thinking_sphinx/facet_search'
+require 'thinking_sphinx/field'
+require 'thinking_sphinx/index'
+require 'thinking_sphinx/join'
+require 'thinking_sphinx/source'
+require 'thinking_sphinx/search'
+require 'thinking_sphinx/search_methods'
+require 'thinking_sphinx/deltas'
+
+require 'thinking_sphinx/adapters/abstract_adapter'
+require 'thinking_sphinx/adapters/mysql_adapter'
+require 'thinking_sphinx/adapters/postgresql_adapter'
+
+require 'thinking_sphinx/railtie' if defined?(Rails)
+
+module ThinkingSphinx
+  mattr_accessor :database_adapter
+
+  # A ConnectionError will get thrown when a connection to Sphinx can't be
+  # made.
+  class ConnectionError < StandardError
+  end
+
+  # A StaleIdsException is thrown by Collection.instances_from_matches if there
+  # are records in Sphinx but not in the database, so the search can be retried.
+  class StaleIdsException < StandardError
+    attr_accessor :ids
+    def initialize(ids)
+      self.ids = ids
+    end
+  end
+
+  # A SphinxError occurs when Sphinx responds with an error due to problematic
+  # queries or indexes.
+  class SphinxError < RuntimeError
+    attr_accessor :results
+    def initialize(message = nil, results = nil)
+      super(message)
+      self.results = results
+    end
+  end
+
+  # The collection of indexed models. Keep in mind that Rails lazily loads
+  # its classes, so this may not actually be populated with _all_ the models
+  # that have Sphinx indexes.
+  @@sphinx_mutex          = Mutex.new
+  @@context               = nil
+  @@define_indexes        = true
+  @@deltas_enabled        = nil
+  @@updates_enabled       = nil
+  @@suppress_delta_output = false
+  @@remote_sphinx         = false
+  @@use_group_by_shortcut = nil
+
+  def self.mutex
+    @@sphinx_mutex
+  end
+
+  def self.context
+    if @@context.nil?
+      mutex.synchronize do
+        if @@context.nil?
+          @@context = ThinkingSphinx::Context.new
+          @@context.prepare
+        end
+      end
+    end
+
+    @@context
+  end
+
+  def self.reset_context!(context = nil)
+    mutex.synchronize do
+      @@context = context
+    end
+  end
+
+  def self.unique_id_expression(adapter, offset = nil)
+    "* #{adapter.cast_to_int context.indexed_models.size} + #{offset || 0}"
+  end
+
+  # Check if index definition is disabled.
+  #
+  def self.define_indexes?
+    @@define_indexes
+  end
+
+  # Enable/disable indexes - you may want to do this while migrating data.
+  #
+  #   ThinkingSphinx.define_indexes = false
+  #
+  def self.define_indexes=(value)
+    mutex.synchronize do
+      @@define_indexes = value
+    end
+  end
+
+  # Check if delta indexing is enabled/disabled.
+  #
+  def self.deltas_enabled?
+    if @@deltas_enabled.nil?
+      mutex.synchronize do
+        if @@deltas_enabled.nil?
+          @@deltas_enabled = (
+            ThinkingSphinx::Configuration.environment != "test"
+          )
+        end
+      end
+    end
+
+    @@deltas_enabled && !deltas_suspended?
+  end
+
+  # Enable/disable delta indexing.
+  #
+  #   ThinkingSphinx.deltas_enabled = false
+  #
+  def self.deltas_enabled=(value)
+    mutex.synchronize do
+      @@deltas_enabled = value
+    end
+  end
+
+  # Check if delta indexing is suspended.
+  #
+  def self.deltas_suspended?
+    if Thread.current[:thinking_sphinx_deltas_suspended].nil?
+      Thread.current[:thinking_sphinx_deltas_suspended] = false
+    end
+
+    Thread.current[:thinking_sphinx_deltas_suspended]
+  end
+
+  # Suspend/resume delta indexing.
+  #
+  #   ThinkingSphinx.deltas_suspended = false
+  #
+  def self.deltas_suspended=(value)
+    Thread.current[:thinking_sphinx_deltas_suspended] = value
+  end
+
+  # Check if updates are enabled. True by default, unless within the test
+  # environment.
+  #
+  def self.updates_enabled?
+    if @@updates_enabled.nil?
+      mutex.synchronize do
+        if @@updates_enabled.nil?
+          @@updates_enabled = (
+            ThinkingSphinx::Configuration.environment != "test"
+          )
+        end
+      end
+    end
+
+    @@updates_enabled
+  end
+
+  # Enable/disable updates to Sphinx
+  #
+  #   ThinkingSphinx.updates_enabled = false
+  #
+  def self.updates_enabled=(value)
+    mutex.synchronize do
+      @@updates_enabled = value
+    end
+  end
+
+  def self.suppress_delta_output?
+    @@suppress_delta_output
+  end
+
+  def self.suppress_delta_output=(value)
+    mutex.synchronize do
+      @@suppress_delta_output = value
+    end
+  end
+
+  # Checks to see if MySQL will allow simplistic GROUP BY statements. If not,
+  # or if not using MySQL, this will return false.
+  #
+  def self.use_group_by_shortcut?
+    if @@use_group_by_shortcut.nil?
+      mutex.synchronize do
+        if @@use_group_by_shortcut.nil?
+          @@use_group_by_shortcut = !!(
+            mysql? && ::ActiveRecord::Base.connection.select_all(
+              "SELECT @@global.sql_mode, @@session.sql_mode;"
+            ).all? { |key, value|
+              value.nil? || value[/ONLY_FULL_GROUP_BY/].nil?
+            }
+          )
+        end
+      end
+    end
+
+    @@use_group_by_shortcut
+  end
+
+  def self.reset_use_group_by_shortcut
+    mutex.synchronize do
+      @@use_group_by_shortcut = nil
+    end
+  end
+
+  # An indication of whether Sphinx is running on a remote machine instead of
+  # the same machine.
+  #
+  def self.remote_sphinx?
+    @@remote_sphinx
+  end
+
+  # Tells Thinking Sphinx that Sphinx is running on a different machine, and
+  # thus it can't reliably guess whether it is running or not (ie: the
+  # #sphinx_running? method), and so just assumes it is.
+  #
+  # Useful for multi-machine deployments. Set it in your production.rb file.
+  #
+  #   ThinkingSphinx.remote_sphinx = true
+  #
+  def self.remote_sphinx=(value)
+    mutex.synchronize do
+      @@remote_sphinx = value
+    end
+  end
+
+  # Check if Sphinx is running. If remote_sphinx is set to true (indicating
+  # Sphinx is on a different machine), this will always return true, and you
+  # will have to handle any connection errors yourself.
+  #
+  def self.sphinx_running?
+    remote_sphinx? || sphinx_running_by_pid?
+  end
+
+  # Check if Sphinx is actually running, provided the pid is on the same
+  # machine as this code.
+  #
+  def self.sphinx_running_by_pid?
+    !!sphinx_pid && pid_active?(sphinx_pid)
+  end
+
+  def self.sphinx_pid
+    if File.exists?(ThinkingSphinx::Configuration.instance.pid_file)
+      File.read(ThinkingSphinx::Configuration.instance.pid_file)[/\d+/]
+    else
+      nil
+    end
+  end
+
+  def self.pid_active?(pid)
+    !!Process.kill(0, pid.to_i)
+  rescue Errno::EPERM => e
+    true
+  rescue Exception => e
+    false
+  end
+
+  def self.microsoft?
+    RUBY_PLATFORM =~ /mswin/
+  end
+
+  def self.jruby?
+    defined?(JRUBY_VERSION)
+  end
+
+  def self.mysql?
+    ::ActiveRecord::Base.connection.class.name.demodulize == "MysqlAdapter" ||
+    ::ActiveRecord::Base.connection.class.name.demodulize == "Mysql2Adapter" ||
+    ::ActiveRecord::Base.connection.class.name.demodulize == "MysqlplusAdapter" || (
+      jruby? && ::ActiveRecord::Base.connection.config[:adapter] == "jdbcmysql"
+    )
+  end
+
+  def self.rails_3_1?
+    !!defined?(::ActiveRecord::Associations::CollectionProxy)
+  end
+
+  extend ThinkingSphinx::SearchMethods::ClassMethods
+end