thinking-sphinx 2.1.0 → 3.0.0.pre

data/lib/thinking_sphinx/active_record/sql_source.rb

@@ -0,0 +1,138 @@
+class ThinkingSphinx::ActiveRecord::SQLSource < Riddle::Configuration::SQLSource
+  attr_reader :model, :database_settings, :options
+  attr_accessor :fields, :attributes, :associations, :conditions, :groupings
+
+  # Options:
+  # - :name
+  # - :offset
+  # - :delta_processor
+  # - :delta?
+  # - :disable_range?
+  # - :group_concat_max_len
+  # - :utf8?
+  #
+  def initialize(model, options = {})
+    @model             = model
+    @database_settings = model.connection.instance_variable_get(:@config).clone
+    @options           = options
+
+    @fields            = []
+    @attributes        = []
+    @associations      = []
+    @conditions        = []
+    @groupings         = []
+
+    Template.new(self).apply
+
+    name = "#{options[:name] || model.name.downcase}_#{name_suffix}"
+
+    super name, type
+  end
+
+  def adapter
+    @adapter ||= ThinkingSphinx::ActiveRecord::DatabaseAdapters.
+      adapter_for(@model)
+  end
+
+  def delta_processor
+    options[:delta_processor].try(:new, adapter)
+  end
+
+  def delta?
+    options[:delta?]
+  end
+
+  def disable_range?
+    options[:disable_range?]
+  end
+
+  def facets
+    properties.select(&:facet?)
+  end
+
+  def offset
+    options[:offset]
+  end
+
+  def primary_key
+    options[:primary_key]
+  end
+
+  def render
+    self.class.settings.each do |setting|
+      value = config.settings[setting.to_s]
+      send("#{setting}=", value) unless value.nil?
+    end
+
+    prepare_for_render unless @prepared
+
+    super
+  end
+
+  def type
+    @type ||= case adapter
+    when ThinkingSphinx::ActiveRecord::DatabaseAdapters::MySQLAdapter
+      'mysql'
+    when ThinkingSphinx::ActiveRecord::DatabaseAdapters::PostgreSQLAdapter
+      'pgsql'
+    else
+      raise "Unknown Adapter Type: #{adapter.class.name}"
+    end
+  end
+
+  private
+
+  def attribute_array_for(type)
+    instance_variable_get "@sql_attr_#{type}".to_sym
+  end
+
+  def builder
+    @builder ||= ThinkingSphinx::ActiveRecord::SQLBuilder.new self
+  end
+
+  def config
+    ThinkingSphinx::Configuration.instance
+  end
+
+  def name_suffix
+    delta? ? 'delta' : 'core'
+  end
+
+  def prepare_for_render
+    set_database_settings
+
+    fields.each do |field|
+      @sql_field_string << field.name if field.with_attribute?
+      @sql_file_field   << field.name if field.file?
+    end
+
+    attributes.each do |attribute|
+      presenter = ThinkingSphinx::ActiveRecord::Attribute::SphinxPresenter.new(attribute)
+
+      attribute_array_for(presenter.collection_type) << presenter.declaration
+    end
+
+    @sql_query       = builder.sql_query
+    @sql_query_range = builder.sql_query_range
+    @sql_query_info  = builder.sql_query_info
+    @sql_query_pre  += builder.sql_query_pre
+
+    @prepared = true
+  end
+
+  def properties
+    fields + attributes
+  end
+
+  def set_database_settings
+    @sql_host ||= database_settings[:host]     || 'localhost'
+    @sql_user ||= database_settings[:username] || database_settings[:user] ||
+      ENV['USER']
+    @sql_pass ||= database_settings[:password].to_s.gsub('#', '\#')
+    @sql_db   ||= database_settings[:database]
+    @sql_port ||= database_settings[:port]
+    @sql_sock ||= database_settings[:socket]
+  end
+end
+
+require 'thinking_sphinx/active_record/sql_source/template'

data/lib/thinking_sphinx/active_record/sql_source/template.rb

@@ -0,0 +1,46 @@
+class ThinkingSphinx::ActiveRecord::SQLSource::Template
+  attr_reader :source
+
+  def initialize(source)
+    @source = source
+  end
+
+  def apply
+    add_field class_column, :sphinx_internal_class, :facet => true
+
+    add_attribute :id, :sphinx_internal_id, nil
+    add_attribute '0', :sphinx_deleted,     :integer
+  end
+
+  private
+
+  def add_attribute(column, name, type, options = {})
+    source.attributes << ThinkingSphinx::ActiveRecord::Attribute.new(
+      source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
+      options.merge(:as => name, :type => type)
+    )
+  end
+
+  def add_field(column, name, options = {})
+    source.fields << ThinkingSphinx::ActiveRecord::Field.new(
+      source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
+      options.merge(:as => name)
+    )
+  end
+
+  def class_column
+    if inheriting?
+      source.adapter.convert_nulls model.inheritance_column, "'#{model.name}'"
+    else
+      "'#{model.name}'"
+    end
+  end
+
+  def inheriting?
+    model.column_names.include?(model.inheritance_column)
+  end
+
+  def model
+    source.model
+  end
+end

data/lib/thinking_sphinx/batched_search.rb

@@ -0,0 +1,26 @@
+class ThinkingSphinx::BatchedSearch
+  attr_accessor :searches
+
+  def initialize
+    @searches = []
+  end
+
+  def populate(middleware = ThinkingSphinx::Middlewares::DEFAULT)
+    return if populated? || searches.empty?
+
+    middleware.call contexts
+    searches.each &:populated!
+
+    @populated = true
+  end
+
+  private
+
+  def contexts
+    searches.collect &:context
+  end
+
+  def populated?
+    @populated
+  end
+end

data/lib/thinking_sphinx/callbacks.rb

@@ -0,0 +1,15 @@
+class ThinkingSphinx::Callbacks
+  attr_reader :instance
+
+  def self.callbacks(*methods)
+    mod = Module.new
+    methods.each do |method|
+      mod.send(:define_method, method) { |instance| new(instance).send(method) }
+    end
+    extend mod
+  end
+
+  def initialize(instance)
+    @instance = instance
+  end
+end

data/lib/thinking_sphinx/configuration.rb

@@ -1,361 +1,110 @@
-require 'erb'
-require 'singleton'
+class ThinkingSphinx::Configuration < Riddle::Configuration
+  attr_accessor :configuration_file, :indices_location
 
-module ThinkingSphinx
-  # This class both keeps track of the configuration settings for Sphinx and
-  # also generates the resulting file for Sphinx to use.
-  #
-  # Here are the default settings, relative to Rails.root where relevant:
-  #
-  # config file::           config/#{environment}.sphinx.conf
-  # searchd log file::      log/searchd.log
-  # query log file::        log/searchd.query.log
-  # pid file::              log/searchd.#{environment}.pid
-  # searchd files::         db/sphinx/#{environment}/
-  # address::               127.0.0.1
-  # port::                  9312
-  # allow star::            false
-  # stop timeout::          5
-  # min prefix length::     1
-  # min infix length::      1
-  # mem limit::             64M
-  # max matches::           1000
-  # morphology::            nil
-  # charset type::          utf-8
-  # charset table::         nil
-  # ignore chars::          nil
-  # html strip::            false
-  # html remove elements::  ''
-  # searchd_binary_name::   searchd
-  # indexer_binary_name::   indexer
-  # hard_retry_count::      0
-  #
-  # If you want to change these settings, create a YAML file at
-  # config/sphinx.yml with settings for each environment, in a similar
-  # fashion to database.yml - using the following keys: config_file,
-  # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
-  # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
-  # max_matches, morphology, charset_type, charset_table, ignore_chars,
-  # html_strip, html_remove_elements, delayed_job_priority,
-  # searchd_binary_name, indexer_binary_name.
-  #
-  # I think you've got the idea.
-  #
-  # Each setting in the YAML file is optional - so only put in the ones you
-  # want to change.
-  #
-  # Keep in mind, if for some particular reason you're using a version of
-  # Sphinx older than 0.9.8 r871 (that's prior to the proper 0.9.8 release),
-  # don't set allow_star to true.
-  #
-  class Configuration
-    include Singleton
+  attr_reader :index_paths
 
-    SourceOptions = Riddle::Configuration::SQLSource.settings.map { |setting|
-      setting.to_s
-    } - %w( type sql_query sql_joined_field sql_file_field
-      sql_query_range sql_attr_uint sql_attr_bool sql_attr_bigint sql_query_info
-      sql_attr_timestamp sql_attr_str2ordinal sql_attr_float sql_attr_multi
-      sql_attr_string sql_attr_str2wordcount sql_column_buffers sql_field_string
-      sql_field_str2wordcount )
-    IndexOptions  = Riddle::Configuration::Index.settings.map     { |setting|
-      setting.to_s
-    } - %w( source prefix_fields infix_fields )
-    CustomOptions = %w( disable_range use_64_bit )
+  def initialize
+    super
 
-    attr_accessor :searchd_file_path, :allow_star, :app_root,
-      :model_directories, :delayed_job_priority, :indexed_models, :use_64_bit,
-      :touched_reindex_file, :stop_timeout, :version, :shuffle,
-      :hard_retry_count
+    @configuration_file = Rails.root.join 'config', "#{Rails.env}.sphinx.conf"
+    @index_paths        = [Rails.root.join('app', 'indices')]
+    @indices_location   = Rails.root.join 'db', 'sphinx', Rails.env
 
-    attr_accessor :source_options, :index_options
-    attr_reader :configuration
-    attr_writer :controller
+    searchd.pid_file  = Rails.root.join 'log', "#{Rails.env}.sphinx.pid"
+    searchd.log       = Rails.root.join 'log', "#{Rails.env}.searchd.log"
+    searchd.query_log = Rails.root.join 'log', "#{Rails.env}.searchd.query.log"
+    searchd.binlog_path = Rails.root.join 'tmp', 'binlog', Rails.env
 
-    @@environment = nil
+    searchd.address   = settings['address']
+    searchd.address   = Defaults::ADDRESS unless searchd.address.present?
+    searchd.mysql41   = settings['mysql41'] || settings['port'] ||
+      Defaults::PORT
+    # searchd.workers   = 'threads'
 
-    # Load in the configuration settings - this will look for config/sphinx.yml
-    # and parse it according to the current environment.
-    #
-    def initialize(app_root = Dir.pwd)
-      self.reset
-    end
-
-    def self.configure(&block)
-      yield instance
-      instance.reset(instance.app_root)
-    end
-
-    def reset(custom_app_root=nil)
-      if custom_app_root
-        self.app_root = custom_app_root
-      else
-        self.app_root   = Merb.root                  if defined?(Merb)
-        self.app_root   = Sinatra::Application.root  if defined?(Sinatra)
-        self.app_root   = Rails.root                 if defined?(Rails)
-        self.app_root ||= app_root
-      end
-
-      @controller    = nil
-      @configuration = Riddle::Configuration.new
-      @configuration.searchd.pid_file   = "#{self.app_root}/log/searchd.#{environment}.pid"
-      @configuration.searchd.log        = "#{self.app_root}/log/searchd.log"
-      @configuration.searchd.query_log  = "#{self.app_root}/log/searchd.query.log"
-
-      self.address              = "127.0.0.1"
-      self.port                 = 9312
-      self.searchd_file_path    = "#{self.app_root}/db/sphinx/#{environment}"
-      self.allow_star           = false
-      self.stop_timeout         = 5
-      self.model_directories    = initial_model_directories
-      self.delayed_job_priority = 0
-      self.indexed_models       = []
-      self.shuffle              = false
-      self.hard_retry_count     = 0
-
-      self.source_options  = {}
-      self.index_options   = {
-        :charset_type => "utf-8"
-      }
-
-      self.version = nil
-      parse_config
-      if controller.respond_to?(:sphinx_version)
-        self.version ||= controller.sphinx_version
-      end
-
-      ThinkingSphinx::Attribute::SphinxTypeMappings.merge!(
-        :string => :sql_attr_string
-      ) if Riddle.loaded_version.to_i > 1
-
-      self
-    end
-
-    def self.environment
-      @@environment ||= if defined?(Merb)
-        Merb.environment
-      elsif defined?(Rails)
-        Rails.env
-      elsif defined?(Sinatra)
-        Sinatra::Application.environment.to_s
-      else
-        ENV['RAILS_ENV'] || 'development'
-      end
-    end
-
-    def self.reset_environment
-      ThinkingSphinx.mutex.synchronize do
-        @@environment = nil
-      end
-    end
-
-    def environment
-      self.class.environment
-    end
-
-    def controller
-      @controller ||= Riddle::Controller.new @configuration,
-        "#{self.app_root}/config/#{environment}.sphinx.conf"
-    end
-
-    def generate
-      @configuration.indices.clear
-
-      ThinkingSphinx.context.indexed_models.each do |model|
-        model = model.constantize
-        model.define_indexes
-        @configuration.indices.concat model.to_riddle
-
-        enforce_common_attribute_types
-      end
-    end
-
-    # Generate the config file for Sphinx by using all the settings defined and
-    # looping through all the models with indexes to build the relevant
-    # indexer and searchd configuration, and sources and indexes details.
-    #
-    def build(file_path=nil)
-      file_path ||= "#{self.config_file}"
-
-      generate
-
-      open(file_path, "w") do |file|
-        file.write @configuration.render
-      end
-    end
-
-    def address
-      @address
-    end
+    @offsets = {}
+  end
 
-    def address=(address)
-      @address = address
-      @configuration.searchd.address = address
-    end
+  def self.instance
+    @instance ||= new
+  end
 
-    def port
-      @port
-    end
+  def self.reset
+    @instance = nil
+  end
 
-    def port=(port)
-      @port = port
-      @configuration.searchd.port = port
-    end
+  def connection
+    # If you use localhost, MySQL insists on a socket connection, but Sphinx
+    # requires a TCP connection. Using 127.0.0.1 fixes that.
+    address = searchd.address || '127.0.0.1'
+    address = '127.0.0.1' if address == 'localhost'
+
+    Mysql2::Client.new({
+      :host  => address,
+      :port  => searchd.mysql41,
+      :flags => Mysql2::Client::MULTI_STATEMENTS
+    }.merge(settings['connection_options'] || {}))
+  end
 
-    def use_socket=(use_socket)
-      if use_socket
-        socket = "#{app_root}/tmp/sockets/searchd.#{self.environment}.sock"
-        @configuration.searchd.listen = socket
-        self.address = socket
+  def controller
+    @controller ||= begin
+      rc = Riddle::Controller.new self, configuration_file
+      if settings['bin_path'].present?
+        rc.bin_path = settings['bin_path'].gsub(/([^\/])$/, '\1/')
       end
+      rc
     end
+  end
 
-    def pid_file
-      @configuration.searchd.pid_file
-    end
-
-    def pid_file=(pid_file)
-      @configuration.searchd.pid_file = pid_file
-    end
-
-    def searchd_log_file
-      @configuration.searchd.log
-    end
-
-    def searchd_log_file=(file)
-      @configuration.searchd.log = file
-    end
-
-    def query_log_file
-      @configuration.searchd.query_log
-    end
-
-    def query_log_file=(file)
-      @configuration.searchd.query_log = file
-    end
-
-    def config_file
-      controller.path
-    end
-
-    def config_file=(file)
-      controller.path = file
-    end
-
-    def bin_path
-      controller.bin_path
-    end
-
-    def bin_path=(path)
-      controller.bin_path = path
-    end
-
-    def searchd_binary_name
-      controller.searchd_binary_name
-    end
-
-    def searchd_binary_name=(name)
-      controller.searchd_binary_name = name
-    end
-
-    def indexer_binary_name
-      controller.indexer_binary_name
-    end
+  def indices_for_references(*references)
+    preload_indices
+    indices.select { |index| references.include?(index.reference) }
+  end
 
-    def indexer_binary_name=(name)
-      controller.indexer_binary_name = name
-    end
+  def next_offset(reference)
+    @offsets[reference] ||= @offsets.keys.count
+  end
 
-    attr_accessor :timeout
+  def preload_indices
+    return if @preloaded_indices
 
-    def models_by_crc
-      @models_by_crc ||= begin
-        ThinkingSphinx.context.indexed_models.inject({}) do |hash, model|
-          hash[model.constantize.to_crc32] = model
-          model.constantize.descendants.each { |subclass|
-            hash[subclass.to_crc32] = subclass.name
-          }
-          hash
-        end
+    index_paths.each do |path|
+      Dir["#{path}/**/*.rb"].each do |file|
+        ActiveSupport::Dependencies.require_or_load file
       end
     end
 
-    def touch_reindex_file(output)
-      return FileUtils.touch(@touched_reindex_file) if @touched_reindex_file and output =~ /succesfully sent SIGHUP to searchd/
-      false
-    end
-
-    private
-
-    # Parse the config/sphinx.yml file - if it exists - then use the attribute
-    # accessors to set the appropriate values. Nothing too clever.
-    #
-    def parse_config
-      path = "#{app_root}/config/sphinx.yml"
-      return unless File.exists?(path)
-
-      conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
-
-      conf.each do |key,value|
-        self.send("#{key}=", value) if self.respond_to?("#{key}=")
-
-        set_sphinx_setting self.source_options, key, value, SourceOptions
-        set_sphinx_setting self.index_options,  key, value, IndexOptions
-        set_sphinx_setting self.index_options,  key, value, CustomOptions
-        set_sphinx_setting @configuration.searchd, key, value
-        set_sphinx_setting @configuration.indexer, key, value
-      end unless conf.nil?
-
-      self.bin_path += '/' unless self.bin_path.blank?
+    @preloaded_indices = true
+  end
 
-      if self.allow_star
-        self.index_options[:enable_star]    = true
-        self.index_options[:min_prefix_len] = 1
-      end
-    end
+  def render
+    preload_indices
 
-    def set_sphinx_setting(object, key, value, allowed = {})
-      if object.is_a?(Hash)
-        object[key.to_sym] = value if allowed.include?(key.to_s)
-      else
-        object.send("#{key}=", value) if object.respond_to?("#{key}")
-        send("#{key}=", value) if self.respond_to?("#{key}")
-      end
-    end
+    ThinkingSphinx::Configuration::ConsistentIds.new(indices).reconcile
 
-    def enforce_common_attribute_types
-      sql_indexes = configuration.indices.reject { |index|
-        index.is_a? Riddle::Configuration::DistributedIndex
-      }
+    super
+  end
 
-      return unless sql_indexes.any? { |index|
-        index.sources.any? { |source|
-          source.sql_attr_bigint.include? :sphinx_internal_id
-        }
-      }
+  def render_to_file
+    FileUtils.mkdir_p searchd.binlog_path
 
-      sql_indexes.each { |index|
-        index.sources.each { |source|
-          next if source.sql_attr_bigint.include? :sphinx_internal_id
+    open(configuration_file, 'w') { |file| file.write render }
+  end
 
-          source.sql_attr_bigint << :sphinx_internal_id
-          source.sql_attr_uint.delete :sphinx_internal_id
-        }
-      }
-    end
+  def settings
+    @settings ||= File.exists?(settings_file) ? settings_to_hash : {}
+  end
 
-    def initial_model_directories
-      directories = ["#{app_root}/app/models/"] +
-        Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
+  private
 
-      if defined?(Rails) && Rails.application
-        directories += Rails.application.paths['app/models'].to_a
-        directories += Rails.application.railties.engines.collect { |engine|
-          engine.paths['app/models'].to_a
-	      }.flatten
-      end
+  def settings_to_hash
+    contents = YAML.load(ERB.new(File.read(settings_file)).result)
+    contents && contents[Rails.env] || {}
+  end
 
-      directories
-    end
+  def settings_file
+    Rails.root.join 'config', 'thinking_sphinx.yml'
   end
 end
+
+require 'thinking_sphinx/configuration/consistent_ids'
+require 'thinking_sphinx/configuration/defaults'