warp-thinking-sphinx 1.2.12

data/lib/thinking_sphinx/attribute.rb

@@ -0,0 +1,340 @@
+module ThinkingSphinx
+  # Attributes - eternally useful when it comes to filtering, sorting or
+  # grouping. This class isn't really useful to you unless you're hacking
+  # around with the internals of Thinking Sphinx - but hey, don't let that
+  # stop you.
+  #
+  # One key thing to remember - if you're using the attribute manually to
+  # generate SQL statements, you'll need to set the base model, and all the
+  # associations. Which can get messy. Use Index.link!, it really helps.
+  #
+  class Attribute < ThinkingSphinx::Property
+    attr_accessor :query_source
+
+    # To create a new attribute, you'll need to pass in either a single Column
+    # or an array of them, and some (optional) options.
+    #
+    # Valid options are:
+    # - :as     => :alias_name
+    # - :type   => :attribute_type
+    # - :source => :field, :query, :ranged_query
+    #
+    # Alias is only required in three circumstances: when there's
+    # another attribute or field with the same name, when the column name is
+    # 'id', or when there's more than one column.
+    #
+    # Type is not required, unless you want to force a column to be a certain
+    # type (but keep in mind the value will not be CASTed in the SQL
+    # statements). The only time you really need to use this is when the type
+    # can't be figured out by the column - ie: when not actually using a
+    # database column as your source.
+    #
+    # Source is only used for multi-value attributes (MVA). By default this will
+    # use a left-join and a group_concat to obtain the values. For better performance
+    # during indexing it can be beneficial to let Sphinx use a separate query to retrieve
+    # all document,value-pairs.
+    # Either :query or :ranged_query will enable this feature, where :ranged_query will cause
+    # the query to be executed incremental.
+    #
+    # Example usage:
+    #
+    #   Attribute.new(
+    #     Column.new(:created_at)
+    #   )
+    #
+    #   Attribute.new(
+    #     Column.new(:posts, :id),
+    #     :as => :post_ids
+    #   )
+    #
+    #   Attribute.new(
+    #     Column.new(:posts, :id),
+    #     :as => :post_ids,
+    #     :source => :ranged_query
+    #   )
+    #
+    #   Attribute.new(
+    #     [Column.new(:pages, :id), Column.new(:articles, :id)],
+    #     :as => :content_ids
+    #   )
+    #
+    #   Attribute.new(
+    #     Column.new("NOW()"),
+    #     :as   => :indexed_at,
+    #     :type => :datetime
+    #   )
+    #
+    # If you're creating attributes for latitude and longitude, don't forget
+    # that Sphinx expects these values to be in radians.
+    #
+    def initialize(source, columns, options = {})
+      super
+
+      @type           = options[:type]
+      @query_source   = options[:source]
+      @crc            = options[:crc]
+
+      @type         ||= :multi    unless @query_source.nil?
+      if @type == :string && @crc
+        @type = is_many? ? :multi : :integer
+      end
+
+      source.attributes << self
+    end
+
+    # Get the part of the SELECT clause related to this attribute. Don't forget
+    # to set your model and associations first though.
+    #
+    # This will concatenate strings and arrays of integers, and convert
+    # datetimes to timestamps, as needed.
+    #
+    def to_select_sql
+      return nil unless include_as_association?
+
+      separator = all_ints? || all_datetimes? || @crc ? ',' : ' '
+
+      clause = @columns.collect { |column|
+        part = column_with_prefix(column)
+        case type
+        when :string
+          adapter.convert_nulls(part)
+        when :datetime
+          adapter.cast_to_datetime(part)
+        when :multi
+          part = adapter.cast_to_datetime(part) if is_many_datetimes?
+          adapter.convert_nulls(part, 0)
+        else
+          part
+        end
+      }.join(', ')
+
+      # clause = adapter.cast_to_datetime(clause)             if type == :datetime
+      clause = adapter.crc(clause)                          if @crc
+      clause = adapter.concatenate(clause, separator)       if concat_ws?
+
+      "#{clause} AS #{quote_column(unique_name)}"
+    end
+
+    def type_to_config
+      {
+        :multi    => :sql_attr_multi,
+        :datetime => :sql_attr_timestamp,
+        :string   => :sql_attr_str2ordinal,
+        :float    => :sql_attr_float,
+        :boolean  => :sql_attr_bool,
+        :integer  => :sql_attr_uint
+      }[type]
+    end
+
+    def include_as_association?
+      ! (type == :multi && (query_source == :query || query_source == :ranged_query))
+    end
+
+    # Returns the configuration value that should be used for
+    # the attribute.
+    # Special case is the multi-valued attribute that needs some
+    # extra configuration.
+    #
+    def config_value(offset = nil, delta = false)
+      if type == :multi
+        multi_config = include_as_association? ? "field" :
+          source_value(offset, delta).gsub(/\s+/m, " ").strip
+        "uint #{unique_name} from #{multi_config}"
+      else
+        unique_name
+      end
+    end
+
+    # Returns the type of the column. If that's not already set, it returns
+    # :multi if there's the possibility of more than one value, :string if
+    # there's more than one association, otherwise it figures out what the
+    # actual column's datatype is and returns that.
+    #
+    def type
+      @type ||= begin
+        base_type = case
+        when is_many?, is_many_ints?
+          :multi
+        when @associations.values.flatten.length > 1
+          :string
+        else
+          translated_type_from_database
+        end
+
+        if base_type == :string && @crc
+          base_type = :integer
+        else
+          @crc = false unless base_type == :multi && is_many_strings? && @crc
+        end
+
+        base_type
+      end
+    end
+
+    def updatable?
+      [:integer, :datetime, :boolean].include?(type) && !is_string?
+    end
+
+    def live_value(instance)
+      object = instance
+      column = @columns.first
+      column.__stack.each { |method| object = object.send(method) }
+      object.send(column.__name)
+    end
+
+    def all_ints?
+      all_of_type?(:integer)
+    end
+
+    def all_datetimes?
+      all_of_type?(:datetime, :date, :timestamp)
+    end
+
+    def all_strings?
+      all_of_type?(:string, :text)
+    end
+
+    private
+
+    def source_value(offset, delta)
+      if is_string?
+        return "#{query_source.to_s.dasherize}; #{columns.first.__name}"
+      end
+
+      query = query(offset)
+
+      if query_source == :ranged_query
+        query += query_clause
+        query += " AND #{query_delta.strip}" if delta
+        "ranged-query; #{query}; #{range_query}"
+      else
+        query += "WHERE #{query_delta.strip}" if delta
+        "query; #{query}"
+      end
+    end
+
+    def query(offset)
+      base_assoc = base_association_for_mva
+      end_assoc  = end_association_for_mva
+      raise "Could not determine SQL for MVA" if base_assoc.nil?
+
+      <<-SQL
+SELECT #{foreign_key_for_mva base_assoc}
+  #{ThinkingSphinx.unique_id_expression(offset)} AS #{quote_column('id')},
+  #{primary_key_for_mva(end_assoc)} AS #{quote_column(unique_name)}
+FROM #{quote_table_name base_assoc.table} #{association_joins}
+      SQL
+    end
+
+    def query_clause
+      foreign_key = foreign_key_for_mva base_association_for_mva
+      "WHERE #{foreign_key} >= $start AND #{foreign_key} <= $end"
+    end
+
+    def query_delta
+      foreign_key = foreign_key_for_mva base_association_for_mva
+      <<-SQL
+#{foreign_key} IN (SELECT #{quote_column model.primary_key}
+FROM #{model.quoted_table_name}
+WHERE #{@source.index.delta_object.clause(model, true)})
+      SQL
+    end
+
+    def range_query
+      assoc       = base_association_for_mva
+      foreign_key = foreign_key_for_mva assoc
+      "SELECT MIN(#{foreign_key}), MAX(#{foreign_key}) FROM #{quote_table_name assoc.table}"
+    end
+
+    def primary_key_for_mva(assoc)
+      quote_with_table(
+        assoc.table, assoc.primary_key_from_reflection || columns.first.__name
+      )
+    end
+
+    def foreign_key_for_mva(assoc)
+      quote_with_table assoc.table, assoc.reflection.primary_key_name
+    end
+
+    def end_association_for_mva
+      @association_for_mva ||= associations[columns.first].detect { |assoc|
+        assoc.has_column?(columns.first.__name)
+      }
+    end
+
+    def base_association_for_mva
+      @first_association_for_mva ||= begin
+        assoc = end_association_for_mva
+        while !assoc.parent.nil?
+          assoc = assoc.parent
+        end
+
+        assoc
+      end
+    end
+
+    def association_joins
+      joins = []
+      assoc = end_association_for_mva
+      while assoc != base_association_for_mva
+        joins << assoc.to_sql
+        assoc = assoc.parent
+      end
+
+      joins.join(' ')
+    end
+
+    def is_many_ints?
+      concat_ws? && all_ints?
+    end
+
+    def is_many_datetimes?
+      is_many? && all_datetimes?
+    end
+
+    def is_many_strings?
+      is_many? && all_strings?
+    end
+
+    def type_from_database
+      klass = @associations.values.flatten.first ?
+        @associations.values.flatten.first.reflection.klass : @model
+
+      column = klass.columns.detect { |col|
+        @columns.collect { |c| c.__name.to_s }.include? col.name
+      }
+      column.nil? ? nil : column.type
+    end
+
+    def translated_type_from_database
+      case type_from_db = type_from_database
+      when :datetime, :string, :float, :boolean, :integer
+        type_from_db
+      when :decimal
+        :float
+      when :timestamp, :date
+        :datetime
+      else
+        raise <<-MESSAGE
+
+Cannot automatically map attribute #{unique_name} in #{@model.name} to an
+equivalent Sphinx type (integer, float, boolean, datetime, string as ordinal).
+You could try to explicitly convert the column's value in your define_index
+block:
+  has "CAST(column AS INT)", :type => :integer, :as => :column
+        MESSAGE
+      end
+    end
+
+    def all_of_type?(*column_types)
+      @columns.all? { |col|
+        klasses = @associations[col].empty? ? [@model] :
+          @associations[col].collect { |assoc| assoc.reflection.klass }
+        klasses.all? { |klass|
+          column = klass.columns.detect { |column| column.name == col.__name.to_s }
+          !column.nil? && column_types.include?(column.type)
+        }
+      }
+    end
+  end
+end

data/lib/thinking_sphinx/class_facet.rb

@@ -0,0 +1,15 @@
+module ThinkingSphinx
+  class ClassFacet < ThinkingSphinx::Facet
+    def name
+      :class
+    end
+    
+    def attribute_name
+      "class_crc"
+    end
+    
+    def value(object, attribute_value)
+      object.class.name
+    end
+  end
+end

data/lib/thinking_sphinx/configuration.rb

@@ -0,0 +1,282 @@
+require 'erb'
+require 'singleton'
+
+module ThinkingSphinx
+  # This class both keeps track of the configuration settings for Sphinx and
+  # also generates the resulting file for Sphinx to use.
+  # 
+  # Here are the default settings, relative to RAILS_ROOT where relevant:
+  #
+  # config file::           config/#{environment}.sphinx.conf
+  # searchd log file::      log/searchd.log
+  # query log file::        log/searchd.query.log
+  # pid file::              log/searchd.#{environment}.pid
+  # searchd files::         db/sphinx/#{environment}/
+  # address::               127.0.0.1
+  # port::                  3312
+  # allow star::            false
+  # min prefix length::     1
+  # min infix length::      1
+  # mem limit::             64M
+  # max matches::           1000
+  # morphology::            nil
+  # charset type::          utf-8
+  # charset table::         nil
+  # ignore chars::          nil
+  # html strip::            false
+  # html remove elements::  ''
+  # searchd_binary_name::   searchd
+  # indexer_binary_name::   indexer
+  #
+  # If you want to change these settings, create a YAML file at
+  # config/sphinx.yml with settings for each environment, in a similar
+  # fashion to database.yml - using the following keys: config_file,
+  # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
+  # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
+  # max_matches, morphology, charset_type, charset_table, ignore_chars,
+  # html_strip, html_remove_elements, delayed_job_priority,
+  # searchd_binary_name, indexer_binary_name.
+  #
+  # I think you've got the idea.
+  # 
+  # Each setting in the YAML file is optional - so only put in the ones you
+  # want to change.
+  #
+  # Keep in mind, if for some particular reason you're using a version of
+  # Sphinx older than 0.9.8 r871 (that's prior to the proper 0.9.8 release),
+  # don't set allow_star to true.
+  # 
+  class Configuration
+    include Singleton
+    
+    SourceOptions = %w( mysql_connect_flags sql_range_step sql_query_pre
+      sql_query_post sql_ranged_throttle sql_query_post_index )
+    
+    IndexOptions  = %w( charset_table charset_type docinfo enable_star
+      exceptions html_index_attrs html_remove_elements html_strip ignore_chars
+      min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars
+      ngram_len phrase_boundary phrase_boundary_step preopen stopwords
+      wordforms )
+    
+    CustomOptions = %w( disable_range )
+        
+    attr_accessor :config_file, :searchd_log_file, :query_log_file,
+      :pid_file, :searchd_file_path, :address, :port, :allow_star,
+      :database_yml_file, :app_root, :bin_path, :model_directories,
+      :delayed_job_priority, :searchd_binary_name, :indexer_binary_name
+    
+    attr_accessor :source_options, :index_options
+    
+    attr_reader :environment, :configuration
+    
+    # Load in the configuration settings - this will look for config/sphinx.yml
+    # and parse it according to the current environment.
+    # 
+    def initialize(app_root = Dir.pwd)
+      self.reset
+    end
+    
+    def self.configure(&block)
+      yield instance
+      instance.reset(instance.app_root)
+    end
+    
+    def reset(custom_app_root=nil)
+      if custom_app_root
+        self.app_root = custom_app_root
+      else
+        self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
+        self.app_root          = Merb.root  if defined?(Merb)
+        self.app_root        ||= app_root
+      end
+      
+      @configuration = Riddle::Configuration.new
+      @configuration.searchd.address    = "127.0.0.1"
+      @configuration.searchd.port       = 3312
+      @configuration.searchd.pid_file   = "#{self.app_root}/log/searchd.#{environment}.pid"
+      @configuration.searchd.log        = "#{self.app_root}/log/searchd.log"
+      @configuration.searchd.query_log  = "#{self.app_root}/log/searchd.query.log"
+      
+      self.database_yml_file    = "#{self.app_root}/config/database.yml"
+      self.config_file          = "#{self.app_root}/config/#{environment}.sphinx.conf"
+      self.searchd_file_path    = "#{self.app_root}/db/sphinx/#{environment}"
+      self.allow_star           = false
+      self.bin_path             = ""
+      self.model_directories    = ["#{app_root}/app/models/"] +
+        Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
+      self.delayed_job_priority = 0
+      
+      self.source_options  = {}
+      self.index_options   = {
+        :charset_type => "utf-8"
+      }
+      
+      self.searchd_binary_name = "searchd"
+      self.indexer_binary_name = "indexer"
+            
+      parse_config
+      
+      self
+    end
+    
+    def self.environment
+      @@environment ||= (
+        defined?(Merb) ? Merb.environment : ENV['RAILS_ENV']
+      ) || "development"
+    end
+    
+    def environment
+      self.class.environment
+    end
+    
+    def controller
+      @controller ||= Riddle::Controller.new(@configuration, self.config_file)
+    end
+    
+    # Generate the config file for Sphinx by using all the settings defined and
+    # looping through all the models with indexes to build the relevant
+    # indexer and searchd configuration, and sources and indexes details.
+    #
+    def build(file_path=nil)
+      load_models
+      file_path ||= "#{self.config_file}"
+      
+      @configuration.indexes.clear
+      
+      ThinkingSphinx.indexed_models.each_with_index do |model, model_index|
+        @configuration.indexes.concat model.constantize.to_riddle(model_index)
+      end
+      
+      open(file_path, "w") do |file|
+        file.write @configuration.render
+      end
+    end
+    
+    # Make sure all models are loaded - without reloading any that
+    # ActiveRecord::Base is already aware of (otherwise we start to hit some
+    # messy dependencies issues).
+    # 
+    def load_models
+      return if defined?(Rails) &&
+        Rails::VERSION::STRING.to_f > 2.1 &&
+        Rails.configuration.cache_classes
+      
+      self.model_directories.each do |base|
+        Dir["#{base}**/*.rb"].each do |file|
+          model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
+        
+          next if model_name.nil?
+          next if ::ActiveRecord::Base.send(:subclasses).detect { |model|
+            model.name == model_name
+          }
+        
+          begin
+            model_name.camelize.constantize
+          rescue LoadError
+            model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
+          rescue NameError
+            next
+          rescue StandardError
+            puts "Warning: Error loading #{file}"
+          end
+        end
+      end
+    end
+    
+    def address
+      @configuration.searchd.address
+    end
+    
+    def address=(address)
+      @configuration.searchd.address = address
+    end
+    
+    def port
+      @configuration.searchd.port
+    end
+    
+    def port=(port)
+      @configuration.searchd.port = port
+    end
+    
+    def pid_file
+      @configuration.searchd.pid_file
+    end
+    
+    def pid_file=(pid_file)
+      @configuration.searchd.pid_file = pid_file
+    end
+    
+    def searchd_log_file
+      @configuration.searchd.log
+    end
+    
+    def searchd_log_file=(file)
+      @configuration.searchd.log = file
+    end
+    
+    def query_log_file
+      @configuration.searchd.query_log
+    end
+    
+    def query_log_file=(file)
+      @configuration.searchd.query_log = file
+    end
+    
+    def client
+      client = Riddle::Client.new address, port
+      client.max_matches = configuration.searchd.max_matches || 1000
+      client
+    end
+    
+    def models_by_crc
+      @models_by_crc ||= begin
+        ThinkingSphinx.indexed_models.inject({}) do |hash, model|
+          hash[model.constantize.to_crc32] = model
+          Object.subclasses_of(model.constantize).each { |subclass|
+            hash[subclass.to_crc32] = subclass.name
+          }
+          hash
+        end
+      end
+    end
+    
+    private
+    
+    # Parse the config/sphinx.yml file - if it exists - then use the attribute
+    # accessors to set the appropriate values. Nothing too clever.
+    # 
+    def parse_config
+      path = "#{app_root}/config/sphinx.yml"
+      return unless File.exists?(path)
+      
+      conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
+      
+      conf.each do |key,value|
+        self.send("#{key}=", value) if self.respond_to?("#{key}=")
+        
+        set_sphinx_setting self.source_options, key, value, SourceOptions
+        set_sphinx_setting self.index_options,  key, value, IndexOptions
+        set_sphinx_setting self.index_options,  key, value, CustomOptions
+        set_sphinx_setting @configuration.searchd, key, value
+        set_sphinx_setting @configuration.indexer, key, value
+      end unless conf.nil?
+      
+      self.bin_path += '/' unless self.bin_path.blank?
+      
+      if self.allow_star
+        self.index_options[:enable_star]    = true
+        self.index_options[:min_prefix_len] = 1
+      end
+    end
+    
+    def set_sphinx_setting(object, key, value, allowed = {})
+      if object.is_a?(Hash)
+        object[key.to_sym] = value if allowed.include?(key.to_s)
+      else
+        object.send("#{key}=", value) if object.respond_to?("#{key}")
+        send("#{key}=", value) if self.respond_to?("#{key}")
+      end
+    end
+  end
+end